scanoss 0.2.28 → 0.3.0

package/build/module/lib/scanner/ScannerCfg.js

@@ -12,10 +12,10 @@ export class ScannerCfg {
     WFP_FILE_MAX_SIZE = 64 * 1000;
     MAX_RETRIES_FOR_RECOVERABLES_ERRORS = 5;
     ABORT_ON_MAX_RETRIES = true;
-    // Persist results after [ X ] number of server responses.
+    // Persist results after [ X ] server responses
     MAX_RESPONSES_IN_BUFFER = 300;
     DISPATCHER_QUEUE_SIZE_MAX_LIMIT = 300;
     DISPATCHER_QUEUE_SIZE_MIN_LIMIT = 200;
 }
 ;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiU2Nhbm5lckNmZy5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uLy4uL3NyYy9saWIvc2Nhbm5lci9TY2FubmVyQ2ZnLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE1BQU0sT0FBTyxVQUFVO0lBQ3JCLG1CQUFtQjtJQUNuQixnQkFBZ0IsR0FBRyxZQUFZLENBQUE7SUFFL0IsVUFBVTtJQUNWLE9BQU8sR0FBRyxtQ0FBbUMsQ0FBQztJQUU5QyxPQUFPLEdBQUcsRUFBRSxDQUFDO0lBRWIsdUJBQXVCO0lBQ3ZCLGlCQUFpQixHQUFHLEVBQUUsQ0FBQztJQUV2QiwrQkFBK0I7SUFDL0IsT0FBTyxHQUFHLEtBQUssQ0FBQztJQUVoQixzQ0FBc0M7SUFDdEMsaUJBQWlCLEdBQUcsRUFBRSxHQUFHLElBQUksQ0FBQztJQUU5QixtQ0FBbUMsR0FBRyxDQUFDLENBQUM7SUFFeEMsb0JBQW9CLEdBQUcsSUFBSSxDQUFDO0lBRTVCLDBEQUEwRDtJQUMxRCx1QkFBdUIsR0FBRyxHQUFHLENBQUM7SUFFOUIsK0JBQStCLEdBQUcsR0FBRyxDQUFDO0lBRXRDLCtCQUErQixHQUFHLEdBQUcsQ0FBQztDQUN2QztBQUFBLENBQUMifQ==
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiU2Nhbm5lckNmZy5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uLy4uL3NyYy9saWIvc2Nhbm5lci9TY2FubmVyQ2ZnLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE1BQU0sT0FBTyxVQUFVO0lBQ3JCLG1CQUFtQjtJQUNuQixnQkFBZ0IsR0FBRyxZQUFZLENBQUE7SUFFL0IsVUFBVTtJQUNWLE9BQU8sR0FBRyxtQ0FBbUMsQ0FBQztJQUU5QyxPQUFPLEdBQUcsRUFBRSxDQUFDO0lBRWIsdUJBQXVCO0lBQ3ZCLGlCQUFpQixHQUFHLEVBQUUsQ0FBQztJQUV2QiwrQkFBK0I7SUFDL0IsT0FBTyxHQUFHLEtBQUssQ0FBQztJQUVoQixzQ0FBc0M7SUFDdEMsaUJBQWlCLEdBQUcsRUFBRSxHQUFHLElBQUksQ0FBQztJQUU5QixtQ0FBbUMsR0FBRyxDQUFDLENBQUM7SUFFeEMsb0JBQW9CLEdBQUcsSUFBSSxDQUFDO0lBRTVCLCtDQUErQztJQUMvQyx1QkFBdUIsR0FBRyxHQUFHLENBQUM7SUFFOUIsK0JBQStCLEdBQUcsR0FBRyxDQUFDO0lBRXRDLCtCQUErQixHQUFHLEdBQUcsQ0FBQztDQUV2QztBQUFBLENBQUMifQ==

package/build/module/lib/scanner/ScannerTypes.d.ts

@@ -22,6 +22,7 @@ export declare enum ScannerEvents {
 }
 export declare enum WinnowingMode {
     FULL_WINNOWING = "FULL_WINNOWING",
+    FULL_WINNOWING_HPSM = "FULL_WINNOWING_HPSM",
     WINNOWING_ONLY_MD5 = "WINNOWING_ONLY_MD5"
 }
 export declare enum SbomMode {

package/build/module/lib/scanner/ScannerTypes.js

@@ -25,6 +25,7 @@ export var ScannerEvents;
 export var WinnowingMode;
 (function (WinnowingMode) {
     WinnowingMode["FULL_WINNOWING"] = "FULL_WINNOWING";
+    WinnowingMode["FULL_WINNOWING_HPSM"] = "FULL_WINNOWING_HPSM";
     WinnowingMode["WINNOWING_ONLY_MD5"] = "WINNOWING_ONLY_MD5";
 })(WinnowingMode || (WinnowingMode = {}));
 ;
@@ -34,4 +35,4 @@ export var SbomMode;
     SbomMode["SBOM_IDENTIFY"] = "identify";
 })(SbomMode || (SbomMode = {}));
 ;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiU2Nhbm5lclR5cGVzLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vLi4vc3JjL2xpYi9zY2FubmVyL1NjYW5uZXJUeXBlcy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxNQUFNLENBQU4sSUFBWSxhQTRCWDtBQTVCRCxXQUFZLGFBQWE7SUFDdkIsMERBQXlDLENBQUE7SUFDekMsZ0VBQStDLENBQUE7SUFDL0MsMERBQXlDLENBQUE7SUFDekMsOENBQTZCLENBQUE7SUFFN0IsZ0VBQStDLENBQUE7SUFDL0MsNERBQTJDLENBQUE7SUFDM0MsNERBQTJDLENBQUE7SUFDM0MsZ0ZBQStELENBQUE7SUFDL0QsMEVBQXlELENBQUE7SUFDekQsb0ZBQW1FLENBQUE7SUFDbkUsa0RBQWlDLENBQUE7SUFFakMsZ0VBQStDLENBQUE7SUFFL0Msd0RBQXVDLENBQUE7SUFDdkMsd0RBQXVDLENBQUE7SUFFdkMsd0RBQXVDLENBQUE7SUFDdkMsb0RBQW1DLENBQUE7SUFFbkMsd0NBQXVCLENBQUE7SUFDdkIsc0RBQXFDLENBQUE7SUFFckMsNENBQTJCLENBQUE7SUFFM0IsZ0NBQWUsQ0FBQTtBQUNqQixDQUFDLEVBNUJXLGFBQWEsS0FBYixhQUFhLFFBNEJ4QjtBQUFBLENBQUM7QUFFRixNQUFNLENBQU4sSUFBWSxhQUdYO0FBSEQsV0FBWSxhQUFhO0lBQ3ZCLGtEQUFpQyxDQUFBO0lBQ2pDLDBEQUF5QyxDQUFBO0FBQzNDLENBQUMsRUFIVyxhQUFhLEtBQWIsYUFBYSxRQUd4QjtBQUFBLENBQUM7QUFFRixNQUFNLENBQU4sSUFBWSxRQUdYO0FBSEQsV0FBWSxRQUFRO0lBQ2xCLHFDQUF5QixDQUFBO0lBQ3pCLHNDQUEwQixDQUFBO0FBQzVCLENBQUMsRUFIVyxRQUFRLEtBQVIsUUFBUSxRQUduQjtBQVVBLENBQUMifQ==
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiU2Nhbm5lclR5cGVzLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vLi4vc3JjL2xpYi9zY2FubmVyL1NjYW5uZXJUeXBlcy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxNQUFNLENBQU4sSUFBWSxhQTRCWDtBQTVCRCxXQUFZLGFBQWE7SUFDdkIsMERBQXlDLENBQUE7SUFDekMsZ0VBQStDLENBQUE7SUFDL0MsMERBQXlDLENBQUE7SUFDekMsOENBQTZCLENBQUE7SUFFN0IsZ0VBQStDLENBQUE7SUFDL0MsNERBQTJDLENBQUE7SUFDM0MsNERBQTJDLENBQUE7SUFDM0MsZ0ZBQStELENBQUE7SUFDL0QsMEVBQXlELENBQUE7SUFDekQsb0ZBQW1FLENBQUE7SUFDbkUsa0RBQWlDLENBQUE7SUFFakMsZ0VBQStDLENBQUE7SUFFL0Msd0RBQXVDLENBQUE7SUFDdkMsd0RBQXVDLENBQUE7SUFFdkMsd0RBQXVDLENBQUE7SUFDdkMsb0RBQW1DLENBQUE7SUFFbkMsd0NBQXVCLENBQUE7SUFDdkIsc0RBQXFDLENBQUE7SUFFckMsNENBQTJCLENBQUE7SUFFM0IsZ0NBQWUsQ0FBQTtBQUNqQixDQUFDLEVBNUJXLGFBQWEsS0FBYixhQUFhLFFBNEJ4QjtBQUFBLENBQUM7QUFFRixNQUFNLENBQU4sSUFBWSxhQUlYO0FBSkQsV0FBWSxhQUFhO0lBQ3ZCLGtEQUFpQyxDQUFBO0lBQ2pDLDREQUEyQyxDQUFBO0lBQzNDLDBEQUF5QyxDQUFBO0FBQzNDLENBQUMsRUFKVyxhQUFhLEtBQWIsYUFBYSxRQUl4QjtBQUFBLENBQUM7QUFFRixNQUFNLENBQU4sSUFBWSxRQUdYO0FBSEQsV0FBWSxRQUFRO0lBQ2xCLHFDQUF5QixDQUFBO0lBQ3pCLHNDQUEwQixDQUFBO0FBQzVCLENBQUMsRUFIVyxRQUFRLEtBQVIsUUFBUSxRQUduQjtBQVVBLENBQUMifQ==

package/build/module/lib/scanner/WfpProvider/WfpCalculator/WfpCalculator.js

@@ -16,7 +16,13 @@ parentPort.on('message', async (scannableItem) => {
       scannableItem.contentSource,
       scannableItem.maxSizeWfp
     );
-  } else {
+  } else if (scannableItem.winnowingMode === "FULL_WINNOWING_HPSM") {
+    fingerprint = wfp_hpsm_for_content(
+      scannableItem.content,
+      scannableItem.contentSource,
+      scannableItem.maxSizeWfp
+    );
+  } else if (scannableItem.winnowingMode === "WINNOWING_ONLY_MD5") {
     fingerprint = wfp_only_md5(
       scannableItem.content,
       scannableItem.contentSource
@@ -24,13 +30,15 @@ parentPort.on('message', async (scannableItem) => {
   }
 
   scannableItem.fingerprint = fingerprint;
-
   parentPort.postMessage(scannableItem);
-});
 
+  });
 
-const crypto = require('crypto');
+  /**
+  * Winnowing and HPSM algorithm begins
+  */
 
+const crypto = require('crypto');
 const isWin = process.platform === 'win32';
 const pathSeparator = isWin ? String.fromCharCode(92) : '/';
 
@@ -70,18 +78,6 @@ function min_hex_array(array) {
   return min;
 }
 
-function wfp_for_content(content, contentSource, maxSize) {
-  let wfp = wfp_only_md5(content, contentSource);
-  wfp += calc_wfp(content, maxSize);
-  return wfp;
-}
-
-function wfp_only_md5(contents, contentSource) {
-  const file_md5 = crypto.createHash('md5').update(contents).digest('hex');
-  let wfp = 'file=' + String(file_md5) + ',' + String(contents.length) + ',' + String(contentSource)+ String.fromCharCode(10);
-  return wfp;
-}
-
 function calc_wfp(contents, maxSize) {
   let gram = '';
   const window = [];
@@ -224,6 +220,168 @@ function crc32c_hex(str) {
   return crc32c(str).toString(16).padStart(8, '0');
 }
 
+function truncate_string(input, maxSize){
+  return input.slice(0, maxSize)
+}
+
+function toHexString(byteArray) {
+  return Array.from(byteArray, function(byte) {
+    return ('0' + (byte & 0xFF).toString(16)).slice(-2);
+  }).join('')
+}
+
+function calc_hpsm(content) {
+  let list_normalized = [];    //Array of numbers
+  let crc_lines = [];  //Array of numbers that represent the crc8_maxim for each line of the file
+
+  let last_line = 0;
+  crc8_MAXIM_DOW_GenerateTable();
+
+  for(let i = 0; i<content.length; i++) {
+    const c = content[i];
+    if (c == ASCII_LF) {  //When there is a new line
+      if (list_normalized.length) {
+        crc_lines.push(crc8_MAXIM_DOW_Buffer(list_normalized))
+        list_normalized=[];
+      } else if(last_line+1 == i) {
+        crc_lines.push(0xFF);
+      } else if (i-last_line  > 1) {
+        crc_lines.push(0x00)
+      }
+      last_line = i;
+    } else {
+      const c_normalized = normalize(c);
+      if (c_normalized != 0)  list_normalized.push(c_normalized)
+    }
+  }
+  return "hpsm=" + toHexString(crc_lines)
+}
+
+
+/**
+ * Create a wfp_hpsm package joining wfp (with md5 line) and a hpsm.
+ *
+ * Example:
+ * wfp = file=508cb9dfbe1c7dca5ed24f124473f33d,300,asd.c
+ *         11=b19bdbfa
+ *
+ * hpsm = hpsm=1909ff06ff688630ff45a92b52f47eff3500ffff
+ *
+ * wfp_hpsm = file=508cb9dfbe1c7dca5ed24f124473f33d,300,asd.c
+ *             hpsm=1909ff06ff688630ff45a92b52f47eff3500ffff
+ *             11=b19bdbfa
+ *
+ * @param {string} wfp Complete wfp string (with md5 line)
+ * @param {string} hpsm
+ * @returns {string}
+ */
+function join_wfp_hpsm(wfp, hpsm) {
+  let header = wfp.match(/file=.*/);
+  header += String.fromCharCode(10);
+  header += hpsm;
+  wfp = wfp.replace(/file=.*/, "")
+  return header + wfp;
+}
+
+/**
+ *
+ * @param {Buffer} content
+ * @param {string} contentSource
+ * @param {number} maxSize
+ * @returns {string}
+ */
+function wfp_hpsm_for_content(content, contentSource, maxSize) {
+  let wfp = wfp_for_content(content, contentSource, maxSize)
+  let hpsm = calc_hpsm(content)   //Returns a string
+  let wfp_hpsm_joined = join_wfp_hpsm(wfp, hpsm)
+  return truncate_string(wfp_hpsm_joined, maxSize)
+}
+
+function wfp_for_content(content, contentSource, maxSize) {
+  let wfp = wfp_only_md5(content, contentSource);
+  wfp += calc_wfp(content, maxSize);
+  return wfp;
+}
+
+function wfp_only_md5(contents, contentSource) {
+  const file_md5 = crypto.createHash('md5').update(contents).digest('hex');
+  let wfp = 'file=' + String(file_md5) + ',' + String(contents.length) + ',' + String(contentSource)+ String.fromCharCode(10);
+  return wfp;
+}
+
+/**
+* CRC8_MAXIM algorithm begins
+*/
+
+
+//=============================================================================
+// Implementation of CRC-8/MAXIM-DOW,CRC-8/MAXIM,DOW-CRC.
+// Notes: Code ported from C Auto-generated by http://crc.drque.net/
+//=============================================================================
+
+//Table size
+const CRC8_MAXIM_DOW_TABLE_SIZE = 0x100
+const CRC8_MAXIM_DOW_POLYNOMIAL = 0x8C // 0x31 reflected
+const CRC8_MAXIM_DOW_INITIAL = 0x00 // 0x00 reflected
+const CRC8_MAXIM_DOW_FINAL = 0x00 // 0x00 reflected
+
+let crc8_MAXIM_DOW_Table = []
+
+/**
+ * Compute CRC of byte without lookup tables.
+ *
+ * @param {number} crc Current CRC or initial value
+ * @param {number} byte New byte to accumulate.
+ * @returns {number} Updated CRC.
+ */
+function crc8_MAXIM_DOW_ByteNoTable(crc, byte) {
+  crc ^= byte;
+  for (let count = 0; count<8; count++) {
+    const isSet = (crc & 0x01)
+    crc >>= 1;
+    if (isSet) crc ^= CRC8_MAXIM_DOW_POLYNOMIAL
+  }
+  return crc
+}
+
+/**
+ * Create the lookup table.
+ * Note: Must be called before any table based CRC calculations can be done.
+ */
+function crc8_MAXIM_DOW_GenerateTable() {
+  for(let i = 0; i<CRC8_MAXIM_DOW_TABLE_SIZE; i++) {
+    crc8_MAXIM_DOW_Table.push(crc8_MAXIM_DOW_ByteNoTable(0, i))
+  }
+}
+
+/**
+ * Update CRC by byte.
+ *
+ * @param {number} crc Initial CRC.
+ * @param {number} byte New byte to accumulate.
+ * @returns {number} Updated CRC.
+ */
+function crc8_MAXIM_DOW_Byte( crc, byte )
+{
+  const index = byte ^ crc;
+  return crc8_MAXIM_DOW_Table[ index ] ^ ( crc >> 8 );
+}
+
+/**
+ * Compute CRC of buffer.
+ * Note: crc8_MAXIM_DOW_GenerateTable() must be called before use this function
+ *
+ * @param {Buffer} buffer Buffer with bytes to calculate CRC.
+ * @returns {number} CRC
+ */
+function crc8_MAXIM_DOW_Buffer(buffer) {
+  let crc = CRC8_MAXIM_DOW_INITIAL;
+  for (let index = 0; index < buffer.length ; index ++) {
+    crc = crc8_MAXIM_DOW_Byte(crc, buffer[index])
+  }
+  crc ^= CRC8_MAXIM_DOW_FINAL;
+  return crc;
+}
 `;
 export class WfpCalculator extends WfpProvider {
     fileList;
@@ -327,4 +485,4 @@ export class WfpCalculator extends WfpProvider {
         this.init();
     }
 }
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiV2ZwQ2FsY3VsYXRvci5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uLy4uLy4uLy4uL3NyYy9saWIvc2Nhbm5lci9XZnBQcm92aWRlci9XZnBDYWxjdWxhdG9yL1dmcENhbGN1bGF0b3IudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLE1BQU0sSUFBSSxDQUFDO0FBQ3BCLE9BQU8sRUFBRSxNQUFNLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQztBQUV4QyxPQUFPLEVBQUUsYUFBYSxFQUFFLE1BQU0sK0JBQStCLENBQUM7QUFDOUQsT0FBTyxFQUFFLFVBQVUsRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRzlDLE9BQU8sRUFBRSxrQkFBa0IsRUFBRSxNQUFNLHVCQUF1QixDQUFDO0FBQzNELE9BQU8sRUFBcUIsV0FBVyxFQUFFLE1BQU0sZ0JBQWdCLENBQUM7QUFHaEUsTUFBTSxZQUFZLEdBQUc7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Q0E0TnBCLENBQUM7QUFFRixNQUFNLE9BQU8sYUFBYyxTQUFRLFdBQVc7SUFDcEMsUUFBUSxDQUFNO0lBRWQsYUFBYSxDQUFTO0lBRXRCLFFBQVEsQ0FBVTtJQUUxQixZQUFZLFVBQVUsR0FBRyxJQUFJLFVBQVUsRUFBRTtRQUN2QyxLQUFLLEVBQUUsQ0FBQztRQUNSLElBQUksQ0FBQyxVQUFVLEdBQUcsVUFBVSxDQUFDO0lBQy9CLENBQUM7SUFFRCxJQUFJO1FBQ0YsS0FBSyxDQUFDLElBQUksRUFBRSxDQUFDO1FBQ2IsSUFBSSxDQUFDLFFBQVEsR0FBRyxJQUFJLENBQUM7UUFDckIsSUFBSSxDQUFDLFFBQVEsR0FBRyxFQUFFLENBQUM7UUFDbkIsSUFBSSxDQUFDLGFBQWEsR0FBRyxDQUFDLENBQUM7SUFDekIsQ0FBQztJQUVELGFBQWE7UUFDWCxJQUFJLENBQUMsTUFBTSxHQUFHLElBQUksTUFBTSxDQUFDLFlBQVksRUFBRSxFQUFFLElBQUksRUFBRSxJQUFJLEVBQUUsQ0FBQyxDQUFDO1FBQ3ZELElBQUksQ0FBQyxNQUFNLENBQUMsRUFBRSxDQUFDLFNBQVMsRUFBRSxLQUFLLEVBQUUsYUFBYSxFQUFFLEVBQUU7WUFDaEQsSUFBSSxDQUFDLGlCQUFpQixDQUFDLGFBQWEsQ0FBQyxXQUFXLENBQUMsQ0FBQztZQUNsRCxNQUFNLElBQUksQ0FBQyxlQUFlLEVBQUUsQ0FBQztRQUMvQixDQUFDLENBQUMsQ0FBQztJQUNMLENBQUM7SUFFRCxhQUFhO1FBQ1gsd0RBQXdEO1FBQ3hELE1BQU0sS0FBSyxHQUFHLElBQUksa0JBQWtCLENBQUMsSUFBSSxDQUFDLEdBQUcsRUFBRSxJQUFJLENBQUMsVUFBVSxDQUFDLENBQUMscUJBQXFCLEVBQUUsQ0FBQztRQUN4RixJQUFJLEtBQUssQ0FBQyxNQUFNLEVBQUU7WUFDaEIsTUFBTSxnQkFBZ0IsR0FBRyxLQUFLLENBQUMsS0FBSyxDQUFDLE1BQU0sR0FBRyxDQUFDLENBQUMsQ0FBQztZQUNqRCxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7WUFDVixPQUFPLENBQUMsSUFBSSxLQUFLLENBQUMsTUFBTSxJQUFJLGdCQUFnQixLQUFLLElBQUksQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLGFBQWEsR0FBRyxDQUFDLENBQUMsRUFBRTtnQkFDdEYsQ0FBQyxJQUFJLENBQUMsQ0FBQzthQUNSO1lBQ0Qsc0VBQXNFO1lBQ3RFLElBQUksQ0FBQyxHQUFHLEtBQUssQ0FBQyxNQUFNLEVBQUU7Z0JBQ3BCLElBQUksQ0FBQyxTQUFTLENBQUMsbUNBQW1DLENBQUMsQ0FBQztnQkFDcEQsT0FBTyxDQUFDLENBQUMsQ0FBQzthQUNYO1lBQ0QsSUFBSSxDQUFDLGFBQWEsSUFBSSxDQUFDLENBQUM7WUFDeEIsSUFBSSxJQUFJLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxhQUFhLENBQUMsS0FBSyxnQkFBZ0I7Z0JBQUUsSUFBSSxDQUFDLGFBQWEsSUFBSSxDQUFDLENBQUM7U0FDckY7UUFDRCxPQUFPLENBQUMsQ0FBQztJQUNYLENBQUM7SUFFRCxlQUFlO1FBQ2IsSUFBSSxDQUFDLE1BQU0sQ0FBQyxrQkFBa0IsRUFBRSxDQUFDO1FBQ2pDLElBQUksQ0FBQyxNQUFNLENBQUMsU0FBUyxFQUFFLENBQUM7SUFDMUIsQ0FBQztJQUVTLGdCQUFnQixDQUFDLE9BQU87UUFDaEMsTUFBTSxXQUFXLEdBQUcsSUFBSSxrQkFBa0IsQ0FBQyxPQUFPLEVBQUUsSUFBSSxDQUFDLFVBQVUsQ0FBQyxDQUFDO1FBQ3JFLElBQUksQ0FBQyxlQUFlLENBQUMsV0FBVyxDQUFDLENBQUM7SUFDcEMsQ0FBQztJQUVELEtBQUssQ0FBQyxvQkFBb0I7UUFDeEIsSUFBSSxJQUFJLENBQUMsYUFBYSxJQUFJLElBQUksQ0FBQyxRQUFRLENBQUMsTUFBTTtZQUFFLE9BQU8sSUFBSSxDQUFDO1FBQzVELE1BQU0sSUFBSSxHQUFHLElBQUksQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLGFBQWEsQ0FBQyxDQUFDO1FBQy9DLE1BQU0sYUFBYSxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsR0FBRyxJQUFJLENBQUMsVUFBVSxFQUFFLEVBQUUsRUFBRSxDQUFDLENBQUM7UUFDN0QsTUFBTSxPQUFPLEdBQUcsTUFBTSxFQUFFLENBQUMsUUFBUSxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsQ0FBQztRQUNqRCxJQUFJLENBQUMsYUFBYSxJQUFJLENBQUMsQ0FBQztRQUN4QixNQUFNLFNBQVMsR0FBRyxJQUFJLGFBQWEsQ0FBQyxPQUFPLEVBQUUsYUFBYSxFQUFFLElBQUksQ0FBQyxhQUFhLEVBQUUsSUFBSSxDQUFDLFVBQVUsQ0FBQyxpQkFBaUIsQ0FBQyxDQUFDO1FBQ25ILE9BQU8sU0FBUyxDQUFDO0lBQ25CLENBQUM7SUFFRCxLQUFLLENBQUMsZUFBZTtRQUNuQixJQUFJLENBQUMsSUFBSSxDQUFDLFFBQVE7WUFBRSxPQUFPO1FBQzNCLE1BQU0sYUFBYSxHQUFHLE1BQU0sSUFBSSxDQUFDLG9CQUFvQixFQUFFLENBQUM7UUFDeEQsSUFBSSxhQUFhO1lBQUUsSUFBSSxDQUFDLE1BQU0sQ0FBQyxXQUFXLENBQUMsYUFBYSxDQUFDLENBQUM7YUFDckQ7WUFDSCxJQUFJLENBQUMsZUFBZSxFQUFFLENBQUM7WUFDdkIsSUFBSSxDQUFDLGVBQWUsRUFBRSxDQUFDO1lBQ3ZCLElBQUksQ0FBQyxPQUFPLENBQUMseUNBQXlDLENBQUMsQ0FBQztTQUV6RDtJQUNILENBQUM7SUFHTSxLQUFLLENBQUMsTUFBeUI7UUFFcEMsSUFBRyxDQUFDLE1BQU0sQ0FBQyxRQUFRO1lBQUUsSUFBSSxDQUFDLFNBQVMsQ0FBQyx1QkFBdUIsQ0FBQyxDQUFDO1FBQzdELElBQUksQ0FBQyxPQUFPLENBQUMseUNBQXlDLENBQUMsQ0FBQztRQUV4RCxJQUFJLENBQUMsSUFBSSxFQUFFLENBQUM7UUFDWixJQUFJLENBQUMsYUFBYSxFQUFFLENBQUM7UUFFckIsSUFBRyxNQUFNLENBQUMsYUFBYTtZQUFFLElBQUksQ0FBQyxnQkFBZ0IsQ0FBQyxNQUFNLENBQUMsYUFBYSxDQUFDLENBQUM7UUFDckUsSUFBSSxDQUFDLFlBQVksR0FBRyxJQUFJLENBQUM7UUFDekIsSUFBSSxDQUFDLFVBQVUsR0FBRyxNQUFNLENBQUMsVUFBVSxDQUFDO1FBQ3BDLElBQUksQ0FBQyxRQUFRLEdBQUcsTUFBTSxDQUFDLFFBQVEsQ0FBQztRQUNoQyxJQUFJLENBQUMsZUFBZSxFQUFFLENBQUM7SUFDekIsQ0FBQztJQUtNLEtBQUs7UUFDVixJQUFJLENBQUMsT0FBTyxDQUFDLHVDQUF1QyxDQUFDLENBQUE7UUFDckQsSUFBSSxDQUFDLFFBQVEsR0FBRyxLQUFLLENBQUM7SUFDeEIsQ0FBQztJQUVNLE1BQU07UUFDWCxJQUFJLENBQUMsT0FBTyxDQUFDLHdDQUF3QyxDQUFDLENBQUE7UUFDdEQsSUFBSSxDQUFDLFFBQVEsR0FBRyxJQUFJLENBQUM7UUFDckIsSUFBSSxDQUFDLGFBQWEsRUFBRSxDQUFDO1FBQ3JCLElBQUksQ0FBQyxlQUFlLEVBQUUsQ0FBQztJQUN6QixDQUFDO0lBRU0sSUFBSTtRQUNULElBQUksQ0FBQyxRQUFRLEdBQUcsS0FBSyxDQUFDO1FBQ3RCLElBQUksQ0FBQyxZQUFZLEdBQUcsS0FBSyxDQUFDO1FBQzFCLElBQUksQ0FBQyxlQUFlLEVBQUUsQ0FBQztRQUN2QixJQUFJLENBQUMsYUFBYSxFQUFFLENBQUM7UUFDckIsSUFBSSxDQUFDLElBQUksRUFBRSxDQUFDO0lBQ2QsQ0FBQztDQUVGIn0=
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiV2ZwQ2FsY3VsYXRvci5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uLy4uLy4uLy4uL3NyYy9saWIvc2Nhbm5lci9XZnBQcm92aWRlci9XZnBDYWxjdWxhdG9yL1dmcENhbGN1bGF0b3IudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLE1BQU0sSUFBSSxDQUFDO0FBQ3BCLE9BQU8sRUFBRSxNQUFNLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQztBQUV4QyxPQUFPLEVBQUUsYUFBYSxFQUFFLE1BQU0sK0JBQStCLENBQUM7QUFDOUQsT0FBTyxFQUFFLFVBQVUsRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBRzlDLE9BQU8sRUFBRSxrQkFBa0IsRUFBRSxNQUFNLHVCQUF1QixDQUFDO0FBQzNELE9BQU8sRUFBcUIsV0FBVyxFQUFFLE1BQU0sZ0JBQWdCLENBQUM7QUFFaEUsTUFBTSxZQUFZLEdBQUc7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7OztDQTBYcEIsQ0FBQztBQUVGLE1BQU0sT0FBTyxhQUFjLFNBQVEsV0FBVztJQUNwQyxRQUFRLENBQU07SUFFZCxhQUFhLENBQVM7SUFFdEIsUUFBUSxDQUFVO0lBRTFCLFlBQVksVUFBVSxHQUFHLElBQUksVUFBVSxFQUFFO1FBQ3ZDLEtBQUssRUFBRSxDQUFDO1FBQ1IsSUFBSSxDQUFDLFVBQVUsR0FBRyxVQUFVLENBQUM7SUFDL0IsQ0FBQztJQUVELElBQUk7UUFDRixLQUFLLENBQUMsSUFBSSxFQUFFLENBQUM7UUFDYixJQUFJLENBQUMsUUFBUSxHQUFHLElBQUksQ0FBQztRQUNyQixJQUFJLENBQUMsUUFBUSxHQUFHLEVBQUUsQ0FBQztRQUNuQixJQUFJLENBQUMsYUFBYSxHQUFHLENBQUMsQ0FBQztJQUN6QixDQUFDO0lBRUQsYUFBYTtRQUNYLElBQUksQ0FBQyxNQUFNLEdBQUcsSUFBSSxNQUFNLENBQUMsWUFBWSxFQUFFLEVBQUUsSUFBSSxFQUFFLElBQUksRUFBRSxDQUFDLENBQUM7UUFDdkQsSUFBSSxDQUFDLE1BQU0sQ0FBQyxFQUFFLENBQUMsU0FBUyxFQUFFLEtBQUssRUFBRSxhQUFhLEVBQUUsRUFBRTtZQUNoRCxJQUFJLENBQUMsaUJBQWlCLENBQUMsYUFBYSxDQUFDLFdBQVcsQ0FBQyxDQUFDO1lBQ2xELE1BQU0sSUFBSSxDQUFDLGVBQWUsRUFBRSxDQUFDO1FBQy9CLENBQUMsQ0FBQyxDQUFDO0lBQ0wsQ0FBQztJQUVELGFBQWE7UUFDWCx3REFBd0Q7UUFDeEQsTUFBTSxLQUFLLEdBQUcsSUFBSSxrQkFBa0IsQ0FBQyxJQUFJLENBQUMsR0FBRyxFQUFFLElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQyxxQkFBcUIsRUFBRSxDQUFDO1FBQ3hGLElBQUksS0FBSyxDQUFDLE1BQU0sRUFBRTtZQUNoQixNQUFNLGdCQUFnQixHQUFHLEtBQUssQ0FBQyxLQUFLLENBQUMsTUFBTSxHQUFHLENBQUMsQ0FBQyxDQUFDO1lBQ2pELElBQUksQ0FBQyxHQUFHLENBQUMsQ0FBQztZQUNWLE9BQU8sQ0FBQyxJQUFJLEtBQUssQ0FBQyxNQUFNLElBQUksZ0JBQWdCLEtBQUssSUFBSSxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsYUFBYSxHQUFHLENBQUMsQ0FBQyxFQUFFO2dCQUN0RixDQUFDLElBQUksQ0FBQyxDQUFDO2FBQ1I7WUFDRCxzRUFBc0U7WUFDdEUsSUFBSSxDQUFDLEdBQUcsS0FBSyxDQUFDLE1BQU0sRUFBRTtnQkFDcEIsSUFBSSxDQUFDLFNBQVMsQ0FBQyxtQ0FBbUMsQ0FBQyxDQUFDO2dCQUNwRCxPQUFPLENBQUMsQ0FBQyxDQUFDO2FBQ1g7WUFDRCxJQUFJLENBQUMsYUFBYSxJQUFJLENBQUMsQ0FBQztZQUN4QixJQUFJLElBQUksQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLGFBQWEsQ0FBQyxLQUFLLGdCQUFnQjtnQkFBRSxJQUFJLENBQUMsYUFBYSxJQUFJLENBQUMsQ0FBQztTQUNyRjtRQUNELE9BQU8sQ0FBQyxDQUFDO0lBQ1gsQ0FBQztJQUVELGVBQWU7UUFDYixJQUFJLENBQUMsTUFBTSxDQUFDLGtCQUFrQixFQUFFLENBQUM7UUFDakMsSUFBSSxDQUFDLE1BQU0sQ0FBQyxTQUFTLEVBQUUsQ0FBQztJQUMxQixDQUFDO0lBRVMsZ0JBQWdCLENBQUMsT0FBTztRQUNoQyxNQUFNLFdBQVcsR0FBRyxJQUFJLGtCQUFrQixDQUFDLE9BQU8sRUFBRSxJQUFJLENBQUMsVUFBVSxDQUFDLENBQUM7UUFDckUsSUFBSSxDQUFDLGVBQWUsQ0FBQyxXQUFXLENBQUMsQ0FBQztJQUNwQyxDQUFDO0lBRUQsS0FBSyxDQUFDLG9CQUFvQjtRQUN4QixJQUFJLElBQUksQ0FBQyxhQUFhLElBQUksSUFBSSxDQUFDLFFBQVEsQ0FBQyxNQUFNO1lBQUUsT0FBTyxJQUFJLENBQUM7UUFDNUQsTUFBTSxJQUFJLEdBQUcsSUFBSSxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsYUFBYSxDQUFDLENBQUM7UUFDL0MsTUFBTSxhQUFhLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxHQUFHLElBQUksQ0FBQyxVQUFVLEVBQUUsRUFBRSxFQUFFLENBQUMsQ0FBQztRQUM3RCxNQUFNLE9BQU8sR0FBRyxNQUFNLEVBQUUsQ0FBQyxRQUFRLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxDQUFDO1FBQ2pELElBQUksQ0FBQyxhQUFhLElBQUksQ0FBQyxDQUFDO1FBQ3hCLE1BQU0sU0FBUyxHQUFHLElBQUksYUFBYSxDQUFDLE9BQU8sRUFBRSxhQUFhLEVBQUUsSUFBSSxDQUFDLGFBQWEsRUFBRSxJQUFJLENBQUMsVUFBVSxDQUFDLGlCQUFpQixDQUFDLENBQUM7UUFDbkgsT0FBTyxTQUFTLENBQUM7SUFDbkIsQ0FBQztJQUVELEtBQUssQ0FBQyxlQUFlO1FBQ25CLElBQUksQ0FBQyxJQUFJLENBQUMsUUFBUTtZQUFFLE9BQU87UUFDM0IsTUFBTSxhQUFhLEdBQUcsTUFBTSxJQUFJLENBQUMsb0JBQW9CLEVBQUUsQ0FBQztRQUN4RCxJQUFJLGFBQWE7WUFBRSxJQUFJLENBQUMsTUFBTSxDQUFDLFdBQVcsQ0FBQyxhQUFhLENBQUMsQ0FBQzthQUNyRDtZQUNILElBQUksQ0FBQyxlQUFlLEVBQUUsQ0FBQztZQUN2QixJQUFJLENBQUMsZUFBZSxFQUFFLENBQUM7WUFDdkIsSUFBSSxDQUFDLE9BQU8sQ0FBQyx5Q0FBeUMsQ0FBQyxDQUFDO1NBRXpEO0lBQ0gsQ0FBQztJQUdNLEtBQUssQ0FBQyxNQUF5QjtRQUVwQyxJQUFHLENBQUMsTUFBTSxDQUFDLFFBQVE7WUFBRSxJQUFJLENBQUMsU0FBUyxDQUFDLHVCQUF1QixDQUFDLENBQUM7UUFDN0QsSUFBSSxDQUFDLE9BQU8sQ0FBQyx5Q0FBeUMsQ0FBQyxDQUFDO1FBRXhELElBQUksQ0FBQyxJQUFJLEVBQUUsQ0FBQztRQUNaLElBQUksQ0FBQyxhQUFhLEVBQUUsQ0FBQztRQUVyQixJQUFHLE1BQU0sQ0FBQyxhQUFhO1lBQUUsSUFBSSxDQUFDLGdCQUFnQixDQUFDLE1BQU0sQ0FBQyxhQUFhLENBQUMsQ0FBQztRQUNyRSxJQUFJLENBQUMsWUFBWSxHQUFHLElBQUksQ0FBQztRQUN6QixJQUFJLENBQUMsVUFBVSxHQUFHLE1BQU0sQ0FBQyxVQUFVLENBQUM7UUFDcEMsSUFBSSxDQUFDLFFBQVEsR0FBRyxNQUFNLENBQUMsUUFBUSxDQUFDO1FBQ2hDLElBQUksQ0FBQyxlQUFlLEVBQUUsQ0FBQztJQUN6QixDQUFDO0lBS00sS0FBSztRQUNWLElBQUksQ0FBQyxPQUFPLENBQUMsdUNBQXVDLENBQUMsQ0FBQTtRQUNyRCxJQUFJLENBQUMsUUFBUSxHQUFHLEtBQUssQ0FBQztJQUN4QixDQUFDO0lBRU0sTUFBTTtRQUNYLElBQUksQ0FBQyxPQUFPLENBQUMsd0NBQXdDLENBQUMsQ0FBQTtRQUN0RCxJQUFJLENBQUMsUUFBUSxHQUFHLElBQUksQ0FBQztRQUNyQixJQUFJLENBQUMsYUFBYSxFQUFFLENBQUM7UUFDckIsSUFBSSxDQUFDLGVBQWUsRUFBRSxDQUFDO0lBQ3pCLENBQUM7SUFFTSxJQUFJO1FBQ1QsSUFBSSxDQUFDLFFBQVEsR0FBRyxLQUFLLENBQUM7UUFDdEIsSUFBSSxDQUFDLFlBQVksR0FBRyxLQUFLLENBQUM7UUFDMUIsSUFBSSxDQUFDLGVBQWUsRUFBRSxDQUFDO1FBQ3ZCLElBQUksQ0FBQyxhQUFhLEVBQUUsQ0FBQztRQUNyQixJQUFJLENBQUMsSUFBSSxFQUFFLENBQUM7SUFDZCxDQUFDO0NBRUYifQ==

package/build/module/lib/scanner/WfpProvider/WfpCalculator/Winnower.d.ts

@@ -0,0 +1,3 @@
+export function wfp_hpsm_for_content(content: any, contentSource: any, maxSize: any): string;
+export function wfp_for_content(content: any, contentSource: any, maxSize: any): string;
+export function wfp_only_md5(contents: any, contentSource: any): string;

package/build/module/lib/scanner/WfpProvider/WfpCalculator/Winnower.js

@@ -0,0 +1,211 @@
+const crypto = require('crypto');
+const isWin = process.platform === 'win32';
+const pathSeparator = isWin ? String.fromCharCode(92) : '/';
+// Winnowing configuration. DO NOT CHANGE.
+const GRAM = 30;
+const WINDOW = 64;
+// ASCII characters
+const ASCII_0 = 48;
+const ASCII_9 = 57;
+const ASCII_A = 65;
+const ASCII_Z = 90;
+const ASCII_a = 97;
+const ASCII_z = 122;
+const ASCII_LF = 10;
+function normalize(byte) {
+    if (byte < ASCII_0 || byte > ASCII_z) {
+        return 0;
+    }
+    if (byte <= ASCII_9 || byte >= ASCII_a) {
+        return byte;
+    }
+    if (byte >= ASCII_A && byte <= ASCII_Z) {
+        return byte + 32;
+    }
+    return 0;
+}
+function min_hex_array(array) {
+    let min = 'ffffffff';
+    for (let i = 0; i < array.length; i++) {
+        if (array[i] < min) {
+            min = array[i];
+        }
+    }
+    return min;
+}
+function calc_wfp(contents, maxSize) {
+    let gram = '';
+    const window = [];
+    let normalized = 0;
+    let line = 1;
+    let min_hash = 'ffffffff';
+    let last_hash = 'ffffffff';
+    let last_line = 0;
+    let output = '';
+    let gram_crc32 = 0;
+    let wfp = '';
+    for (let i = 0; i < contents.length; i++) {
+        if (wfp.length > maxSize)
+            return wfp;
+        const byte = contents[i];
+        if (byte == ASCII_LF) {
+            line += 1;
+            normalized = 0;
+        }
+        else {
+            normalized = normalize(byte);
+        }
+        // Is this an useful byte?
+        if (normalized) {
+            gram += String.fromCharCode(normalized);
+            if (gram.length >= GRAM) {
+                gram_crc32 = crc32c_hex(gram);
+                window.push(gram_crc32);
+                if (window.length >= WINDOW) {
+                    min_hash = min_hex_array(window);
+                    if (min_hash !== last_hash) {
+                        // Hashing the hash will result in a better balanced output data set
+                        // as it will counter the winnowing effect which selects the "minimum"
+                        // hash in each window
+                        const min_hash_bytes_le = parseHexString(toLittleEndianCRCHex(min_hash));
+                        const crc_hex = crc32c_for_bytes_hex(min_hash_bytes_le);
+                        if (last_line != line) {
+                            if (output.length > 0) {
+                                wfp += String(output) + String.fromCharCode(10);
+                            }
+                            output = String(line) + '=' + String(crc_hex);
+                        }
+                        else {
+                            output += ',' + String(crc_hex);
+                        }
+                        last_line = line;
+                        last_hash = min_hash;
+                    }
+                    window.shift();
+                }
+                gram = gram.slice(1);
+            }
+        }
+    }
+    if (output.length > 0) {
+        wfp += String(output) + String.fromCharCode(10);
+    }
+    return wfp;
+}
+function parseHexString(str) {
+    const result = [];
+    while (str.length >= 2) {
+        result.push(parseInt(str.substring(0, 2), 16));
+        str = str.substring(2, str.length);
+    }
+    return result;
+}
+/**
+ *
+ * @param {string} hex
+ */
+function toLittleEndianCRCHex(hex) {
+    return (hex.charAt(6) +
+        hex.charAt(7) +
+        hex.charAt(4) +
+        hex.charAt(5) +
+        hex.charAt(2) +
+        hex.charAt(3) +
+        hex.charAt(0) +
+        hex.charAt(1));
+}
+let CRC_TABLE = [];
+function makeCRCTable() {
+    let c;
+    const crcTable = [];
+    for (let n = 0; n < 256; n++) {
+        c = n;
+        for (let k = 0; k < 8; k++) {
+            c = c & 1 ? 0x82f63b78 ^ (c >>> 1) : c >>> 1;
+        }
+        crcTable[n] = c;
+    }
+    return crcTable;
+}
+function crc32c(str) {
+    if (CRC_TABLE.length == 0) {
+        CRC_TABLE = makeCRCTable();
+    }
+    let crc = 0 ^ -1;
+    for (let i = 0; i < str.length; i++) {
+        crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ str.charCodeAt(i)) & 0xff];
+    }
+    return (crc ^ -1) >>> 0;
+}
+function crc32c_for_bytes(bytes) {
+    if (CRC_TABLE.length == 0) {
+        CRC_TABLE = makeCRCTable();
+    }
+    let crc = 0 ^ -1;
+    for (let i = 0; i < bytes.length; i++) {
+        crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ bytes[i]) & 0xff];
+    }
+    return (crc ^ -1) >>> 0;
+}
+function crc32c_for_bytes_hex(bytes) {
+    return crc32c_for_bytes(bytes).toString(16).padStart(8, '0');
+}
+function crc32c_hex(str) {
+    return crc32c(str).toString(16).padStart(8, '0');
+}
+function truncate_string(input, maxSize) {
+    return "";
+}
+function calc_hpsm(content) {
+    for (let i = 0; i < content.length; i++) {
+        let a = 0;
+    }
+    return "";
+}
+/*
+Receives a wfp with the md5 line, a hpsm and joins both.
+
+Example:
+wfp = `file=508cb9dfbe1c7dca5ed24f124473f33d,300,asd.c
+        11=b19bdbfa
+        13=8fd738fa,b6f47c1e
+        `
+hpsm = `hpsm=1909ff06ff688630ff45a92b52f47eff3500ffff`
+
+wfp_hpsm = `file=508cb9dfbe1c7dca5ed24f124473f33d,300,asd.c
+            hpsm=1909ff06ff688630ff45a92b52f47eff3500ffff
+            11=b19bdbfa
+            13=8fd738fa,b6f47c1e`
+ */
+function join_wfp_hpsm(wfp, hpsm) {
+    return "";
+}
+function wfp_hpsm_for_content(content, contentSource, maxSize) {
+    let wfp = wfp_for_content(content, contentSource, maxSize);
+    let hpsm = calc_hpsm(content);
+    if (hpsm.length >= maxSize) {
+        //If the hpsm is longer than maxSize?
+    }
+    let wfp_hpsm_joined = join_wfp_hpsm(wfp, hpsm);
+    // if the joined package is longer than maxSize truncate wfps lines
+    if (wfp_hpsm_joined.length > maxSize) {
+    }
+    let output = "";
+    return truncate_string(output, maxSize);
+}
+function wfp_for_content(content, contentSource, maxSize) {
+    let wfp = wfp_only_md5(content, contentSource);
+    wfp += calc_wfp(content, maxSize);
+    return wfp;
+}
+function wfp_only_md5(contents, contentSource) {
+    const file_md5 = crypto.createHash('md5').update(contents).digest('hex');
+    let wfp = 'file=' + String(file_md5) + ',' + String(contents.length) + ',' + String(contentSource) + String.fromCharCode(10);
+    return wfp;
+}
+module.exports = {
+    wfp_hpsm_for_content,
+    wfp_for_content,
+    wfp_only_md5,
+};
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiV2lubm93ZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi8uLi8uLi9zcmMvbGliL3NjYW5uZXIvV2ZwUHJvdmlkZXIvV2ZwQ2FsY3VsYXRvci9XaW5ub3dlci5qcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxNQUFNLE1BQU0sR0FBRyxPQUFPLENBQUMsUUFBUSxDQUFDLENBQUM7QUFFakMsTUFBTSxLQUFLLEdBQUcsT0FBTyxDQUFDLFFBQVEsS0FBSyxPQUFPLENBQUM7QUFDM0MsTUFBTSxhQUFhLEdBQUcsS0FBSyxDQUFDLENBQUMsQ0FBQyxNQUFNLENBQUMsWUFBWSxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQyxHQUFHLENBQUM7QUFFNUQsMENBQTBDO0FBQzFDLE1BQU0sSUFBSSxHQUFHLEVBQUUsQ0FBQztBQUNoQixNQUFNLE1BQU0sR0FBRyxFQUFFLENBQUM7QUFFbEIsbUJBQW1CO0FBQ25CLE1BQU0sT0FBTyxHQUFHLEVBQUUsQ0FBQztBQUNuQixNQUFNLE9BQU8sR0FBRyxFQUFFLENBQUM7QUFDbkIsTUFBTSxPQUFPLEdBQUcsRUFBRSxDQUFDO0FBQ25CLE1BQU0sT0FBTyxHQUFHLEVBQUUsQ0FBQztBQUNuQixNQUFNLE9BQU8sR0FBRyxFQUFFLENBQUM7QUFDbkIsTUFBTSxPQUFPLEdBQUcsR0FBRyxDQUFDO0FBQ3BCLE1BQU0sUUFBUSxHQUFHLEVBQUUsQ0FBQztBQUVwQixTQUFTLFNBQVMsQ0FBQyxJQUFJO0lBQ3JCLElBQUksSUFBSSxHQUFHLE9BQU8sSUFBSSxJQUFJLEdBQUcsT0FBTyxFQUFFO1FBQ3BDLE9BQU8sQ0FBQyxDQUFDO0tBQ1Y7SUFDRCxJQUFJLElBQUksSUFBSSxPQUFPLElBQUksSUFBSSxJQUFJLE9BQU8sRUFBRTtRQUN0QyxPQUFPLElBQUksQ0FBQztLQUNiO0lBQ0QsSUFBSSxJQUFJLElBQUksT0FBTyxJQUFJLElBQUksSUFBSSxPQUFPLEVBQUU7UUFDdEMsT0FBTyxJQUFJLEdBQUcsRUFBRSxDQUFDO0tBQ2xCO0lBQ0QsT0FBTyxDQUFDLENBQUM7QUFDWCxDQUFDO0FBRUQsU0FBUyxhQUFhLENBQUMsS0FBSztJQUMxQixJQUFJLEdBQUcsR0FBRyxVQUFVLENBQUM7SUFDckIsS0FBSyxJQUFJLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxHQUFHLEtBQUssQ0FBQyxNQUFNLEVBQUUsQ0FBQyxFQUFFLEVBQUU7UUFDckMsSUFBSSxLQUFLLENBQUMsQ0FBQyxDQUFDLEdBQUcsR0FBRyxFQUFFO1lBQ2xCLEdBQUcsR0FBRyxLQUFLLENBQUMsQ0FBQyxDQUFDLENBQUM7U0FDaEI7S0FDRjtJQUNELE9BQU8sR0FBRyxDQUFDO0FBQ2IsQ0FBQztBQUVELFNBQVMsUUFBUSxDQUFDLFFBQVEsRUFBRSxPQUFPO0lBQ2pDLElBQUksSUFBSSxHQUFHLEVBQUUsQ0FBQztJQUNkLE1BQU0sTUFBTSxHQUFHLEVBQUUsQ0FBQztJQUNsQixJQUFJLFVBQVUsR0FBRyxDQUFDLENBQUM7SUFDbkIsSUFBSSxJQUFJLEdBQUcsQ0FBQyxDQUFDO0lBQ2IsSUFBSSxRQUFRLEdBQUcsVUFBVSxDQUFDO0lBQzFCLElBQUksU0FBUyxHQUFHLFVBQVUsQ0FBQztJQUMzQixJQUFJLFNBQVMsR0FBRyxDQUFDLENBQUM7SUFDbEIsSUFBSSxNQUFNLEdBQUcsRUFBRSxDQUFDO0lBQ2hCLElBQUksVUFBVSxHQUFHLENBQUMsQ0FBQztJQUNuQixJQUFJLEdBQUcsR0FBRyxFQUFFLENBQUM7SUFFYixLQUFLLElBQUksQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLEdBQUcsUUFBUSxDQUFDLE1BQU0sRUFBRSxDQUFDLEVBQUUsRUFBRTtRQUN4QyxJQUFHLEdBQUcsQ0FBQyxNQUFNLEdBQUcsT0FBTztZQUNyQixPQUFPLEdBQUcsQ0FBQztRQUViLE1BQU0sSUFBSSxHQUFHLFFBQVEsQ0FBQyxDQUFDLENBQUMsQ0FBQztRQUN6QixJQUFJLElBQUksSUFBSSxRQUFRLEVBQUU7WUFDcEIsSUFBSSxJQUFJLENBQUMsQ0FBQztZQUNWLFVBQVUsR0FBRyxDQUFDLENBQUM7U0FDaEI7YUFBTTtZQUNMLFVBQVUsR0FBRyxTQUFTLENBQUMsSUFBSSxDQUFDLENBQUM7U0FDOUI7UUFDRCwwQkFBMEI7UUFDMUIsSUFBSSxVQUFVLEVBQUU7WUFDZCxJQUFJLElBQUksTUFBTSxDQUFDLFlBQVksQ0FBQyxVQUFVLENBQUMsQ0FBQztZQUV4QyxJQUFJLElBQUksQ0FBQyxNQUFNLElBQUksSUFBSSxFQUFFO2dCQUN2QixVQUFVLEdBQUcsVUFBVSxDQUFDLElBQUksQ0FBQyxDQUFDO2dCQUM5QixNQUFNLENBQUMsSUFBSSxDQUFDLFVBQVUsQ0FBQyxDQUFDO2dCQUV4QixJQUFJLE1BQU0sQ0FBQyxNQUFNLElBQUksTUFBTSxFQUFFO29CQUMzQixRQUFRLEdBQUcsYUFBYSxDQUFDLE1BQU0sQ0FBQyxDQUFDO29CQUNqQyxJQUFJLFFBQVEsS0FBSyxTQUFTLEVBQUU7d0JBQzFCLG9FQUFvRTt3QkFDcEUsc0VBQXNFO3dCQUN0RSxzQkFBc0I7d0JBQ3RCLE1BQU0saUJBQWlCLEdBQUcsY0FBYyxDQUN0QyxvQkFBb0IsQ0FBQyxRQUFRLENBQUMsQ0FDL0IsQ0FBQzt3QkFDRixNQUFNLE9BQU8sR0FBRyxvQkFBb0IsQ0FBQyxpQkFBaUIsQ0FBQyxDQUFDO3dCQUV4RCxJQUFJLFNBQVMsSUFBSSxJQUFJLEVBQUU7NEJBQ3JCLElBQUksTUFBTSxDQUFDLE1BQU0sR0FBRyxDQUFDLEVBQUU7Z0NBQ3JCLEdBQUcsSUFBSSxNQUFNLENBQUMsTUFBTSxDQUFDLEdBQUcsTUFBTSxDQUFDLFlBQVksQ0FBQyxFQUFFLENBQUMsQ0FBQzs2QkFDakQ7NEJBQ0QsTUFBTSxHQUFHLE1BQU0sQ0FBQyxJQUFJLENBQUMsR0FBRyxHQUFHLEdBQUcsTUFBTSxDQUFDLE9BQU8sQ0FBQyxDQUFDO3lCQUMvQzs2QkFBTTs0QkFDTCxNQUFNLElBQUksR0FBRyxHQUFHLE1BQU0sQ0FBQyxPQUFPLENBQUMsQ0FBQzt5QkFDakM7d0JBQ0QsU0FBUyxHQUFHLElBQUksQ0FBQzt3QkFDakIsU0FBUyxHQUFHLFFBQVEsQ0FBQztxQkFDdEI7b0JBQ0QsTUFBTSxDQUFDLEtBQUssRUFBRSxDQUFDO2lCQUNoQjtnQkFDRCxJQUFJLEdBQUcsSUFBSSxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsQ0FBQzthQUN0QjtTQUNGO0tBQ0Y7SUFDRCxJQUFJLE1BQU0sQ0FBQyxNQUFNLEdBQUcsQ0FBQyxFQUFFO1FBQ3JCLEdBQUcsSUFBSSxNQUFNLENBQUMsTUFBTSxDQUFDLEdBQUcsTUFBTSxDQUFDLFlBQVksQ0FBQyxFQUFFLENBQUMsQ0FBQztLQUNqRDtJQUVELE9BQU8sR0FBRyxDQUFDO0FBQ2IsQ0FBQztBQUVELFNBQVMsY0FBYyxDQUFDLEdBQUc7SUFDekIsTUFBTSxNQUFNLEdBQUcsRUFBRSxDQUFDO0lBQ2xCLE9BQU8sR0FBRyxDQUFDLE1BQU0sSUFBSSxDQUFDLEVBQUU7UUFDdEIsTUFBTSxDQUFDLElBQUksQ0FBQyxRQUFRLENBQUMsR0FBRyxDQUFDLFNBQVMsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUMsQ0FBQztRQUMvQyxHQUFHLEdBQUcsR0FBRyxDQUFDLFNBQVMsQ0FBQyxDQUFDLEVBQUUsR0FBRyxDQUFDLE1BQU0sQ0FBQyxDQUFDO0tBQ3BDO0lBRUQsT0FBTyxNQUFNLENBQUM7QUFDaEIsQ0FBQztBQUVEOzs7R0FHRztBQUNILFNBQVMsb0JBQW9CLENBQUMsR0FBRztJQUMvQixPQUFPLENBQ0wsR0FBRyxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUM7UUFDYixHQUFHLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQztRQUNiLEdBQUcsQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDO1FBQ2IsR0FBRyxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUM7UUFDYixHQUFHLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQztRQUNiLEdBQUcsQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDO1FBQ2IsR0FBRyxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUM7UUFDYixHQUFHLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQyxDQUNkLENBQUM7QUFDSixDQUFDO0FBRUQsSUFBSSxTQUFTLEdBQUcsRUFBRSxDQUFDO0FBRW5CLFNBQVMsWUFBWTtJQUNuQixJQUFJLENBQUMsQ0FBQztJQUNOLE1BQU0sUUFBUSxHQUFHLEVBQUUsQ0FBQztJQUNwQixLQUFLLElBQUksQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLEdBQUcsR0FBRyxFQUFFLENBQUMsRUFBRSxFQUFFO1FBQzVCLENBQUMsR0FBRyxDQUFDLENBQUM7UUFDTixLQUFLLElBQUksQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsRUFBRSxFQUFFO1lBQzFCLENBQUMsR0FBRyxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxVQUFVLEdBQUcsQ0FBQyxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLENBQUM7U0FDOUM7UUFDRCxRQUFRLENBQUMsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxDQUFDO0tBQ2pCO0lBQ0QsT0FBTyxRQUFRLENBQUM7QUFDbEIsQ0FBQztBQUVELFNBQVMsTUFBTSxDQUFDLEdBQUc7SUFDakIsSUFBSSxTQUFTLENBQUMsTUFBTSxJQUFJLENBQUMsRUFBRTtRQUN6QixTQUFTLEdBQUcsWUFBWSxFQUFFLENBQUM7S0FDNUI7SUFDRCxJQUFJLEdBQUcsR0FBRyxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUM7SUFFakIsS0FBSyxJQUFJLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxHQUFHLEdBQUcsQ0FBQyxNQUFNLEVBQUUsQ0FBQyxFQUFFLEVBQUU7UUFDbkMsR0FBRyxHQUFHLENBQUMsR0FBRyxLQUFLLENBQUMsQ0FBQyxHQUFHLFNBQVMsQ0FBQyxDQUFDLEdBQUcsR0FBRyxHQUFHLENBQUMsVUFBVSxDQUFDLENBQUMsQ0FBQyxDQUFDLEdBQUcsSUFBSSxDQUFDLENBQUM7S0FDakU7SUFFRCxPQUFPLENBQUMsR0FBRyxHQUFHLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQyxDQUFDO0FBQzFCLENBQUM7QUFFRCxTQUFTLGdCQUFnQixDQUFDLEtBQUs7SUFDN0IsSUFBSSxTQUFTLENBQUMsTUFBTSxJQUFJLENBQUMsRUFBRTtRQUN6QixTQUFTLEdBQUcsWUFBWSxFQUFFLENBQUM7S0FDNUI7SUFDRCxJQUFJLEdBQUcsR0FBRyxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUM7SUFFakIsS0FBSyxJQUFJLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxHQUFHLEtBQUssQ0FBQyxNQUFNLEVBQUUsQ0FBQyxFQUFFLEVBQUU7UUFDckMsR0FBRyxHQUFHLENBQUMsR0FBRyxLQUFLLENBQUMsQ0FBQyxHQUFHLFNBQVMsQ0FBQyxDQUFDLEdBQUcsR0FBRyxLQUFLLENBQUMsQ0FBQyxDQUFDLENBQUMsR0FBRyxJQUFJLENBQUMsQ0FBQztLQUN4RDtJQUVELE9BQU8sQ0FBQyxHQUFHLEdBQUcsQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLENBQUM7QUFDMUIsQ0FBQztBQUVELFNBQVMsb0JBQW9CLENBQUMsS0FBSztJQUNqQyxPQUFPLGdCQUFnQixDQUFDLEtBQUssQ0FBQyxDQUFDLFFBQVEsQ0FBQyxFQUFFLENBQUMsQ0FBQyxRQUFRLENBQUMsQ0FBQyxFQUFFLEdBQUcsQ0FBQyxDQUFDO0FBQy9ELENBQUM7QUFFRCxTQUFTLFVBQVUsQ0FBQyxHQUFHO0lBQ3JCLE9BQU8sTUFBTSxDQUFDLEdBQUcsQ0FBQyxDQUFDLFFBQVEsQ0FBQyxFQUFFLENBQUMsQ0FBQyxRQUFRLENBQUMsQ0FBQyxFQUFFLEdBQUcsQ0FBQyxDQUFDO0FBQ25ELENBQUM7QUFJRCxTQUFTLGVBQWUsQ0FBQyxLQUFLLEVBQUUsT0FBTztJQUVyQyxPQUFPLEVBQUUsQ0FBQTtBQUNYLENBQUM7QUFHRCxTQUFTLFNBQVMsQ0FBQyxPQUFPO0lBRXhCLEtBQUssSUFBSSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsR0FBRyxPQUFPLENBQUMsTUFBTSxFQUFFLENBQUMsRUFBRSxFQUFFO1FBQ3ZDLElBQUksQ0FBQyxHQUFHLENBQUMsQ0FBQTtLQUNWO0lBRUQsT0FBTyxFQUFFLENBQUE7QUFFWCxDQUFDO0FBRUQ7Ozs7Ozs7Ozs7Ozs7O0dBY0c7QUFDSCxTQUFTLGFBQWEsQ0FBQyxHQUFHLEVBQUUsSUFBSTtJQUM5QixPQUFPLEVBQUUsQ0FBQTtBQUNYLENBQUM7QUFHRCxTQUFTLG9CQUFvQixDQUFDLE9BQU8sRUFBRSxhQUFhLEVBQUUsT0FBTztJQUMzRCxJQUFJLEdBQUcsR0FBRyxlQUFlLENBQUMsT0FBTyxFQUFFLGFBQWEsRUFBRSxPQUFPLENBQUMsQ0FBQTtJQUMxRCxJQUFJLElBQUksR0FBRyxTQUFTLENBQUMsT0FBTyxDQUFDLENBQUE7SUFFN0IsSUFBSSxJQUFJLENBQUMsTUFBTSxJQUFJLE9BQU8sRUFBRTtRQUMxQixxQ0FBcUM7S0FDdEM7SUFFRCxJQUFJLGVBQWUsR0FBRyxhQUFhLENBQUMsR0FBRyxFQUFFLElBQUksQ0FBQyxDQUFBO0lBRTlDLG1FQUFtRTtJQUNuRSxJQUFJLGVBQWUsQ0FBQyxNQUFNLEdBQUcsT0FBTyxFQUFDO0tBRXBDO0lBQ0QsSUFBSSxNQUFNLEdBQUcsRUFBRSxDQUFBO0lBRWYsT0FBTyxlQUFlLENBQUMsTUFBTSxFQUFFLE9BQU8sQ0FBQyxDQUFBO0FBRXpDLENBQUM7QUFHRCxTQUFTLGVBQWUsQ0FBQyxPQUFPLEVBQUUsYUFBYSxFQUFFLE9BQU87SUFDdEQsSUFBSSxHQUFHLEdBQUcsWUFBWSxDQUFDLE9BQU8sRUFBRSxhQUFhLENBQUMsQ0FBQztJQUMvQyxHQUFHLElBQUksUUFBUSxDQUFDLE9BQU8sRUFBRSxPQUFPLENBQUMsQ0FBQztJQUNsQyxPQUFPLEdBQUcsQ0FBQztBQUNiLENBQUM7QUFFRCxTQUFTLFlBQVksQ0FBQyxRQUFRLEVBQUUsYUFBYTtJQUMzQyxNQUFNLFFBQVEsR0FBRyxNQUFNLENBQUMsVUFBVSxDQUFDLEtBQUssQ0FBQyxDQUFDLE1BQU0sQ0FBQyxRQUFRLENBQUMsQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLENBQUM7SUFDekUsSUFBSSxHQUFHLEdBQUcsT0FBTyxHQUFHLE1BQU0sQ0FBQyxRQUFRLENBQUMsR0FBRyxHQUFHLEdBQUcsTUFBTSxDQUFDLFFBQVEsQ0FBQyxNQUFNLENBQUMsR0FBRyxHQUFHLEdBQUcsTUFBTSxDQUFDLGFBQWEsQ0FBQyxHQUFFLE1BQU0sQ0FBQyxZQUFZLENBQUMsRUFBRSxDQUFDLENBQUM7SUFDNUgsT0FBTyxHQUFHLENBQUM7QUFDYixDQUFDO0FBR0QsTUFBTSxDQUFDLE9BQU8sR0FBRztJQUNmLG9CQUFvQjtJQUNwQixlQUFlO0lBQ2YsWUFBWTtDQUNiLENBQUEifQ==