scanoss 0.2.26 → 0.3.0

package/src/lib/scanner/WfpProvider/WfpCalculator/WfpCalculator.ts

@@ -5,10 +5,9 @@ import { ScannableItem } from '../../Scannable/ScannableItem';
 import { ScannerCfg } from '../../ScannerCfg';
 import { ScannerEvents, ScannerInput, WinnowingMode } from '../../ScannerTypes';
 
-import { FingerprintPacket } from '../FingerprintPacket';
+import { FingerprintPackage } from '../FingerprintPackage';
 import { IWfpProviderInput, WfpProvider } from '../WfpProvider';
 
-
 const stringWorker = `
 const { parentPort } = require('worker_threads');
 
@@ -21,7 +20,13 @@ parentPort.on('message', async (scannableItem) => {
       scannableItem.contentSource,
       scannableItem.maxSizeWfp
     );
-  } else {
+  } else if (scannableItem.winnowingMode === "FULL_WINNOWING_HPSM") {
+    fingerprint = wfp_hpsm_for_content(
+      scannableItem.content,
+      scannableItem.contentSource,
+      scannableItem.maxSizeWfp
+    );
+  } else if (scannableItem.winnowingMode === "WINNOWING_ONLY_MD5") {
     fingerprint = wfp_only_md5(
       scannableItem.content,
       scannableItem.contentSource
@@ -29,13 +34,15 @@ parentPort.on('message', async (scannableItem) => {
   }
 
   scannableItem.fingerprint = fingerprint;
-
   parentPort.postMessage(scannableItem);
-});
 
+  });
 
-const crypto = require('crypto');
+  /**
+  * Winnowing and HPSM algorithm begins
+  */
 
+const crypto = require('crypto');
 const isWin = process.platform === 'win32';
 const pathSeparator = isWin ? String.fromCharCode(92) : '/';
 
@@ -75,18 +82,6 @@ function min_hex_array(array) {
   return min;
 }
 
-function wfp_for_content(content, contentSource, maxSize) {
-  let wfp = wfp_only_md5(content, contentSource);
-  wfp += calc_wfp(content, maxSize);
-  return wfp;
-}
-
-function wfp_only_md5(contents, contentSource) {
-  const file_md5 = crypto.createHash('md5').update(contents).digest('hex');
-  let wfp = 'file=' + String(file_md5) + ',' + String(contents.length) + ',' + String(contentSource)+ String.fromCharCode(10);
-  return wfp;
-}
-
 function calc_wfp(contents, maxSize) {
   let gram = '';
   const window = [];
@@ -229,6 +224,168 @@ function crc32c_hex(str) {
   return crc32c(str).toString(16).padStart(8, '0');
 }
 
+function truncate_string(input, maxSize){
+  return input.slice(0, maxSize)
+}
+
+function toHexString(byteArray) {
+  return Array.from(byteArray, function(byte) {
+    return ('0' + (byte & 0xFF).toString(16)).slice(-2);
+  }).join('')
+}
+
+function calc_hpsm(content) {
+  let list_normalized = [];    //Array of numbers
+  let crc_lines = [];  //Array of numbers that represent the crc8_maxim for each line of the file
+
+  let last_line = 0;
+  crc8_MAXIM_DOW_GenerateTable();
+
+  for(let i = 0; i<content.length; i++) {
+    const c = content[i];
+    if (c == ASCII_LF) {  //When there is a new line
+      if (list_normalized.length) {
+        crc_lines.push(crc8_MAXIM_DOW_Buffer(list_normalized))
+        list_normalized=[];
+      } else if(last_line+1 == i) {
+        crc_lines.push(0xFF);
+      } else if (i-last_line  > 1) {
+        crc_lines.push(0x00)
+      }
+      last_line = i;
+    } else {
+      const c_normalized = normalize(c);
+      if (c_normalized != 0)  list_normalized.push(c_normalized)
+    }
+  }
+  return "hpsm=" + toHexString(crc_lines)
+}
+
+
+/**
+ * Create a wfp_hpsm package joining wfp (with md5 line) and a hpsm.
+ *
+ * Example:
+ * wfp = file=508cb9dfbe1c7dca5ed24f124473f33d,300,asd.c
+ *         11=b19bdbfa
+ *
+ * hpsm = hpsm=1909ff06ff688630ff45a92b52f47eff3500ffff
+ *
+ * wfp_hpsm = file=508cb9dfbe1c7dca5ed24f124473f33d,300,asd.c
+ *             hpsm=1909ff06ff688630ff45a92b52f47eff3500ffff
+ *             11=b19bdbfa
+ *
+ * @param {string} wfp Complete wfp string (with md5 line)
+ * @param {string} hpsm
+ * @returns {string}
+ */
+function join_wfp_hpsm(wfp, hpsm) {
+  let header = wfp.match(/file=.*/);
+  header += String.fromCharCode(10);
+  header += hpsm;
+  wfp = wfp.replace(/file=.*/, "")
+  return header + wfp;
+}
+
+/**
+ *
+ * @param {Buffer} content
+ * @param {string} contentSource
+ * @param {number} maxSize
+ * @returns {string}
+ */
+function wfp_hpsm_for_content(content, contentSource, maxSize) {
+  let wfp = wfp_for_content(content, contentSource, maxSize)
+  let hpsm = calc_hpsm(content)   //Returns a string
+  let wfp_hpsm_joined = join_wfp_hpsm(wfp, hpsm)
+  return truncate_string(wfp_hpsm_joined, maxSize)
+}
+
+function wfp_for_content(content, contentSource, maxSize) {
+  let wfp = wfp_only_md5(content, contentSource);
+  wfp += calc_wfp(content, maxSize);
+  return wfp;
+}
+
+function wfp_only_md5(contents, contentSource) {
+  const file_md5 = crypto.createHash('md5').update(contents).digest('hex');
+  let wfp = 'file=' + String(file_md5) + ',' + String(contents.length) + ',' + String(contentSource)+ String.fromCharCode(10);
+  return wfp;
+}
+
+/**
+* CRC8_MAXIM algorithm begins
+*/
+
+
+//=============================================================================
+// Implementation of CRC-8/MAXIM-DOW,CRC-8/MAXIM,DOW-CRC.
+// Notes: Code ported from C Auto-generated by http://crc.drque.net/
+//=============================================================================
+
+//Table size
+const CRC8_MAXIM_DOW_TABLE_SIZE = 0x100
+const CRC8_MAXIM_DOW_POLYNOMIAL = 0x8C // 0x31 reflected
+const CRC8_MAXIM_DOW_INITIAL = 0x00 // 0x00 reflected
+const CRC8_MAXIM_DOW_FINAL = 0x00 // 0x00 reflected
+
+let crc8_MAXIM_DOW_Table = []
+
+/**
+ * Compute CRC of byte without lookup tables.
+ *
+ * @param {number} crc Current CRC or initial value
+ * @param {number} byte New byte to accumulate.
+ * @returns {number} Updated CRC.
+ */
+function crc8_MAXIM_DOW_ByteNoTable(crc, byte) {
+  crc ^= byte;
+  for (let count = 0; count<8; count++) {
+    const isSet = (crc & 0x01)
+    crc >>= 1;
+    if (isSet) crc ^= CRC8_MAXIM_DOW_POLYNOMIAL
+  }
+  return crc
+}
+
+/**
+ * Create the lookup table.
+ * Note: Must be called before any table based CRC calculations can be done.
+ */
+function crc8_MAXIM_DOW_GenerateTable() {
+  for(let i = 0; i<CRC8_MAXIM_DOW_TABLE_SIZE; i++) {
+    crc8_MAXIM_DOW_Table.push(crc8_MAXIM_DOW_ByteNoTable(0, i))
+  }
+}
+
+/**
+ * Update CRC by byte.
+ *
+ * @param {number} crc Initial CRC.
+ * @param {number} byte New byte to accumulate.
+ * @returns {number} Updated CRC.
+ */
+function crc8_MAXIM_DOW_Byte( crc, byte )
+{
+  const index = byte ^ crc;
+  return crc8_MAXIM_DOW_Table[ index ] ^ ( crc >> 8 );
+}
+
+/**
+ * Compute CRC of buffer.
+ * Note: crc8_MAXIM_DOW_GenerateTable() must be called before use this function
+ *
+ * @param {Buffer} buffer Buffer with bytes to calculate CRC.
+ * @returns {number} CRC
+ */
+function crc8_MAXIM_DOW_Buffer(buffer) {
+  let crc = CRC8_MAXIM_DOW_INITIAL;
+  for (let index = 0; index < buffer.length ; index ++) {
+    crc = crc8_MAXIM_DOW_Byte(crc, buffer[index])
+  }
+  crc ^= CRC8_MAXIM_DOW_FINAL;
+  return crc;
+}
 `;
 
 export class WfpCalculator extends WfpProvider {
@@ -260,7 +417,7 @@ export class WfpCalculator extends WfpProvider {
 
   recoveryIndex() {
     // Files: contains all files winnowed but not packed yet
-    const files = new FingerprintPacket(this.wfp, this.folderRoot).getFilesFingerprinted();
+    const files = new FingerprintPackage(this.wfp, this.folderRoot).getFilesFingerprinted();
     if (files.length) {
       const lastFileWinnowed = files[files.length - 1];
       let i = 0;
@@ -284,7 +441,7 @@ export class WfpCalculator extends WfpProvider {
   }
 
   protected processPackedWfp(content) {
-    const fingerprint = new FingerprintPacket(content, this.folderRoot);
+    const fingerprint = new FingerprintPackage(content, this.folderRoot);
     this.sendFingerprint(fingerprint);
   }
 

package/src/lib/scanner/WfpProvider/WfpProvider.ts

@@ -2,7 +2,7 @@ import { Worker } from 'worker_threads';
 import { EventEmitter } from "stream";
 import { ScannerCfg } from "../ScannerCfg";
 import { ScannerEvents, ScannerInput, WinnowingMode } from "../ScannerTypes";
-import { FingerprintPacket } from "./FingerprintPacket";
+import { FingerprintPackage } from "./FingerprintPackage";
 
 
 export interface IWfpProviderInput {
@@ -46,7 +46,7 @@ export abstract class WfpProvider extends EventEmitter {
     }
 
     if (this.wfp.length + fingerprint.length >= this.scannerCfg.WFP_FILE_MAX_SIZE) {
-      this.sendFingerprint(new FingerprintPacket(this.wfp, this.folderRoot));
+      this.sendFingerprint(new FingerprintPackage(this.wfp, this.folderRoot));
       this.wfp = '';
     }
     this.wfp += fingerprint;
@@ -56,14 +56,14 @@ export abstract class WfpProvider extends EventEmitter {
   }
 
   protected finishWinnowing() {
-    if (this.wfp.length !== 0) this.sendFingerprint(new FingerprintPacket(this.wfp, this.folderRoot));
+    if (this.wfp.length !== 0) this.sendFingerprint(new FingerprintPackage(this.wfp, this.folderRoot));
     this.pendingFiles = false;
     this.emit(ScannerEvents.WINNOWING_FINISHED);
   }
 
 
-  protected sendFingerprint(fingerprintPacket: FingerprintPacket) {
-    this.emit(ScannerEvents.WINNOWING_NEW_CONTENT, fingerprintPacket);
+  protected sendFingerprint(fingerprintPackage: FingerprintPackage) {
+    this.emit(ScannerEvents.WINNOWING_NEW_CONTENT, fingerprintPackage);
   }
 
   protected sendLog(logMsg: string): void {

package/tests/WfpCalculator.spec.ts

@@ -0,0 +1,103 @@
+import {WfpCalculator} from '../src/lib/scanner/WfpProvider/WfpCalculator/WfpCalculator';
+import { IWfpProviderInput } from '../src/lib/scanner/WfpProvider/WfpProvider';
+import {ScannerEvents, WinnowingMode} from '../src/lib/scanner/ScannerTypes'
+import {
+  FingerprintPackage
+} from '../src/lib/scanner/WfpProvider/FingerprintPackage';
+
+
+
+describe('Suit test for WfpCalculator Class', () => {
+
+  describe('Test Winnowings Modes', function () {
+
+    let wfpInput: IWfpProviderInput;
+    let wfpCalculator: WfpCalculator;
+
+    beforeEach(function() {
+      wfpCalculator = new WfpCalculator();
+      wfpCalculator.init();
+    });
+
+    it('Winnowing Full mode', function (done) {
+      wfpInput  = {
+        fileList: [__dirname + "/data/scanner/file1.c"],
+        folderRoot: __dirname + "/data/scanner/"
+      };
+
+      let wfpExpected = `file=736c0a4b4440e35baffc9378304ee588,907,file1.c
+8=c0e39912
+9=61eee030
+11=eff88a55
+16=9d9c5251,c8ca7af5
+21=520ee8e5
+26=1f2fa230,2e83cf84,57d0c3d2
+28=4a9386b1
+31=fbf8548e,b6002136
+36=5622280b
+39=8503d2f9
+40=0b62ff71
+`
+
+      wfpCalculator.on(ScannerEvents.WINNOWING_FINISHED, () => {
+        done()
+      });
+
+      wfpCalculator.on(ScannerEvents.ERROR, (err) => {
+        done(err)
+      })
+
+      wfpInput.winnowingMode=WinnowingMode.FULL_WINNOWING,
+        wfpCalculator.start(wfpInput)
+    });
+
+
+    it('Winnowing HPSM mode', function (done) {
+
+      wfpInput  = {
+        fileList: [__dirname + "/data/scanner/file1.c"],
+        folderRoot: __dirname + "/data/scanner/"
+      };
+
+      let wfpExpected = `file=736c0a4b4440e35baffc9378304ee588,907,file1.c
+hpsm=91ffe7ff989bffe7fcff91bdff2dff3d00a400ff9b00a400ffb6004c42ff9c004200ffc700ff62408b
+8=c0e39912
+9=61eee030
+11=eff88a55
+16=9d9c5251,c8ca7af5
+21=520ee8e5
+26=1f2fa230,2e83cf84,57d0c3d2
+28=4a9386b1
+31=fbf8548e,b6002136
+36=5622280b
+39=8503d2f9
+40=0b62ff71
+`
+
+      let wfpResponse: FingerprintPackage;
+      wfpCalculator.on(ScannerEvents.WINNOWING_NEW_CONTENT, (c: FingerprintPackage) => {
+        wfpResponse = c;
+      });
+
+      wfpCalculator.on(ScannerEvents.WINNOWING_FINISHED, () => {
+        if (wfpResponse.getContent() === wfpExpected) {
+          done()
+        } else {
+          done(new Error("Diferences in wfp generated and expected"))
+        }
+      });
+
+      wfpCalculator.on(ScannerEvents.ERROR, (err) => {
+        done(err)
+      })
+
+      wfpInput.winnowingMode=WinnowingMode.FULL_WINNOWING_HPSM;
+      wfpCalculator.start(wfpInput)
+    });
+
+
+  });
+
+});
+
+

package/tests/data/dependencies/Gemfile/1/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+gem 'nokogiri'
+gem 'rails', '3.0.0.beta3'
+gem 'rack',  '>=1.0'
+gem 'thin',  '~>1.1'
+

package/tests/data/dependencies/Gemfile/2/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+# Gems here

package/tests/data/dependencies/Gemfile/2/Gemfile~

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+gem 'nokogiri'
+gem 'rails', '3.0.0.beta3'
+gem 'rack',  '>=1.0'
+gem 'thin',  '~>1.1'
+

package/tests/data/dependencies/Gemfile/3/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+# Gems here
+
+source 'https://gems.example.com' do
+  # Gems from the alternative source here
+end

package/tests/data/dependencies/Gemfile/3/Gemfile~

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+gem 'nokogiri'
+gem 'rails', '3.0.0.beta3'
+gem 'rack',  '>=1.0'
+gem 'thin',  '~>1.1'
+

package/tests/data/dependencies/Gemfile/4/Gemfile

@@ -0,0 +1,31 @@
+source 'http://rubygems.org'
+
+gem 'rails', '3.0.7'
+
+# Bundle edge Rails instead:
+# gem 'rails', :git => 'git://github.com/rails/rails.git'
+
+gem 'sqlite3'
+
+# Use unicorn as the web server
+# gem 'unicorn'
+
+# Deploy with Capistrano
+# gem 'capistrano'
+
+# To use debugger (ruby-debug for Ruby 1.8.7+, ruby-debug19 for Ruby 1.9.2+)
+# gem 'ruby-debug'
+# gem 'ruby-debug19', :require => 'ruby-debug'
+
+# Bundle the extra gems:
+# gem 'bj'
+# gem 'nokogiri'
+# gem 'sqlite3-ruby', :require => 'sqlite3'
+# gem 'aws-s3', :require => 'aws/s3'
+
+# Bundle gems for the local environment. Make sure to
+# put test-only gems in this group so their generators
+# and rake tasks are available in development mode:
+# group :development, :test do
+#   gem 'webrat'
+# end

package/tests/data/dependencies/Gemfile/4/Gemfile~

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+# Gems here
+
+source 'https://gems.example.com' do
+  # Gems from the alternative source here
+end

package/tests/data/dependencies/Gemfile.lock/1/Gemfile.lock

@@ -0,0 +1,180 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actioncable (5.0.6)
+      actionpack (= 5.0.6)
+      nio4r (>= 1.2, < 3.0)
+      websocket-driver (~> 0.6.1)
+    actionmailer (5.0.6)
+      actionpack (= 5.0.6)
+      actionview (= 5.0.6)
+      activejob (= 5.0.6)
+      mail (~> 2.5, >= 2.5.4)
+      rails-dom-testing (~> 2.0)
+    actionpack (5.0.6)
+      actionview (= 5.0.6)
+      activesupport (= 5.0.6)
+      rack (~> 2.0)
+      rack-test (~> 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (5.0.6)
+      activesupport (= 5.0.6)
+      builder (~> 3.1)
+      erubis (~> 2.7.0)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.3)
+    activejob (5.0.6)
+      activesupport (= 5.0.6)
+      globalid (>= 0.3.6)
+    activemodel (5.0.6)
+      activesupport (= 5.0.6)
+    activerecord (5.0.6)
+      activemodel (= 5.0.6)
+      activesupport (= 5.0.6)
+      arel (~> 7.0)
+    activesupport (5.0.6)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (~> 0.7)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    arel (7.1.4)
+    bindex (0.5.0)
+    builder (3.2.3)
+    byebug (9.1.0)
+    coffee-rails (4.2.2)
+      coffee-script (>= 2.2.0)
+      railties (>= 4.0.0)
+    coffee-script (2.4.1)
+      coffee-script-source
+      execjs
+    coffee-script-source (1.12.2)
+    concurrent-ruby (1.0.5)
+    crass (1.0.2)
+    erubis (2.7.0)
+    execjs (2.7.0)
+    ffi (1.9.18)
+    globalid (0.4.0)
+      activesupport (>= 4.2.0)
+    i18n (0.8.6)
+    jbuilder (2.7.0)
+      activesupport (>= 4.2.0)
+      multi_json (>= 1.2)
+    jquery-rails (4.3.1)
+      rails-dom-testing (>= 1, < 3)
+      railties (>= 4.2.0)
+      thor (>= 0.14, < 2.0)
+    listen (3.0.8)
+      rb-fsevent (~> 0.9, >= 0.9.4)
+      rb-inotify (~> 0.9, >= 0.9.7)
+    loofah (2.1.1)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    mail (2.6.6)
+      mime-types (>= 1.16, < 4)
+    method_source (0.9.0)
+    mime-types (3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2016.0521)
+    mini_portile2 (2.3.0)
+    minitest (5.10.3)
+    multi_json (1.12.2)
+    nio4r (2.1.0)
+    nokogiri (1.8.1)
+      mini_portile2 (~> 2.3.0)
+    puma (3.10.0)
+    rack (2.0.3)
+    rack-test (0.6.3)
+      rack (>= 1.0)
+    rails (5.0.6)
+      actioncable (= 5.0.6)
+      actionmailer (= 5.0.6)
+      actionpack (= 5.0.6)
+      actionview (= 5.0.6)
+      activejob (= 5.0.6)
+      activemodel (= 5.0.6)
+      activerecord (= 5.0.6)
+      activesupport (= 5.0.6)
+      bundler (>= 1.3.0)
+      railties (= 5.0.6)
+      sprockets-rails (>= 2.0.0)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.0.3)
+      loofah (~> 2.0)
+    railties (5.0.6)
+      actionpack (= 5.0.6)
+      activesupport (= 5.0.6)
+      method_source
+      rake (>= 0.8.7)
+      thor (>= 0.18.1, < 2.0)
+    rake (12.1.0)
+    rb-fsevent (0.10.2)
+    rb-inotify (0.9.10)
+      ffi (>= 0.5.0, < 2)
+    sass (3.5.2)
+      sass-listen (~> 4.0.0)
+    sass-listen (4.0.0)
+      rb-fsevent (~> 0.9, >= 0.9.4)
+      rb-inotify (~> 0.9, >= 0.9.7)
+    sass-rails (5.0.6)
+      railties (>= 4.0.0, < 6)
+      sass (~> 3.1)
+      sprockets (>= 2.8, < 4.0)
+      sprockets-rails (>= 2.0, < 4.0)
+      tilt (>= 1.1, < 3)
+    spring (2.0.2)
+      activesupport (>= 4.2)
+    spring-watcher-listen (2.0.1)
+      listen (>= 2.7, < 4.0)
+      spring (>= 1.2, < 3.0)
+    sprockets (3.7.1)
+      concurrent-ruby (~> 1.0)
+      rack (> 1, < 3)
+    sprockets-rails (3.2.1)
+      actionpack (>= 4.0)
+      activesupport (>= 4.0)
+      sprockets (>= 3.0.0)
+    sqlite3 (1.3.13)
+    thor (0.20.0)
+    thread_safe (0.3.6)
+    tilt (2.0.8)
+    turbolinks (5.0.1)
+      turbolinks-source (~> 5)
+    turbolinks-source (5.0.3)
+    tzinfo (1.2.3)
+      thread_safe (~> 0.1)
+    uglifier (3.2.0)
+      execjs (>= 0.3.0, < 3)
+    web-console (3.5.1)
+      actionview (>= 5.0)
+      activemodel (>= 5.0)
+      bindex (>= 0.4.0)
+      railties (>= 5.0)
+    websocket-driver (0.6.5)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  byebug
+  coffee-rails (~> 4.2)
+  jbuilder (~> 2.5)
+  jquery-rails
+  listen (~> 3.0.5)
+  puma (~> 3.0)
+  rails (~> 5.0.0)
+  sass-rails (~> 5.0)
+  spring
+  spring-watcher-listen (~> 2.0.0)
+  sqlite3
+  turbolinks (~> 5)
+  tzinfo-data
+  uglifier (>= 1.3.0)
+  web-console
+
+BUNDLED WITH
+   1.15.4