scanoss 0.2.28 → 0.3.0

package/src/lib/scanner/WfpProvider/WfpCalculator/WfpCalculator.ts

@@ -8,7 +8,6 @@ import { ScannerEvents, ScannerInput, WinnowingMode } from '../../ScannerTypes';
 import { FingerprintPackage } from '../FingerprintPackage';
 import { IWfpProviderInput, WfpProvider } from '../WfpProvider';
 
-
 const stringWorker = `
 const { parentPort } = require('worker_threads');
 
@@ -21,7 +20,13 @@ parentPort.on('message', async (scannableItem) => {
       scannableItem.contentSource,
       scannableItem.maxSizeWfp
     );
-  } else {
+  } else if (scannableItem.winnowingMode === "FULL_WINNOWING_HPSM") {
+    fingerprint = wfp_hpsm_for_content(
+      scannableItem.content,
+      scannableItem.contentSource,
+      scannableItem.maxSizeWfp
+    );
+  } else if (scannableItem.winnowingMode === "WINNOWING_ONLY_MD5") {
     fingerprint = wfp_only_md5(
       scannableItem.content,
       scannableItem.contentSource
@@ -29,13 +34,15 @@ parentPort.on('message', async (scannableItem) => {
   }
 
   scannableItem.fingerprint = fingerprint;
-
   parentPort.postMessage(scannableItem);
-});
 
+  });
 
-const crypto = require('crypto');
+  /**
+  * Winnowing and HPSM algorithm begins
+  */
 
+const crypto = require('crypto');
 const isWin = process.platform === 'win32';
 const pathSeparator = isWin ? String.fromCharCode(92) : '/';
 
@@ -75,18 +82,6 @@ function min_hex_array(array) {
   return min;
 }
 
-function wfp_for_content(content, contentSource, maxSize) {
-  let wfp = wfp_only_md5(content, contentSource);
-  wfp += calc_wfp(content, maxSize);
-  return wfp;
-}
-
-function wfp_only_md5(contents, contentSource) {
-  const file_md5 = crypto.createHash('md5').update(contents).digest('hex');
-  let wfp = 'file=' + String(file_md5) + ',' + String(contents.length) + ',' + String(contentSource)+ String.fromCharCode(10);
-  return wfp;
-}
-
 function calc_wfp(contents, maxSize) {
   let gram = '';
   const window = [];
@@ -229,6 +224,168 @@ function crc32c_hex(str) {
   return crc32c(str).toString(16).padStart(8, '0');
 }
 
+function truncate_string(input, maxSize){
+  return input.slice(0, maxSize)
+}
+
+function toHexString(byteArray) {
+  return Array.from(byteArray, function(byte) {
+    return ('0' + (byte & 0xFF).toString(16)).slice(-2);
+  }).join('')
+}
+
+function calc_hpsm(content) {
+  let list_normalized = [];    //Array of numbers
+  let crc_lines = [];  //Array of numbers that represent the crc8_maxim for each line of the file
+
+  let last_line = 0;
+  crc8_MAXIM_DOW_GenerateTable();
+
+  for(let i = 0; i<content.length; i++) {
+    const c = content[i];
+    if (c == ASCII_LF) {  //When there is a new line
+      if (list_normalized.length) {
+        crc_lines.push(crc8_MAXIM_DOW_Buffer(list_normalized))
+        list_normalized=[];
+      } else if(last_line+1 == i) {
+        crc_lines.push(0xFF);
+      } else if (i-last_line  > 1) {
+        crc_lines.push(0x00)
+      }
+      last_line = i;
+    } else {
+      const c_normalized = normalize(c);
+      if (c_normalized != 0)  list_normalized.push(c_normalized)
+    }
+  }
+  return "hpsm=" + toHexString(crc_lines)
+}
+
+
+/**
+ * Create a wfp_hpsm package joining wfp (with md5 line) and a hpsm.
+ *
+ * Example:
+ * wfp = file=508cb9dfbe1c7dca5ed24f124473f33d,300,asd.c
+ *         11=b19bdbfa
+ *
+ * hpsm = hpsm=1909ff06ff688630ff45a92b52f47eff3500ffff
+ *
+ * wfp_hpsm = file=508cb9dfbe1c7dca5ed24f124473f33d,300,asd.c
+ *             hpsm=1909ff06ff688630ff45a92b52f47eff3500ffff
+ *             11=b19bdbfa
+ *
+ * @param {string} wfp Complete wfp string (with md5 line)
+ * @param {string} hpsm
+ * @returns {string}
+ */
+function join_wfp_hpsm(wfp, hpsm) {
+  let header = wfp.match(/file=.*/);
+  header += String.fromCharCode(10);
+  header += hpsm;
+  wfp = wfp.replace(/file=.*/, "")
+  return header + wfp;
+}
+
+/**
+ *
+ * @param {Buffer} content
+ * @param {string} contentSource
+ * @param {number} maxSize
+ * @returns {string}
+ */
+function wfp_hpsm_for_content(content, contentSource, maxSize) {
+  let wfp = wfp_for_content(content, contentSource, maxSize)
+  let hpsm = calc_hpsm(content)   //Returns a string
+  let wfp_hpsm_joined = join_wfp_hpsm(wfp, hpsm)
+  return truncate_string(wfp_hpsm_joined, maxSize)
+}
+
+function wfp_for_content(content, contentSource, maxSize) {
+  let wfp = wfp_only_md5(content, contentSource);
+  wfp += calc_wfp(content, maxSize);
+  return wfp;
+}
+
+function wfp_only_md5(contents, contentSource) {
+  const file_md5 = crypto.createHash('md5').update(contents).digest('hex');
+  let wfp = 'file=' + String(file_md5) + ',' + String(contents.length) + ',' + String(contentSource)+ String.fromCharCode(10);
+  return wfp;
+}
+
+/**
+* CRC8_MAXIM algorithm begins
+*/
+
+
+//=============================================================================
+// Implementation of CRC-8/MAXIM-DOW,CRC-8/MAXIM,DOW-CRC.
+// Notes: Code ported from C Auto-generated by http://crc.drque.net/
+//=============================================================================
+
+//Table size
+const CRC8_MAXIM_DOW_TABLE_SIZE = 0x100
+const CRC8_MAXIM_DOW_POLYNOMIAL = 0x8C // 0x31 reflected
+const CRC8_MAXIM_DOW_INITIAL = 0x00 // 0x00 reflected
+const CRC8_MAXIM_DOW_FINAL = 0x00 // 0x00 reflected
+
+let crc8_MAXIM_DOW_Table = []
+
+/**
+ * Compute CRC of byte without lookup tables.
+ *
+ * @param {number} crc Current CRC or initial value
+ * @param {number} byte New byte to accumulate.
+ * @returns {number} Updated CRC.
+ */
+function crc8_MAXIM_DOW_ByteNoTable(crc, byte) {
+  crc ^= byte;
+  for (let count = 0; count<8; count++) {
+    const isSet = (crc & 0x01)
+    crc >>= 1;
+    if (isSet) crc ^= CRC8_MAXIM_DOW_POLYNOMIAL
+  }
+  return crc
+}
+
+/**
+ * Create the lookup table.
+ * Note: Must be called before any table based CRC calculations can be done.
+ */
+function crc8_MAXIM_DOW_GenerateTable() {
+  for(let i = 0; i<CRC8_MAXIM_DOW_TABLE_SIZE; i++) {
+    crc8_MAXIM_DOW_Table.push(crc8_MAXIM_DOW_ByteNoTable(0, i))
+  }
+}
+
+/**
+ * Update CRC by byte.
+ *
+ * @param {number} crc Initial CRC.
+ * @param {number} byte New byte to accumulate.
+ * @returns {number} Updated CRC.
+ */
+function crc8_MAXIM_DOW_Byte( crc, byte )
+{
+  const index = byte ^ crc;
+  return crc8_MAXIM_DOW_Table[ index ] ^ ( crc >> 8 );
+}
+
+/**
+ * Compute CRC of buffer.
+ * Note: crc8_MAXIM_DOW_GenerateTable() must be called before use this function
+ *
+ * @param {Buffer} buffer Buffer with bytes to calculate CRC.
+ * @returns {number} CRC
+ */
+function crc8_MAXIM_DOW_Buffer(buffer) {
+  let crc = CRC8_MAXIM_DOW_INITIAL;
+  for (let index = 0; index < buffer.length ; index ++) {
+    crc = crc8_MAXIM_DOW_Byte(crc, buffer[index])
+  }
+  crc ^= CRC8_MAXIM_DOW_FINAL;
+  return crc;
+}
 `;
 
 export class WfpCalculator extends WfpProvider {

package/tests/WfpCalculator.spec.ts

@@ -0,0 +1,103 @@
+import {WfpCalculator} from '../src/lib/scanner/WfpProvider/WfpCalculator/WfpCalculator';
+import { IWfpProviderInput } from '../src/lib/scanner/WfpProvider/WfpProvider';
+import {ScannerEvents, WinnowingMode} from '../src/lib/scanner/ScannerTypes'
+import {
+  FingerprintPackage
+} from '../src/lib/scanner/WfpProvider/FingerprintPackage';
+
+
+
+describe('Suit test for WfpCalculator Class', () => {
+
+  describe('Test Winnowings Modes', function () {
+
+    let wfpInput: IWfpProviderInput;
+    let wfpCalculator: WfpCalculator;
+
+    beforeEach(function() {
+      wfpCalculator = new WfpCalculator();
+      wfpCalculator.init();
+    });
+
+    it('Winnowing Full mode', function (done) {
+      wfpInput  = {
+        fileList: [__dirname + "/data/scanner/file1.c"],
+        folderRoot: __dirname + "/data/scanner/"
+      };
+
+      let wfpExpected = `file=736c0a4b4440e35baffc9378304ee588,907,file1.c
+8=c0e39912
+9=61eee030
+11=eff88a55
+16=9d9c5251,c8ca7af5
+21=520ee8e5
+26=1f2fa230,2e83cf84,57d0c3d2
+28=4a9386b1
+31=fbf8548e,b6002136
+36=5622280b
+39=8503d2f9
+40=0b62ff71
+`
+
+      wfpCalculator.on(ScannerEvents.WINNOWING_FINISHED, () => {
+        done()
+      });
+
+      wfpCalculator.on(ScannerEvents.ERROR, (err) => {
+        done(err)
+      })
+
+      wfpInput.winnowingMode=WinnowingMode.FULL_WINNOWING,
+        wfpCalculator.start(wfpInput)
+    });
+
+
+    it('Winnowing HPSM mode', function (done) {
+
+      wfpInput  = {
+        fileList: [__dirname + "/data/scanner/file1.c"],
+        folderRoot: __dirname + "/data/scanner/"
+      };
+
+      let wfpExpected = `file=736c0a4b4440e35baffc9378304ee588,907,file1.c
+hpsm=91ffe7ff989bffe7fcff91bdff2dff3d00a400ff9b00a400ffb6004c42ff9c004200ffc700ff62408b
+8=c0e39912
+9=61eee030
+11=eff88a55
+16=9d9c5251,c8ca7af5
+21=520ee8e5
+26=1f2fa230,2e83cf84,57d0c3d2
+28=4a9386b1
+31=fbf8548e,b6002136
+36=5622280b
+39=8503d2f9
+40=0b62ff71
+`
+
+      let wfpResponse: FingerprintPackage;
+      wfpCalculator.on(ScannerEvents.WINNOWING_NEW_CONTENT, (c: FingerprintPackage) => {
+        wfpResponse = c;
+      });
+
+      wfpCalculator.on(ScannerEvents.WINNOWING_FINISHED, () => {
+        if (wfpResponse.getContent() === wfpExpected) {
+          done()
+        } else {
+          done(new Error("Diferences in wfp generated and expected"))
+        }
+      });
+
+      wfpCalculator.on(ScannerEvents.ERROR, (err) => {
+        done(err)
+      })
+
+      wfpInput.winnowingMode=WinnowingMode.FULL_WINNOWING_HPSM;
+      wfpCalculator.start(wfpInput)
+    });
+
+
+  });
+
+});
+
+

package/tests/data/dependencies/go.sum/1/go.sum

@@ -0,0 +1,119 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
+github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A=
+google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

package/tests/data/dependencies/go.sum/depJSON.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Reads a go.sum from file and prints to stdout all objects with purls and requirements
+
+# Example:
+# Input: cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+# Output: {purl: "pkg:golang/cloud.google.com/go", requirement: "v0.26.0"},
+
+# This is used to create the expectedOutput for the go.sum test
+# WARNING: This does not scape the special characters in the dep name
+
+
+
+#clean blank spaces
+file_content=$(sed '/^ *$/d' "$1")
+
+
+purls_and_versions=$(echo "$file_content" | awk '{print "{purl:" "\"" "pkg:golang/"$1 "\"" ", requirement:" "\"" $2 "\"" "},"}')
+
+#Delete go.mod substrings
+results=${purls_and_versions//\/go.mod/}
+
+echo "$results"