scanoss 0.2.26 → 0.3.0

package/src/lib/dependencies/LocalDependency/parsers/npmParser.ts

@@ -16,7 +16,6 @@ export function packageParser(fileContent: string, filePath: string): ILocalDepe
     const o = JSON.parse(fileContent);
     let devDeps = Object.keys(o.devDependencies || {});
     let deps = Object.keys(o.dependencies || {});
-    let listDeps = [...deps, ...devDeps];
 
     for(const name of deps){
         const purlString = new PackageURL(PURL_TYPE, undefined, name, undefined, undefined, undefined).toString();
@@ -34,18 +33,194 @@ export function packageParser(fileContent: string, filePath: string): ILocalDepe
 
 // Parse a package-lock.json file from node projects
 // See reference on: https://docs.npmjs.com/cli/v8/configuring-npm/package-json
-const MANIFEST_FILE_1 = 'package-lock.json';
 export function packagelockParser(fileContent: string, filePath: string): ILocalDependency {
 
     const results: ILocalDependency = {file: filePath, purls: []};
-    if(path.basename(filePath) != MANIFEST_FILE_1)
+
+    if(path.basename(filePath) != 'package-lock.json')
         return results;
 
-    const o = JSON.parse(fileContent).dependencies;
-    for (const [key, value] of Object.entries(o)) {
+    const packages = JSON.parse(fileContent)?.packages;
+
+    if(!packages) return results;
+
+    for (const [key, value] of Object.entries(packages)) {
         if(!key) continue;
-        let purl = new PackageURL(PURL_TYPE, undefined, key,value['version'], undefined, undefined).toString();
-        results.purls.push({purl});
+
+        const keySplit = key.split("/")
+        const depName = keySplit[keySplit.length-1]
+
+        let purl = new PackageURL(PURL_TYPE, undefined, depName,undefined, undefined, undefined).toString();
+        let req = value['version'];
+        results.purls.push({purl: purl, requirement: req});
     }
+
+    return results;
+}
+
+
+
+export function yarnLockParser(fileContent: string, filePath: string): ILocalDependency {
+  const results: ILocalDependency = {file: filePath, purls: []};
+
+  if(path.basename(filePath) != 'yarn.lock')
     return results;
+
+  const yarnVersion = yarnLockRecognizeVersion(fileContent)
+  if (yarnVersion === YarnLockVersionEnum.V1) return yarnLockV1Parser(fileContent, filePath)
+  else if (yarnVersion === YarnLockVersionEnum.V2) return yarnLockV2Parser(fileContent, filePath)
+
+  return results;
+}
+
+enum YarnLockVersionEnum {
+  "V1"  ,
+  "V2",
+  UnknownYarnLockFormat
+}
+
+/*
+    The start of v1 file has this:
+        # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+        # yarn lockfile v1
+
+    The start of v2 file has this:
+        # This file is generated by running "yarn install" inside your project.
+        # Manual changes might be lost - proceed with caution!
+
+        __metadata:
+ */
+export function yarnLockRecognizeVersion(fileContent: string): YarnLockVersionEnum {
+
+  const yarn = fileContent.split("\n", 10) //Check only the first 10 lines;
+  for (const line of yarn) {
+    if ( line.includes('__metadata:') ) return YarnLockVersionEnum.V2
+    if ( line.includes('yarn lockfile v1') ) return YarnLockVersionEnum.V1
+  }
+  return YarnLockVersionEnum.UnknownYarnLockFormat
+}
+
+export function yarnLockV1Parser(fileContent: string, filePath: string): ILocalDependency {
+
+  const results: ILocalDependency = {file: filePath, purls: []};
+
+  //Yield an array with each element is a dependency
+  /*
+    "@babel/core@^7.1.0", "@babel/core@^7.3.4":
+      version "7.3.4"
+      resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.3.4.tgz#921a5a13746c21e32445bf0798680e9d11a6530b"
+      integrity sha512-jRsuseXBo9pN197KnDwhhaaBzyZr2oIcLHHTt2oDdQrej5Qp57dCCJafWx5ivU8/alEYDpssYqv1MUqcxwQlrA==
+      dependencies:
+        "@babel/code-frame" "^7.0.0"
+        "@babel/generator" "^7.3.4"
+   */
+  const yl_dependencies = fileContent.split("\n\n");
+
+  for (const yl_dependency of yl_dependencies) {
+
+
+
+    const dependencyData: Record<string, string> = {}
+    const topRequirements = [];
+
+    const dep_lines = yl_dependency.split("\n");
+    if (dep_lines.every((line) =>  line.trim().startsWith("#") == true)) continue //All lines are coments
+    if (dep_lines.every((line) =>  line.trim() == "")) continue  //All lines are empty lines
+
+    for (const dep_line of dep_lines) {
+
+      // Clean comments and empty lines
+      const trimmed = dep_line.trim();
+      const comment = trimmed.startsWith('#');
+      if (!trimmed || comment) continue
+
+      // Do nothing with it's own dependencies
+      //    "@babel/code-frame" "^7.0.0"
+      //    "@babel/generator" "^7.3.4"
+      if (dep_line.startsWith(' '.repeat(4))) {}
+
+      //  version "7.3.4"
+      //  resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.3.4.tgz#921a5a13746c21e32445bf0798680e9d11a6530b"
+      //  integrity sha512-jRsuseXBo9pN197KnDwhhaaBzyZr2oIcLHHTt2oDdQrej5Qp57dCCJafWx5ivU8/alEYDpssYqv1MUqcxwQlrA==
+      //  dependencies:
+      else if (dep_line.startsWith(' '.repeat(2))) {
+        const dep = trimmed.split(" ")
+        const key = dep[0].trim();
+        if (key !== "dependencies:") {
+          dependencyData[key] = dep[1].replace(/"|'/g, "");
+        }
+      }
+
+      // the first line of a dependency has the name and requirements
+      //"@babel/core@^7.1.0", "@babel/core@^7.3.4":
+      else if (!dep_line.startsWith(' ')){
+        const dep = dep_line.replace(/:/g, "").split(",");
+        const requirements = dep.map(line => line.trim().replace(/"|'/g, ""));
+
+        for (const req of requirements) {
+
+          const atIndex = req.lastIndexOf("@")
+
+          let constraint = req.slice(atIndex+1)  // gets ^7.1.0
+          constraint = constraint.replace(/"|'/g, "");
+
+          const ns_name = req.slice(0, atIndex)
+
+          let ns = '';
+          let name = ns_name;
+          if (ns_name.includes("/")) {
+            const slashIndex = req.lastIndexOf("/")
+            ns = ns_name.slice(0,slashIndex);
+            name = ns_name.slice(slashIndex+1)
+          }
+
+          topRequirements.push({constraint: constraint, ns: ns, name: name });
+        }
+
+      }
+
+
+    }
+
+    //Make sure that name and namespace are equal for the same dependency
+    const isNsNameEqual = topRequirements.every((topRequirement) => {
+      return topRequirement.ns === topRequirements[0].ns && topRequirement.name === topRequirements[0].name
+    });
+
+    if (!isNsNameEqual) {
+      console.error("Different names for same dependency is not supported")
+      continue
+    }
+    const topRequirement = topRequirements[0];
+    const namespace = topRequirement.ns;
+    const name = topRequirement.name;
+    const version = dependencyData['version'];
+    const purl = new PackageURL(PURL_TYPE, namespace, name, version, undefined, undefined).toString()
+
+    let requirement = ''
+    for (const topRequirement of topRequirements) {
+      requirement += topRequirement.constraint + ", "
+    }
+    if (requirement.endsWith(", ")) {
+      requirement = requirement.slice(0, requirement.length-2)
+    }
+
+    results.purls.push({purl: purl, requirement: requirement})
+
+  }
+
+
+  return results;
+
+
+}
+
+
+export function yarnLockV2Parser(fileContent: string, filePath: string): ILocalDependency {
+
+  const results: ILocalDependency = {file: filePath, purls: []};
+
+
+  return results;
+
 }

package/src/lib/scanner/Dispatcher/DispatchableItem.ts

@@ -1,28 +1,62 @@
-import { FingerprintPacket } from "../WfpProvider/FingerprintPacket";
+import { FingerprintPackage } from '../WfpProvider/FingerprintPackage';
+import FormData from 'form-data';
+import { SbomMode } from '../ScannerTypes';
+
 export class DispatchableItem {
-  private fingerprintPacket: FingerprintPacket;
+  private readonly form: FormData;
 
   private errorCounter: number;
 
-  constructor(fingerprintPacket: FingerprintPacket) {
-    this.fingerprintPacket = fingerprintPacket;
+  private fingerprintPackage: FingerprintPackage;
+
+  private engineFlags: number;
+
+  private sbom: string;
+
+  private sbomMode: SbomMode;
+
+  constructor() {
     this.errorCounter = 0;
+    this.form = new FormData();
+  }
+
+  public getForm(): FormData {
+    this.form.append('filename', Buffer.from(this.fingerprintPackage.getContent()), 'data.wfp');
+    if(this.engineFlags) this.form.append('flags', this.engineFlags);
+
+    if(this.sbomMode && this.sbom) {
+      this.form.append('assets', this.sbom);
+      this.form.append('type', this.sbomMode);
+    }
+
+    return this.form;
   }
 
-  increaseErrorCounter() {
+  public increaseErrorCounter() {
     this.errorCounter += 1;
   }
 
-  public getWinnowerResponse(): FingerprintPacket {
-    return this.fingerprintPacket;
+  public getErrorCounter() {
+    return this.errorCounter;
   }
 
-  getContent() {
-    return this.fingerprintPacket.getContent();
+  public setFingerprintPackage(fingerprintPackage: FingerprintPackage) {
+    this.fingerprintPackage = fingerprintPackage;
   }
 
-  getErrorCounter() {
-    return this.errorCounter;
+  public getFingerprintPackage(): FingerprintPackage {
+    return this.fingerprintPackage;
+  }
+
+  public setEngineFlags(engineFlags: number) {
+    this.engineFlags = engineFlags;
   }
 
+  public setSbom(sbom: string, sbomMode: SbomMode) {
+    this.sbom = sbom;
+    this.sbomMode = sbomMode;
+  }
+
+
+
 }

package/src/lib/scanner/Dispatcher/Dispatcher.ts

@@ -11,6 +11,8 @@ import { ScannerCfg } from "../ScannerCfg";
 import { GlobalControllerAborter } from "./GlobalControllerAborter";
 import { DispatchableItem } from './DispatchableItem';
 
+const MAX_CONCURRENT_REQUEST = 30;
+
 export class Dispatcher extends EventEmitter {
   private scannerCfg: ScannerCfg;
 
@@ -27,6 +29,9 @@ export class Dispatcher extends EventEmitter {
   constructor(scannerCfg = new ScannerCfg()) {
     super();
     this.scannerCfg = scannerCfg;
+    if(this.scannerCfg.CONCURRENCY_LIMIT > MAX_CONCURRENT_REQUEST)
+      this.scannerCfg.CONCURRENCY_LIMIT = MAX_CONCURRENT_REQUEST;
+
     this.init();
   }
 
@@ -64,8 +69,8 @@ export class Dispatcher extends EventEmitter {
     this.globalAbortController.abortAll();
   }
 
-  public dispatchItem(disptItem: DispatchableItem): void {
-    this.pQueue.add(() => this.dispatch(disptItem));
+  public dispatchItem(item: DispatchableItem): void {
+    this.pQueue.add(() => this.dispatch(item));
 
     if (
       this.pQueue.size + this.pQueue.pending >= this.scannerCfg.DISPATCHER_QUEUE_SIZE_MAX_LIMIT &&
@@ -96,7 +101,6 @@ export class Dispatcher extends EventEmitter {
           if (this.scannerCfg.ABORT_ON_MAX_RETRIES) this.handleUnrecoverableError(error, disptItem);
           return;
         }
-       // const leftRetry = this.scannerCfg.MAX_RETRIES_FOR_RECOVERABLES_ERRORS - disptItem.getErrorCounter();
         this.emit(ScannerEvents.DISPATCHER_LOG,`[ SCANNER ]: Recoverable error happened sending WFP content to server. Reason: ${error.code || error.name}`);
         this.dispatchItem(disptItem);
         return;
@@ -105,21 +109,14 @@ export class Dispatcher extends EventEmitter {
     }
   }
 
-  async dispatch(disptItem: DispatchableItem) {
+  async dispatch(item: DispatchableItem) {
     const timeoutController = this.globalAbortController.getAbortController();
     const timeoutId = setTimeout(() => timeoutController.abort(), this.scannerCfg.TIMEOUT);
     try {
-      const form = new FormData();
-
-      form.append('filename', Buffer.from(disptItem.getContent()), 'data.wfp');
-
-      const engineFlag = disptItem.getWinnowerResponse().getEngineFlags();
-      if(engineFlag) form.append('flags', engineFlag);
-
       this.emit(ScannerEvents.DISPATCHER_WFP_SENDED);
       const response = await fetch(this.scannerCfg.API_URL, {
         method: 'post',
-        body: form,
+        body: item.getForm(),
         headers: { 'User-Agent': this.scannerCfg.CLIENT_TIMESTAMP, 'X-Session': this.scannerCfg.API_KEY },
         signal: timeoutController.signal,
       });
@@ -138,13 +135,13 @@ export class Dispatcher extends EventEmitter {
       const dataAsText = await response.text();
       const dataAsObj = JSON.parse(dataAsText);
 
-      const dispatcherResponse = new DispatcherResponse(dataAsObj, disptItem.getContent());
+      const dispatcherResponse = new DispatcherResponse(dataAsObj, item.getFingerprintPackage().getContent());
       this.emit(ScannerEvents.DISPATCHER_NEW_DATA, dispatcherResponse);
       return Promise.resolve();
     } catch (e) {
         clearTimeout(timeoutId);
         this.globalAbortController.removeAbortController(timeoutController);
-        this.errorHandler(e, disptItem);
+        this.errorHandler(e, item);
         return Promise.resolve();
     }
   }

package/src/lib/scanner/Scanner.ts

@@ -15,15 +15,13 @@ import { ScannerEvents, ScannerInput } from './ScannerTypes';
 import sortPaths from 'sort-paths';
 
 import { WfpProvider } from './WfpProvider/WfpProvider';
-import { FingerprintPacket } from './WfpProvider/FingerprintPacket';
+import { FingerprintPackage } from './WfpProvider/FingerprintPackage';
 import { WfpCalculator } from './WfpProvider/WfpCalculator/WfpCalculator';
 import { WfpSplitter } from './WfpProvider/WfpSplitter/WfpSplitter';
 
 let finishPromiseResolve;
 let finishPromiseReject;
 
-
-
 export class Scanner extends EventEmitter {
   private scannerCfg: ScannerCfg;
 
@@ -88,12 +86,18 @@ export class Scanner extends EventEmitter {
   }
 
   setWinnowerListeners() {
-    this.wfpProvider.on(ScannerEvents.WINNOWING_NEW_CONTENT, (fingerprintPacket: FingerprintPacket) => {
-      this.emit(ScannerEvents.WINNOWING_NEW_CONTENT, fingerprintPacket);
+    this.wfpProvider.on(ScannerEvents.WINNOWING_NEW_CONTENT, (fingerprintPackage: FingerprintPackage) => {
+      this.emit(ScannerEvents.WINNOWING_NEW_CONTENT, fingerprintPackage);
       this.reportLog(`[ SCANNER ]: New WFP content`);
-      fingerprintPacket.setEngineFlags(this.scannerInput[0].engineFlags);
-      const disptItem = new DispatchableItem(fingerprintPacket);
-      this.dispatcher.dispatchItem(disptItem);
+
+      const item = new DispatchableItem();
+      item.setFingerprintPackage(fingerprintPackage);
+      if(this.scannerInput[0]?.engineFlags) item.setEngineFlags(this.scannerInput[0]?.engineFlags);
+
+      if(this.scannerInput[0]?.sbom && this.scannerInput[0]?.sbomMode)
+        item.setSbom(this.scannerInput[0]?.sbom, this.scannerInput[0]?.sbomMode);
+
+      this.dispatcher.dispatchItem(item);
     });
 
     this.wfpProvider.on(ScannerEvents.WINNOWER_LOG, (msg) => {
@@ -116,7 +120,7 @@ export class Scanner extends EventEmitter {
       this.wfpProvider.resume();
     });
 
-    this.dispatcher.on(ScannerEvents.DISPATCHER_NEW_DATA, async (response) => {
+    this.dispatcher.on(ScannerEvents.DISPATCHER_NEW_DATA, async (response: DispatcherResponse) => {
       this.processingNewData = true;
       this.processedFiles += response.getNumberOfFilesScanned();
       this.reportLog(`[ SCANNER ]: Received results of ${response.getNumberOfFilesScanned()} files`);
@@ -134,8 +138,8 @@ export class Scanner extends EventEmitter {
       }
     });
 
-    this.dispatcher.on(ScannerEvents.DISPATCHER_ITEM_NO_DISPATCHED, (disptItem) => {
-      const filesNotScanned = disptItem.getWinnowerResponse().getFilesWinnowed();
+    this.dispatcher.on(ScannerEvents.DISPATCHER_ITEM_NO_DISPATCHED, (disptItem: DispatchableItem) => {
+      const filesNotScanned = disptItem.getFingerprintPackage().getFilesFingerprinted();
       this.appendFilesToNotScanned(filesNotScanned);
     });
 
@@ -143,8 +147,8 @@ export class Scanner extends EventEmitter {
       this.reportLog(msg);
     });
 
-    this.dispatcher.on(ScannerEvents.ERROR, (error, disptItem) => {
-      const wfpContent = disptItem.getWinnowerResponse().getContent();
+    this.dispatcher.on(ScannerEvents.ERROR, (error: Error, disptItem: DispatchableItem) => {
+      const wfpContent = disptItem.getFingerprintPackage().getContent();
       fs.writeFileSync(`${this.workDirectory}/failed.wfp`, wfpContent, 'utf8');
       this.errorHandler(error, ScannerEvents.MODULE_DISPATCHER);
     });

package/src/lib/scanner/ScannerCfg.ts

@@ -20,10 +20,11 @@ export class ScannerCfg {
 
   ABORT_ON_MAX_RETRIES = true;
 
-  // Persist results after [ X ] number of server responses.
+  // Persist results after [ X ] server responses
   MAX_RESPONSES_IN_BUFFER = 300;
 
   DISPATCHER_QUEUE_SIZE_MAX_LIMIT = 300;
 
   DISPATCHER_QUEUE_SIZE_MIN_LIMIT = 200;
+
 };

package/src/lib/scanner/ScannerTypes.ts

@@ -30,13 +30,21 @@ export enum ScannerEvents {
 
 export enum WinnowingMode {
   FULL_WINNOWING = 'FULL_WINNOWING',
+  FULL_WINNOWING_HPSM = 'FULL_WINNOWING_HPSM',
   WINNOWING_ONLY_MD5 = 'WINNOWING_ONLY_MD5',
 };
 
+export enum SbomMode {
+  SBOM_IGNORE = 'blacklist',
+  SBOM_IDENTIFY = 'identify'
+}
+
 export interface ScannerInput {
-  engineFlags?: number;
-  folderRoot?: string;
   fileList: Array<string>;
+  folderRoot?: string;
+  engineFlags?: number;
   winnowingMode?: WinnowingMode;  // Enable winnowing algorithm, otherwise is scanned only MD5
   wfpPath?: string;
+  sbom?: string;
+  sbomMode?: SbomMode;
 };

package/src/lib/scanner/WfpProvider/{FingerprintPacket.ts → FingerprintPackage.ts}

@@ -1,8 +1,6 @@
-export class FingerprintPacket {
+export class FingerprintPackage {
   private wfpContent: string;
 
-  private engineFlags: number;
-
   private scanRoot: string;
 
   constructor(wfpContent: string, scanRoot = '') {
@@ -10,22 +8,14 @@ export class FingerprintPacket {
     this.scanRoot = scanRoot;
   }
 
-  public isEqual(fingerprintPacket: FingerprintPacket): boolean {
-    return this.getContent() === fingerprintPacket.getContent();
+  public isEqual(fingerprintPackage: FingerprintPackage): boolean {
+    return this.getContent() === fingerprintPackage.getContent();
   }
 
-  public getContent() {
+  public getContent(): string {
     return this.wfpContent;
   }
 
-  public setEngineFlags(engineFlags: number): void {
-    this.engineFlags = engineFlags;
-  }
-
-  public getEngineFlags(): number {
-    return this.engineFlags;
-  }
-
   public getNumberFilesFingerprinted() {
     const match = this.getContent().match(/file=/g);
     if(!match) return 0;