scanoss 0.2.28 → 0.3.0

package/src/commands/fingerprint.ts

@@ -1,11 +1,14 @@
-import { isFolder } from "./helpers";
-import { ScannerEvents, WfpCalculator } from "..";
-import { Tree } from "../lib/tree/Tree";
-import { FilterList } from "../lib/filters/filtering";
-import { FingerprintPackage } from "../lib/scanner/WfpProvider/FingerprintPackage";
+import { isFolder } from './helpers';
+import { ScannerEvents, WfpCalculator, WinnowingMode } from '..';
+import { Tree } from '../lib/tree/Tree';
+import { FilterList } from '../lib/filters/filtering';
+import {
+  FingerprintPackage
+} from '../lib/scanner/WfpProvider/FingerprintPackage';
 import fs from 'fs';
-import { defaultFilter } from "../lib/filters/defaultFilter";
+import { defaultFilter } from '../lib/filters/defaultFilter';
 import cliProgress from 'cli-progress';
+import { IWfpProviderInput } from '../lib/scanner/WfpProvider/WfpProvider';
 
 
 export async function fingerprintHandler(rootPath: string, options: any): Promise<void> {
@@ -15,14 +18,19 @@ export async function fingerprintHandler(rootPath: string, options: any): Promis
   const pathIsFolder = await isFolder(rootPath);
   const wfpCalculator = new WfpCalculator();
 
-  const tree = new Tree(rootPath);
-  const filter = new FilterList('');
-  filter.load(defaultFilter as FilterList);
+  let filesToFingerprint: string[] = [];
+  if (pathIsFolder) {
+    const tree = new Tree(rootPath);
+    const filter = new FilterList('');
+    filter.load(defaultFilter as FilterList);
 
-  tree.loadFilter(filter);
-  tree.buildTree();
+    tree.loadFilter(filter);
+    tree.buildTree();
+    filesToFingerprint = tree.getFileList();
+  } else {
+    filesToFingerprint.push(rootPath)
+  }
 
-  const filesToFingerprint = tree.getFileList();
 
   const optBar1 = { format: 'Fingerprinting Progress: [{bar}] {percentage}% | Fingerprinted {value} files of {total}' };
   const bar1 = new cliProgress.SingleBar(optBar1, cliProgress.Presets.shades_classic);
@@ -48,8 +56,9 @@ export async function fingerprintHandler(rootPath: string, options: any): Promis
     }
   });
 
-
-  wfpCalculator.start({fileList: filesToFingerprint, folderRoot: rootPath});
+  const wfpInput: IWfpProviderInput = {fileList: filesToFingerprint, folderRoot: rootPath}
+  if(options.hpsm) wfpInput.winnowingMode = WinnowingMode.FULL_WINNOWING_HPSM;
+  wfpCalculator.start(wfpInput);
 
 
 }

package/src/commands/scan.ts

@@ -1,10 +1,17 @@
 import { Scanner } from '../lib/scanner/Scanner';
-import { SbomMode, ScannerEvents, ScannerInput } from '../lib/scanner/ScannerTypes';
+import {
+  SbomMode,
+  ScannerEvents,
+  ScannerInput,
+  WinnowingMode
+} from '../lib/scanner/ScannerTypes';
 import { ScannerCfg } from '../lib/scanner/ScannerCfg';
 import { Tree } from '../lib/tree/Tree';
 
 import cliProgress from 'cli-progress';
-import { DispatcherResponse } from '../lib/scanner/Dispatcher/DispatcherResponse';
+import {
+  DispatcherResponse
+} from '../lib/scanner/Dispatcher/DispatcherResponse';
 import { defaultFilter } from '../lib/filters/defaultFilter';
 import { FilterList } from '../lib/filters/filtering';
 
@@ -80,6 +87,7 @@ export async function scanHandler(rootPath: string, options: any): Promise<void>
   });
 
   if (options.wfp) scannerInput.wfpPath = rootPath;
+  if (options.hpsm) scannerInput.winnowingMode = WinnowingMode.FULL_WINNOWING_HPSM
 
   if (options.ignore) {
     scannerInput.sbom = fs.readFileSync(options.ignore, 'utf-8');

package/src/lib/dependencies/DependencyScanner.ts

@@ -26,9 +26,10 @@ export class DependencyScanner {
     const grpcResponse = await this.grpcDependencyService.get(request);
     const response = grpcResponse.toObject();
 
-
     // Extract scope from localDependencies and add it to response
-    this.mergeScopeField(localDependencies, response);
+    // Also adds the requirements field from localDependency to the response if the server didn't
+    // replay back a version
+    this.repairOutput(localDependencies, response);
     return response;
   }
 
@@ -71,27 +72,34 @@ export class DependencyScanner {
     }
   }
 
-  private mergeScopeField(localdependency: ILocalDependencies, serverResponse: DependencyResponse.AsObject
-    ): IDependencyResponse {
-
-    const scopeHashMap = {};
+  private repairOutput(localdependency: ILocalDependencies, serverResponse: DependencyResponse.AsObject) {
 
+    // Create a map with key = [filename + purl] and the value is an object containing:
+    // * The scope of the local dependency
+    // * The requirement of the local dependency
+    // Later this map is used to add information in the server response
+    const localDependencyInfo = {};
     for (const file of localdependency.files) {
       const filename = file.file
-      for (const dependency of file.purls) {
-        if (dependency?.scope) scopeHashMap[filename + dependency.purl] = dependency.scope;
+      for (const localDependency of file.purls) {
+        const localInfo = {}
+        if (localDependency?.scope) localInfo['scope'] = localDependency.scope
+        if(localDependency?.requirement) localInfo['requirement'] = localDependency.requirement
+        localDependencyInfo[filename + localDependency.purl] = localInfo;
       }
     }
 
     for (const file of serverResponse.filesList) {
       const filename = file.file
       for (const dependency of file.dependenciesList) {
-        const scope = scopeHashMap[filename + dependency.purl];
-        if (scope) dependency['scope'] = scope;
+        const localDependencyData = localDependencyInfo[filename + dependency.purl];
+        if (localDependencyData?.scope) dependency['scope'] = localDependencyData.scope;
+        if (localDependencyData?.requirement && dependency.version == "") {
+          dependency.version = localDependencyData.requirement;
+        }
       }
     }
-
-    return serverResponse;
   }
 
+
 }

package/src/lib/dependencies/LocalDependency/LocalDependency.ts

@@ -3,9 +3,13 @@ import fs from 'fs';
 import { ParserFuncType, ILocalDependencies } from "./DependencyTypes";
 import { requirementsParser } from "./parsers/pyParser";
 import { pomParser } from "./parsers/mavenParser";
-import { packagelockParser, packageParser } from "./parsers/npmParser";
+import {
+  packagelockParser,
+  packageParser,
+  yarnLockParser
+} from './parsers/npmParser';
 import { gemfilelockParser, gemfileParser } from "./parsers/rubyParser";
-import { goModParser } from './parsers/golangParser';
+import { goModParser, goSumParser } from './parsers/golangParser';
 
 export class LocalDependencies {
 
@@ -24,6 +28,8 @@ export class LocalDependencies {
         'Gemfile': gemfileParser,
         'Gemfile.lock': gemfilelockParser,
         'go.mod': goModParser,
+        'go.sum': goSumParser,
+        'yarn.lock': yarnLockParser
       };
   }
 

package/src/lib/dependencies/LocalDependency/parsers/golangParser.ts

@@ -3,15 +3,6 @@ import { ILocalDependency } from "../DependencyTypes";
 import { PackageURL } from "packageurl-js";
 import path from "path";
 
-function parseModule (str: string) {
-  const res = /(?<type>[^\s]+)(?:\s)+(?<ns_name>[^\s]+)\s?(?<version>(.*))/.exec(str);
-  return {
-    type: res.groups.type,
-    ns_name: res.groups.ns_name,
-    version: res.groups.version
-  };
-}
-
 function parseDepLink (str: string) {
   const res = /.*?(?<ns_name>[^\s]+)\s+(?<version>(.*))/.exec(str);
   return {
@@ -20,6 +11,16 @@ function parseDepLink (str: string) {
   };
 }
 
+function getDepDataGoModFromLine(line: string) {
+  const {ns_name, version} = parseDepLink(line);
+
+  const index = ns_name.lastIndexOf('/');
+  const namespace = ns_name.substring(0, index);
+  const name = ns_name.substring(index + 1);
+
+  return {namespace, name, version}
+}
+
 // Removes comments and spaces
 function preprocessLine(line: string) {
     if (line.includes("//"))
@@ -45,7 +46,6 @@ export function goModParser(fileContent: string, filePath: string): ILocalDepend
   const lines =	fileContent.split('\n');
 
 	const require = [];
-	const exclude = [];
 
   for (let num = 0 ; num < lines.length ; num+=1) {
 
@@ -57,11 +57,7 @@ export function goModParser(fileContent: string, filePath: string): ILocalDepend
       line = preprocessLine(lines[num]);
       while (num < lines.length && line!==')') {
 
-        const {ns_name, version} = parseDepLink(line);
-
-        const index = ns_name.lastIndexOf('/');
-        const namespace = ns_name.substring(0, index);
-        const name = ns_name.substring(index + 1);
+        const {namespace, name, version} = getDepDataGoModFromLine(line)
 
         const purlString = new PackageURL(PURL_TYPE, namespace, name, version, undefined, undefined).toString();
         results.purls.push({purl: purlString});
@@ -76,3 +72,59 @@ export function goModParser(fileContent: string, filePath: string): ILocalDepend
 
   return results;
 }
+
+
+
+
+
+function parseGoSumDepLink (str: string) {
+  const res = /.*?(?<ns_name>[^\s]+)\s+(?<version>(.*))\s+h1:(?<checksum>(.*))/.exec(str);
+  return {
+    ns_name: res?.groups?.ns_name,
+    version: res?.groups?.version,
+    checksum: res?.groups?.checksum
+  };
+}
+
+function getDepDataGoSumFromLine(line: string) {
+  const {ns_name, version} = parseGoSumDepLink(line);
+
+  if (!ns_name) return {};
+
+  const index = ns_name.lastIndexOf('/');
+  const namespace = ns_name.substring(0, index);
+  const name = ns_name.substring(index + 1);
+
+  return {namespace, name, version}
+}
+
+// See reference on: https://go.dev/ref/mod#go-mod-file
+export function goSumParser(fileContent: string, filePath: string): ILocalDependency {
+
+  // If the file is not a go.mod manifest file, return an empty results
+  const results: ILocalDependency = { file: filePath, purls: [] };
+  if (path.basename(filePath) != 'go.sum')
+    return results;
+
+
+  const lines = fileContent.split('\n');
+  for (let num = 0; num < lines.length; num += 1) {
+
+    let line = preprocessLine(lines[num]);  //Deletes coments
+    if(!line) continue
+
+    line = line.replace('/go.mod', '')
+    const {namespace, name, version} = getDepDataGoSumFromLine(line)
+
+    if (!name) continue
+
+    //const purlString = new PackageURL(PURL_TYPE, namespace, name, undefined, undefined, undefined).toString();
+    const purlString = `pkg:${PURL_TYPE}/${namespace}/${name}`
+    results.purls.push({purl: purlString, requirement: version})
+  }
+
+  return results;
+
+
+}
+

package/src/lib/dependencies/LocalDependency/parsers/npmParser.ts

@@ -16,7 +16,6 @@ export function packageParser(fileContent: string, filePath: string): ILocalDepe
     const o = JSON.parse(fileContent);
     let devDeps = Object.keys(o.devDependencies || {});
     let deps = Object.keys(o.dependencies || {});
-    let listDeps = [...deps, ...devDeps];
 
     for(const name of deps){
         const purlString = new PackageURL(PURL_TYPE, undefined, name, undefined, undefined, undefined).toString();
@@ -34,18 +33,194 @@ export function packageParser(fileContent: string, filePath: string): ILocalDepe
 
 // Parse a package-lock.json file from node projects
 // See reference on: https://docs.npmjs.com/cli/v8/configuring-npm/package-json
-const MANIFEST_FILE_1 = 'package-lock.json';
 export function packagelockParser(fileContent: string, filePath: string): ILocalDependency {
 
     const results: ILocalDependency = {file: filePath, purls: []};
-    if(path.basename(filePath) != MANIFEST_FILE_1)
+
+    if(path.basename(filePath) != 'package-lock.json')
         return results;
 
-    const o = JSON.parse(fileContent).dependencies;
-    for (const [key, value] of Object.entries(o)) {
+    const packages = JSON.parse(fileContent)?.packages;
+
+    if(!packages) return results;
+
+    for (const [key, value] of Object.entries(packages)) {
         if(!key) continue;
-        let purl = new PackageURL(PURL_TYPE, undefined, key,value['version'], undefined, undefined).toString();
-        results.purls.push({purl});
+
+        const keySplit = key.split("/")
+        const depName = keySplit[keySplit.length-1]
+
+        let purl = new PackageURL(PURL_TYPE, undefined, depName,undefined, undefined, undefined).toString();
+        let req = value['version'];
+        results.purls.push({purl: purl, requirement: req});
     }
+
+    return results;
+}
+
+
+
+export function yarnLockParser(fileContent: string, filePath: string): ILocalDependency {
+  const results: ILocalDependency = {file: filePath, purls: []};
+
+  if(path.basename(filePath) != 'yarn.lock')
     return results;
+
+  const yarnVersion = yarnLockRecognizeVersion(fileContent)
+  if (yarnVersion === YarnLockVersionEnum.V1) return yarnLockV1Parser(fileContent, filePath)
+  else if (yarnVersion === YarnLockVersionEnum.V2) return yarnLockV2Parser(fileContent, filePath)
+
+  return results;
+}
+
+enum YarnLockVersionEnum {
+  "V1"  ,
+  "V2",
+  UnknownYarnLockFormat
+}
+
+/*
+    The start of v1 file has this:
+        # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+        # yarn lockfile v1
+
+    The start of v2 file has this:
+        # This file is generated by running "yarn install" inside your project.
+        # Manual changes might be lost - proceed with caution!
+
+        __metadata:
+ */
+export function yarnLockRecognizeVersion(fileContent: string): YarnLockVersionEnum {
+
+  const yarn = fileContent.split("\n", 10) //Check only the first 10 lines;
+  for (const line of yarn) {
+    if ( line.includes('__metadata:') ) return YarnLockVersionEnum.V2
+    if ( line.includes('yarn lockfile v1') ) return YarnLockVersionEnum.V1
+  }
+  return YarnLockVersionEnum.UnknownYarnLockFormat
+}
+
+export function yarnLockV1Parser(fileContent: string, filePath: string): ILocalDependency {
+
+  const results: ILocalDependency = {file: filePath, purls: []};
+
+  //Yield an array with each element is a dependency
+  /*
+    "@babel/core@^7.1.0", "@babel/core@^7.3.4":
+      version "7.3.4"
+      resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.3.4.tgz#921a5a13746c21e32445bf0798680e9d11a6530b"
+      integrity sha512-jRsuseXBo9pN197KnDwhhaaBzyZr2oIcLHHTt2oDdQrej5Qp57dCCJafWx5ivU8/alEYDpssYqv1MUqcxwQlrA==
+      dependencies:
+        "@babel/code-frame" "^7.0.0"
+        "@babel/generator" "^7.3.4"
+   */
+  const yl_dependencies = fileContent.split("\n\n");
+
+  for (const yl_dependency of yl_dependencies) {
+
+
+
+    const dependencyData: Record<string, string> = {}
+    const topRequirements = [];
+
+    const dep_lines = yl_dependency.split("\n");
+    if (dep_lines.every((line) =>  line.trim().startsWith("#") == true)) continue //All lines are coments
+    if (dep_lines.every((line) =>  line.trim() == "")) continue  //All lines are empty lines
+
+    for (const dep_line of dep_lines) {
+
+      // Clean comments and empty lines
+      const trimmed = dep_line.trim();
+      const comment = trimmed.startsWith('#');
+      if (!trimmed || comment) continue
+
+      // Do nothing with it's own dependencies
+      //    "@babel/code-frame" "^7.0.0"
+      //    "@babel/generator" "^7.3.4"
+      if (dep_line.startsWith(' '.repeat(4))) {}
+
+      //  version "7.3.4"
+      //  resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.3.4.tgz#921a5a13746c21e32445bf0798680e9d11a6530b"
+      //  integrity sha512-jRsuseXBo9pN197KnDwhhaaBzyZr2oIcLHHTt2oDdQrej5Qp57dCCJafWx5ivU8/alEYDpssYqv1MUqcxwQlrA==
+      //  dependencies:
+      else if (dep_line.startsWith(' '.repeat(2))) {
+        const dep = trimmed.split(" ")
+        const key = dep[0].trim();
+        if (key !== "dependencies:") {
+          dependencyData[key] = dep[1].replace(/"|'/g, "");
+        }
+      }
+
+      // the first line of a dependency has the name and requirements
+      //"@babel/core@^7.1.0", "@babel/core@^7.3.4":
+      else if (!dep_line.startsWith(' ')){
+        const dep = dep_line.replace(/:/g, "").split(",");
+        const requirements = dep.map(line => line.trim().replace(/"|'/g, ""));
+
+        for (const req of requirements) {
+
+          const atIndex = req.lastIndexOf("@")
+
+          let constraint = req.slice(atIndex+1)  // gets ^7.1.0
+          constraint = constraint.replace(/"|'/g, "");
+
+          const ns_name = req.slice(0, atIndex)
+
+          let ns = '';
+          let name = ns_name;
+          if (ns_name.includes("/")) {
+            const slashIndex = req.lastIndexOf("/")
+            ns = ns_name.slice(0,slashIndex);
+            name = ns_name.slice(slashIndex+1)
+          }
+
+          topRequirements.push({constraint: constraint, ns: ns, name: name });
+        }
+
+      }
+
+
+    }
+
+    //Make sure that name and namespace are equal for the same dependency
+    const isNsNameEqual = topRequirements.every((topRequirement) => {
+      return topRequirement.ns === topRequirements[0].ns && topRequirement.name === topRequirements[0].name
+    });
+
+    if (!isNsNameEqual) {
+      console.error("Different names for same dependency is not supported")
+      continue
+    }
+    const topRequirement = topRequirements[0];
+    const namespace = topRequirement.ns;
+    const name = topRequirement.name;
+    const version = dependencyData['version'];
+    const purl = new PackageURL(PURL_TYPE, namespace, name, version, undefined, undefined).toString()
+
+    let requirement = ''
+    for (const topRequirement of topRequirements) {
+      requirement += topRequirement.constraint + ", "
+    }
+    if (requirement.endsWith(", ")) {
+      requirement = requirement.slice(0, requirement.length-2)
+    }
+
+    results.purls.push({purl: purl, requirement: requirement})
+
+  }
+
+
+  return results;
+
+
+}
+
+
+export function yarnLockV2Parser(fileContent: string, filePath: string): ILocalDependency {
+
+  const results: ILocalDependency = {file: filePath, purls: []};
+
+
+  return results;
+
 }

package/src/lib/scanner/ScannerCfg.ts

@@ -20,10 +20,11 @@ export class ScannerCfg {
 
   ABORT_ON_MAX_RETRIES = true;
 
-  // Persist results after [ X ] number of server responses.
+  // Persist results after [ X ] server responses
   MAX_RESPONSES_IN_BUFFER = 300;
 
   DISPATCHER_QUEUE_SIZE_MAX_LIMIT = 300;
 
   DISPATCHER_QUEUE_SIZE_MIN_LIMIT = 200;
+
 };

package/src/lib/scanner/ScannerTypes.ts

@@ -30,6 +30,7 @@ export enum ScannerEvents {
 
 export enum WinnowingMode {
   FULL_WINNOWING = 'FULL_WINNOWING',
+  FULL_WINNOWING_HPSM = 'FULL_WINNOWING_HPSM',
   WINNOWING_ONLY_MD5 = 'WINNOWING_ONLY_MD5',
 };
 

package/src/lib/scanner/WfpProvider/FingerprintPackage.ts

@@ -12,7 +12,7 @@ export class FingerprintPackage {
     return this.getContent() === fingerprintPackage.getContent();
   }
 
-  public getContent() {
+  public getContent(): string {
     return this.wfpContent;
   }