scanoss 0.2.26 → 0.3.0

package/src/commands/fingerprint.ts

@@ -1,11 +1,14 @@
-import { isFolder } from "./helpers";
-import { ScannerEvents, WfpCalculator } from "..";
-import { Tree } from "../lib/tree/Tree";
-import { FilterList } from "../lib/filters/filtering";
-import { FingerprintPacket } from "../lib/scanner/WfpProvider/FingerprintPacket";
+import { isFolder } from './helpers';
+import { ScannerEvents, WfpCalculator, WinnowingMode } from '..';
+import { Tree } from '../lib/tree/Tree';
+import { FilterList } from '../lib/filters/filtering';
+import {
+  FingerprintPackage
+} from '../lib/scanner/WfpProvider/FingerprintPackage';
 import fs from 'fs';
-import { defaultFilter } from "../lib/filters/defaultFilter";
+import { defaultFilter } from '../lib/filters/defaultFilter';
 import cliProgress from 'cli-progress';
+import { IWfpProviderInput } from '../lib/scanner/WfpProvider/WfpProvider';
 
 
 export async function fingerprintHandler(rootPath: string, options: any): Promise<void> {
@@ -15,23 +18,28 @@ export async function fingerprintHandler(rootPath: string, options: any): Promis
   const pathIsFolder = await isFolder(rootPath);
   const wfpCalculator = new WfpCalculator();
 
-  const tree = new Tree(rootPath);
-  const filter = new FilterList('');
-  filter.load(defaultFilter as FilterList);
+  let filesToFingerprint: string[] = [];
+  if (pathIsFolder) {
+    const tree = new Tree(rootPath);
+    const filter = new FilterList('');
+    filter.load(defaultFilter as FilterList);
 
-  tree.loadFilter(filter);
-  tree.buildTree();
+    tree.loadFilter(filter);
+    tree.buildTree();
+    filesToFingerprint = tree.getFileList();
+  } else {
+    filesToFingerprint.push(rootPath)
+  }
 
-  const filesToFingerprint = tree.getFileList();
 
   const optBar1 = { format: 'Fingerprinting Progress: [{bar}] {percentage}% | Fingerprinted {value} files of {total}' };
   const bar1 = new cliProgress.SingleBar(optBar1, cliProgress.Presets.shades_classic);
   bar1.start(filesToFingerprint.length, 0);
 
   let fingerprints = '';
-  wfpCalculator.on(ScannerEvents.WINNOWING_NEW_CONTENT, (fingerprintPacket: FingerprintPacket) => {
-    bar1.increment(fingerprintPacket.getNumberFilesFingerprinted());
-    fingerprints = fingerprints.concat( fingerprintPacket.getContent() );
+  wfpCalculator.on(ScannerEvents.WINNOWING_NEW_CONTENT, (fingerprintPackage: FingerprintPackage) => {
+    bar1.increment(fingerprintPackage.getNumberFilesFingerprinted());
+    fingerprints = fingerprints.concat( fingerprintPackage.getContent() );
   });
 
   if (options.verbose)
@@ -48,8 +56,9 @@ export async function fingerprintHandler(rootPath: string, options: any): Promis
     }
   });
 
-
-  wfpCalculator.start({fileList: filesToFingerprint, folderRoot: rootPath});
+  const wfpInput: IWfpProviderInput = {fileList: filesToFingerprint, folderRoot: rootPath}
+  if(options.hpsm) wfpInput.winnowingMode = WinnowingMode.FULL_WINNOWING_HPSM;
+  wfpCalculator.start(wfpInput);
 
 
 }

package/src/commands/scan.ts

@@ -1,10 +1,17 @@
 import { Scanner } from '../lib/scanner/Scanner';
-import { ScannerEvents, ScannerInput } from '../lib/scanner/ScannerTypes';
+import {
+  SbomMode,
+  ScannerEvents,
+  ScannerInput,
+  WinnowingMode
+} from '../lib/scanner/ScannerTypes';
 import { ScannerCfg } from '../lib/scanner/ScannerCfg';
 import { Tree } from '../lib/tree/Tree';
 
 import cliProgress from 'cli-progress';
-import { DispatcherResponse } from '../lib/scanner/Dispatcher/DispatcherResponse';
+import {
+  DispatcherResponse
+} from '../lib/scanner/Dispatcher/DispatcherResponse';
 import { defaultFilter } from '../lib/filters/defaultFilter';
 import { FilterList } from '../lib/filters/filtering';
 
@@ -13,7 +20,6 @@ import { isFolder } from './helpers';
 import fs from 'fs';
 
 
-
 export async function scanHandler(rootPath: string, options: any): Promise<void> {
 
   let scannerInput: ScannerInput = {fileList: []};
@@ -81,6 +87,13 @@ export async function scanHandler(rootPath: string, options: any): Promise<void>
   });
 
   if (options.wfp) scannerInput.wfpPath = rootPath;
+  if (options.hpsm) scannerInput.winnowingMode = WinnowingMode.FULL_WINNOWING_HPSM
+
+  if (options.ignore) {
+    scannerInput.sbom = fs.readFileSync(options.ignore, 'utf-8');
+    scannerInput.sbomMode = SbomMode.SBOM_IGNORE
+  }
+
   await scanner.scan([scannerInput]);
 
 }

package/src/lib/dependencies/DependencyScanner.ts

@@ -22,14 +22,14 @@ export class DependencyScanner {
     let localDependencies = await this.localDependency.search(files);
     if (localDependencies.files.length === 0) return {filesList: []};
     localDependencies = this.purlAdapter(localDependencies);
-
     const request = this.buildRequest(localDependencies);
     const grpcResponse = await this.grpcDependencyService.get(request);
     const response = grpcResponse.toObject();
 
-
     // Extract scope from localDependencies and add it to response
-    this.mergeScopeField(localDependencies, response);
+    // Also adds the requirements field from localDependency to the response if the server didn't
+    // replay back a version
+    this.repairOutput(localDependencies, response);
     return response;
   }
 
@@ -72,27 +72,34 @@ export class DependencyScanner {
     }
   }
 
-  private mergeScopeField(localdependency: ILocalDependencies, serverResponse: DependencyResponse.AsObject
-    ): IDependencyResponse {
-
-    const scopeHashMap = {};
+  private repairOutput(localdependency: ILocalDependencies, serverResponse: DependencyResponse.AsObject) {
 
+    // Create a map with key = [filename + purl] and the value is an object containing:
+    // * The scope of the local dependency
+    // * The requirement of the local dependency
+    // Later this map is used to add information in the server response
+    const localDependencyInfo = {};
     for (const file of localdependency.files) {
       const filename = file.file
-      for (const dependency of file.purls) {
-        if (dependency?.scope) scopeHashMap[filename + dependency.purl] = dependency.scope;
+      for (const localDependency of file.purls) {
+        const localInfo = {}
+        if (localDependency?.scope) localInfo['scope'] = localDependency.scope
+        if(localDependency?.requirement) localInfo['requirement'] = localDependency.requirement
+        localDependencyInfo[filename + localDependency.purl] = localInfo;
       }
     }
 
     for (const file of serverResponse.filesList) {
       const filename = file.file
       for (const dependency of file.dependenciesList) {
-        const scope = scopeHashMap[filename + dependency.purl];
-        if (scope) dependency['scope'] = scope;
+        const localDependencyData = localDependencyInfo[filename + dependency.purl];
+        if (localDependencyData?.scope) dependency['scope'] = localDependencyData.scope;
+        if (localDependencyData?.requirement && dependency.version == "") {
+          dependency.version = localDependencyData.requirement;
+        }
       }
     }
-
-    return serverResponse;
   }
 
+
 }

package/src/lib/dependencies/LocalDependency/LocalDependency.ts

@@ -3,9 +3,13 @@ import fs from 'fs';
 import { ParserFuncType, ILocalDependencies } from "./DependencyTypes";
 import { requirementsParser } from "./parsers/pyParser";
 import { pomParser } from "./parsers/mavenParser";
-import { packagelockParser, packageParser } from "./parsers/npmParser";
+import {
+  packagelockParser,
+  packageParser,
+  yarnLockParser
+} from './parsers/npmParser';
 import { gemfilelockParser, gemfileParser } from "./parsers/rubyParser";
-import { goModParser } from './parsers/golangParser';
+import { goModParser, goSumParser } from './parsers/golangParser';
 
 export class LocalDependencies {
 
@@ -24,6 +28,8 @@ export class LocalDependencies {
         'Gemfile': gemfileParser,
         'Gemfile.lock': gemfilelockParser,
         'go.mod': goModParser,
+        'go.sum': goSumParser,
+        'yarn.lock': yarnLockParser
       };
   }
 

package/src/lib/dependencies/LocalDependency/parsers/golangParser.ts

@@ -3,15 +3,6 @@ import { ILocalDependency } from "../DependencyTypes";
 import { PackageURL } from "packageurl-js";
 import path from "path";
 
-function parseModule (str: string) {
-  const res = /(?<type>[^\s]+)(?:\s)+(?<ns_name>[^\s]+)\s?(?<version>(.*))/.exec(str);
-  return {
-    type: res.groups.type,
-    ns_name: res.groups.ns_name,
-    version: res.groups.version
-  };
-}
-
 function parseDepLink (str: string) {
   const res = /.*?(?<ns_name>[^\s]+)\s+(?<version>(.*))/.exec(str);
   return {
@@ -20,6 +11,16 @@ function parseDepLink (str: string) {
   };
 }
 
+function getDepDataGoModFromLine(line: string) {
+  const {ns_name, version} = parseDepLink(line);
+
+  const index = ns_name.lastIndexOf('/');
+  const namespace = ns_name.substring(0, index);
+  const name = ns_name.substring(index + 1);
+
+  return {namespace, name, version}
+}
+
 // Removes comments and spaces
 function preprocessLine(line: string) {
     if (line.includes("//"))
@@ -45,7 +46,6 @@ export function goModParser(fileContent: string, filePath: string): ILocalDepend
   const lines =	fileContent.split('\n');
 
 	const require = [];
-	const exclude = [];
 
   for (let num = 0 ; num < lines.length ; num+=1) {
 
@@ -57,11 +57,7 @@ export function goModParser(fileContent: string, filePath: string): ILocalDepend
       line = preprocessLine(lines[num]);
       while (num < lines.length && line!==')') {
 
-        const {ns_name, version} = parseDepLink(line);
-
-        const index = ns_name.lastIndexOf('/');
-        const namespace = ns_name.substring(0, index);
-        const name = ns_name.substring(index + 1);
+        const {namespace, name, version} = getDepDataGoModFromLine(line)
 
         const purlString = new PackageURL(PURL_TYPE, namespace, name, version, undefined, undefined).toString();
         results.purls.push({purl: purlString});
@@ -76,3 +72,59 @@ export function goModParser(fileContent: string, filePath: string): ILocalDepend
 
   return results;
 }
+
+
+
+
+
+function parseGoSumDepLink (str: string) {
+  const res = /.*?(?<ns_name>[^\s]+)\s+(?<version>(.*))\s+h1:(?<checksum>(.*))/.exec(str);
+  return {
+    ns_name: res?.groups?.ns_name,
+    version: res?.groups?.version,
+    checksum: res?.groups?.checksum
+  };
+}
+
+function getDepDataGoSumFromLine(line: string) {
+  const {ns_name, version} = parseGoSumDepLink(line);
+
+  if (!ns_name) return {};
+
+  const index = ns_name.lastIndexOf('/');
+  const namespace = ns_name.substring(0, index);
+  const name = ns_name.substring(index + 1);
+
+  return {namespace, name, version}
+}
+
+// See reference on: https://go.dev/ref/mod#go-mod-file
+export function goSumParser(fileContent: string, filePath: string): ILocalDependency {
+
+  // If the file is not a go.mod manifest file, return an empty results
+  const results: ILocalDependency = { file: filePath, purls: [] };
+  if (path.basename(filePath) != 'go.sum')
+    return results;
+
+
+  const lines = fileContent.split('\n');
+  for (let num = 0; num < lines.length; num += 1) {
+
+    let line = preprocessLine(lines[num]);  //Deletes coments
+    if(!line) continue
+
+    line = line.replace('/go.mod', '')
+    const {namespace, name, version} = getDepDataGoSumFromLine(line)
+
+    if (!name) continue
+
+    //const purlString = new PackageURL(PURL_TYPE, namespace, name, undefined, undefined, undefined).toString();
+    const purlString = `pkg:${PURL_TYPE}/${namespace}/${name}`
+    results.purls.push({purl: purlString, requirement: version})
+  }
+
+  return results;
+
+
+}
+

package/src/lib/dependencies/LocalDependency/parsers/mavenParser.ts

@@ -4,8 +4,6 @@ import { ILocalDependency } from "../DependencyTypes";
 
 const PURL_TYPE = 'maven';
 
-
-
 // Parse a pom.txt file from maven manifest file
 // See reference on: https://maven.apache.org/guides/introduction/introduction-to-the-pom.html
 // and https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html
@@ -31,17 +29,9 @@ export function pomParser(fileContent: string, filePath: string): ILocalDependen
         const name = artifactId ? artifactId[1] : '';
 
         const versionReg = dependency.match(/<version>([^<]*)<\/version>/);
-        let version = versionReg ? versionReg[1] : '';
-
-        const ver = version.match(/\${(.*?)}/);
-        if(ver && ver.length >= 1) {
-          if(ver[1] === 'project.version') { // TODO: Add support for project.version
-            version = undefined;
-          } else {
-            const res = fileContent.match(new RegExp(`<${ver[1]}>([^<]*)<\/${ver[1]}>`));
-            version = res.length >= 1 ? res[1] : '';
-          }
-        }
+        let version = null;
+        if(versionReg && versionReg.length>0) version = resolve_version(versionReg[1], fileContent);
+
 
         let purlQualifiers;
         const type = dependency.match(/<type>([^<]*)<\/type>/);
@@ -52,11 +42,148 @@ export function pomParser(fileContent: string, filePath: string): ILocalDependen
 
         // Extract scope.
         const scopeRes = dependency.match(/<scope>([^<]*)<\/scope>/);
-        const scope = scopeRes ? scopeRes[1] : undefined;
-
+        const scope = scopeRes ? scopeRes[1] : null;
         const purlString = new PackageURL(PURL_TYPE, namespace, name, version, purlQualifiers, undefined).toString();
-        results.purls.push({purl: purlString, scope: scope});
+        results.purls.push({purl: purlString, scope});
       });
     }
     return results;
 }
+
+
+function resolve_version(dependency_version: string, file_content: string): string {
+  // See properties: https://maven.apache.org/pom.html#properties
+  let version = '';
+  if(dependency_version) {
+    if(/\${project.version}/.test(dependency_version)) {
+      version = extract_content_from_tag(file_content, ['project', 'version']);
+    } else if(/\${.*?}/.test(dependency_version)) {
+      const property = dependency_version.match(/\${(.*?)}/)[1];
+      const result = file_content.match(new RegExp(`<${property}>([^<]*)<\/${property}>`));
+      if (result && result.length>0) version = result[1];
+    } else {
+      version = dependency_version.toString();
+    }
+  }
+  return version;
+}
+
+
+function get_start_tag_name(line: string): string {
+  const result = line.match(/\<([\w\-\.]+).*?>/);
+  if (result) return result[1].trim();
+  return '';
+}
+
+function get_end_tag_name(line: string): string {
+  const result = line.match(/\<\/([\w\-\.]+) ?>/);
+  if (result) return result[1].trim();
+  return '';
+}
+
+function get_end_tag(line: string): string {
+  const result = get_end_tag_name(line);
+  if (result !== '') return `</${result}>`
+  return '';
+}
+
+function get_start_tag(line: string): string {
+  const result = get_start_tag_name(line);
+  if (result !== '') return `<${result}>`
+  return '';
+}
+
+function element_match(openTag: string, closeTag:string): boolean {
+  return get_start_tag_name(openTag) === get_end_tag_name(closeTag);
+}
+
+function is_element_complete(line: string) {
+  return get_start_tag_name(line) === get_end_tag_name(line);
+}
+
+
+function get_offset_until_end_of_tag(lines: Array<string>, end_tag_name: string) {
+  let i = 0;
+  for (const line of lines) {
+      if ( get_end_tag_name(line) === end_tag_name) break;
+      i += 1;
+  }
+  return i;
+}
+
+function remove_comments(lines: Array<string>): Array<string> {
+  for (let i=0; i<lines.length; i+=1) {
+      let openCommentFlag = /<!--/.test(lines[i]);
+      let endCommentFlag = /-->/.test(lines[i]);
+
+      if(openCommentFlag && endCommentFlag)
+          lines[i] = lines[i].replace(/<!--.*-->/, '');
+      else if (openCommentFlag){
+          while(!/-->/.test(lines[i]) && i<lines.length) {
+              lines[i] = '';
+              i += 1;
+          }
+          lines[i] = lines[i].replace(/.*-->/, '');
+      }
+
+  }
+  return lines;
+}
+
+
+function extract_content_from_tag(file_content: string, selector: Array<string>): string {
+  let lines = file_content.split('\n');
+  const stack: Array<string> = [];
+
+  let selectorIndex = 0;
+  let startTagName = '';
+  let endTagName = '';
+  let content = '';
+
+  // Sanitize xml: Removes comments
+  lines = remove_comments(lines);
+
+  for (let i=0; i<lines.length; i+=1) {
+    let line = lines[i].trim();
+    if(line === '') continue;
+
+    startTagName = get_start_tag_name(line);
+    endTagName = get_end_tag_name(line);
+
+    // Element complete in the same line and different than my selector
+    if(selector[selectorIndex] !== startTagName && is_element_complete(line)) continue;
+
+    // Element spans multiline and is different than my selector
+    // Loop until find corresponding end tag
+    if (selector[selectorIndex] !== startTagName) {
+       i += 1;
+       while (i<lines.length && !element_match(line, lines[i])) i+=1;
+       continue;
+    }
+
+    // lines[i] points to the opening tag of the current selector[selectorIndex]
+    selectorIndex += 1;
+    stack.push(startTagName);
+
+    // Target reached
+    if(selector.length === stack.length) {
+      // Target has only one line
+      if (is_element_complete(line)) {
+          line = line.replace(get_end_tag(line), '');
+          line = line.replace(get_start_tag(line), '');
+          return line;
+      }
+
+        // Extracts everything beetwen opening and closing tag and return.
+        i += 1;
+        while (i<lines.length && !element_match(line, lines[i]) ) {
+          content += lines[i].trim();
+          i += 1;
+        }
+        return content;
+    }
+    startTagName='';
+    endTagName='';
+  }
+  return '';
+}