scanoss 0.2.8 → 0.2.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scanoss",
-  "version": "0.2.8",
+  "version": "0.2.12",
   "description": "The SCANOSS JS package provides a simple, easy to consume module for interacting with SCANOSS APIs/Engine.",
   "main": "build/main/index.js",
   "typings": "build/main/index.d.ts",
@@ -53,6 +53,7 @@
     "isbinaryfile": "^4.0.8",
     "node-fetch": "2.6.1",
     "p-queue": "6.6.2",
+    "packageurl-js": "^0.0.5",
     "sort-paths": "^1.1.1"
   },
   "devDependencies": {

package/src/bin/cli-bin.ts

@@ -1,18 +1,21 @@
 #!/usr/bin/env node
 
 import { program } from 'commander';
+import { depHandler } from '../commands/dep';
 import { scanHandler } from '../commands/scan';
 
-
 function CLIErrorHandler(e: Error) {
   console.error(' ');
   console.error(e);
   process.exit(1);
 }
 
+
+
+
 async function main() {
   program
-    .version('0.2.0')
+    .version("0.2.12")
     .description('The SCANOSS JS package provides a simple, easy to consume module for interacting with SCANOSS APIs/Engine.')
 
   program
@@ -20,8 +23,9 @@ async function main() {
     .description('Scan a folder/file')
     //.option('-w, --wfp', 'Scan a .wfp file instead of a folder') // In progress
     .option('-c, --concurrency <number>', 'Number of concurrent connections to use while scanning (optional -default 10)')
-    .option('-f, --filter <path>', 'Loads an user defined filter (optional)')
+    .option('--filter <path>', 'Loads an user defined filter (optional)')
     .option('-o, --output <filename>', 'Output result file name (optional - default stdout)')
+    .option('-F, --flags <flags>', 'Scanning engine flags (1: disable snippet matching, 2 enable snippet ids, 4: disable dependencies, 8: disable licenses, 16: disable copyrights,32: disable vulnerabilities, 64: disable quality, 128: disable cryptography,256: disable best match, 512: Report identified files)')
     .option('-P, --post-size <postsize>', 'Number of kilobytes to limit the post to while scanning (optional - default 64)')
     .option('-R, --max-retry <retry>', 'Max number of retries for each POST (optional -default 5)')
     .option('-M, --timeout <timeout>', 'Timeout (in seconds) for API communication (optional -default 120)')
@@ -34,10 +38,13 @@ async function main() {
     $ scanoss-js scan -o scan-output.json <source-folder>`
     );
 
+    program
+    .command('dep <source>')
+    .description('Scan for dependencies [ BETA ]')
+    .option('-o, --output <filename>', 'Output result file name (optional - default stdout)')
+    .action((source, options) => {depHandler(source, options).catch((e) => {CLIErrorHandler(e)})})
 
     await program.parseAsync(process.argv);
-
-
 }
 
 

package/src/commands/dep.ts

@@ -0,0 +1,23 @@
+import fs from "fs";
+import { Dependency } from "..";
+import { Tree } from "../lib/tree/Tree";
+
+export async function depHandler(rootPath: string, options: any): Promise<void> {
+  rootPath = rootPath.replace(/\/$/, '');  // Remove trailing slash if exists
+  rootPath = rootPath.replace(/^\./, process.env.PWD);  // Convert relative path to absolute path.
+
+  const tree = new Tree(rootPath);
+  tree.buildTree();
+
+  const fileList = tree.getRootFolder().getFiles().map((path) => {return rootPath+path});
+
+  const dependency = new Dependency();
+  const results = await dependency.scan(fileList);
+
+  if(options.output) {
+    fs.promises.writeFile(options.output, JSON.stringify(results, null, 2));
+  } else {
+    console.log(JSON.stringify(results, null, 2));
+  }
+
+}

package/src/commands/scan.ts

@@ -1,32 +1,15 @@
 import { Scanner } from '../lib/scanner/Scanner';
-import { ScannerEvents } from '../lib/scanner/ScannerEvents';
+import { ScannerEvents, ScannerInput } from '../lib/scanner/ScannerTypes';
 import { ScannerCfg } from '../lib/scanner/ScannerCfg';
 import { Tree } from '../lib/tree/Tree';
-import File from '../lib/tree/File';
 
 import cliProgress from 'cli-progress';
-import { WinnowerResponse } from '../lib/scanner/Winnower/WinnowerResponse';
 import { DispatcherResponse } from '../lib/scanner/Dispatcher/DispatcherResponse';
 import { defaultFilter } from '../lib/filters/defaultFilter';
 import { FilterList } from '../lib/filters/filtering';
 
 import fs from 'fs';
 
-enum FilterTypes {
-  BANNED = 'BANNED',
-  WHITELIST = 'WHITELIST',
-  FULL_SCAN = 'FULL_SCAN',
-  QUICK_SCAN = 'QUICK_SCAN',
-}
-
-
-interface ScannerInput {
-  EngineFlags: string;
-  FolderRoot?: string;
-  FileList: Array<string>;
-  Snippets: boolean;
-};
-
 // Async function that verify if a path is a folder. If the path is not valid the promise will be rejected
 const isFolder = (path: string): Promise<boolean> => {
   return new Promise((resolve, reject) => {
@@ -40,35 +23,29 @@ const isFolder = (path: string): Promise<boolean> => {
   });
 }
 
-async function printJSON(path: string) {
-  const jsonTxt = await fs.promises.readFile(path, 'utf8');
-  console.log(jsonTxt);
-}
-
-
-
 export async function scanHandler(rootPath: string, options: any): Promise<void> {
 
-  let fileList = [];
-  let scannerInput = {};
-  let filesCounter = 0; // Used by the progress bar
+  let scannerInput: ScannerInput = {fileList: []};
+
+  rootPath = rootPath.replace(/\/$/, '');  // Remove trailing slash if exists
+  rootPath = rootPath.replace(/^\./, process.env.PWD);  // Convert relative path to absolute path.
   const pathIsFolder = await isFolder(rootPath);
 
   // Create scanner and set connections parameters
   const scannerCfg = new ScannerCfg();
   if(options.concurrency) scannerCfg.CONCURRENCY_LIMIT = parseInt(options.concurrency);
   if(options.postSize) scannerCfg.WFP_FILE_MAX_SIZE = parseInt(options.postSize) * 1024;
-  if(options.apiUrl) scannerCfg.API_URL = options.apiurl;
+  if(options.apiurl) scannerCfg.API_URL = options.apiurl;
   if(options.key) scannerCfg.API_KEY = options.key;
   if(options.timeout) scannerCfg.TIMEOUT = options.timeout * 1000;
   if(options.maxRetry) scannerCfg.MAX_RETRIES_FOR_RECOVERABLES_ERRORS = options.maxRetry;
-
   const scanner = new Scanner(scannerCfg);
 
-  if(!options.wfp) {  // --wfp change the behaviour of the scanner. Scan a WFP File instead of a folder
-    if(pathIsFolder) { // If a folder is specified, we scan the folder and all its subfolders
-      if(rootPath.charAt(rootPath.length -1) !== '/') rootPath += '/';
+  scannerInput.folderRoot = rootPath + '/'; // This will remove the project root path from the results.
+  if(options.flags) scannerInput.engineFlags = options.flags;
 
+  if(!options.wfp) {
+    if(pathIsFolder) {
       const tree = new Tree(rootPath);
       const filter = new FilterList('');
 
@@ -79,31 +56,22 @@ export async function scanHandler(rootPath: string, options: any): Promise<void>
         console.error('Loading default filters...');
         filter.load(defaultFilter as FilterList);
       }
-
       console.error('Reading directory...  ');
       tree.loadFilter(filter);
       tree.buildTree();
-
-      fileList = tree.getFileList();
-      filesCounter = fileList.length;
-
-      // Converts to scanner input format
-      // This is temporary, until the scanner is ready to receive a ScannerInput object
-      for(let f of fileList) scannerInput = Object.assign(scannerInput, {[f]: 'FULL_SCAN'});
+      scannerInput.fileList = tree.getFileList();
     } else {
-      scannerInput = {[rootPath]: 'FULL_SCAN'};
-      filesCounter = 1;
+      scannerInput.fileList = [rootPath];
     }
   } else {
     const winnowing = fs.readFileSync(rootPath, {encoding: 'utf-8'});
-    const reg = /file=/g;
-    filesCounter = [...winnowing.matchAll(reg)].length;
+    //filesCounter = [...winnowing.matchAll(/file=/g)].length;
   }
 
   if (!options.verbose) {
     const optBar1 = { format: 'Scan Progress: [{bar}] {percentage}% | Scanned {value} files of {total}' };
     const bar1 = new cliProgress.SingleBar(optBar1, cliProgress.Presets.shades_classic);
-    bar1.start(filesCounter, 0);
+    bar1.start(scannerInput.fileList.length, 0);
 
     scanner.on(ScannerEvents.DISPATCHER_NEW_DATA, (dispResp: DispatcherResponse) => {
       bar1.increment(dispResp.getFilesScanned().length);
@@ -122,7 +90,7 @@ export async function scanHandler(rootPath: string, options: any): Promise<void>
   });
 
   if (options.wfp) await scanner.scanFromWinnowingFile(rootPath);
-  else await scanner.scanList(scannerInput);
+  else await scanner.scan([scannerInput]);
 
 }
 

package/src/index.ts

@@ -1,4 +1,5 @@
-export * from './lib/scanner/ScannerEvents';
+export * from './lib/scanner/ScannerTypes';
 export * from './lib/scanner/ScannerCfg'
 export * from './lib/scanner/Scanner'
-
+export * from './lib/dependencies/DependencyTypes';
+export * from './lib/dependencies/Dependency';

package/src/lib/dependencies/Dependency.ts

@@ -0,0 +1,58 @@
+import os from "os";
+import fs from "fs";
+
+import { IDependencyResponse, IFile, IDependency } from "./DependencyTypes";
+import { FileListDependency } from "./parsers/types";
+import { generateDependenciesPurls } from "./PurlGenerator";
+
+export class Dependency {
+
+  private workDirectory: string;
+
+  private resultFilePath: string;
+
+  constructor () {
+    this.setWorkDirectory(`${os.tmpdir()}/depscanner-${new Date().getTime()}`);
+  }
+
+  public async scan (fileList: Array<string>): Promise<IDependencyResponse> {
+    const toGrpc = await generateDependenciesPurls(fileList);
+    // Here we should call to the grpc server
+    return this.adapterToDependencyResponse(toGrpc);
+  }
+
+  public setWorkDirectory(workDirectory: string) {
+    this.workDirectory = workDirectory;
+    this.resultFilePath = `${this.workDirectory}/dependencies.json`;
+
+    if (!fs.existsSync(this.workDirectory)) fs.mkdirSync(this.workDirectory);
+  }
+
+  private adapterToDependencyResponse (dependencies: FileListDependency): IDependencyResponse {
+    const results = <IDependencyResponse>{files: []};
+
+    const depArr: Array<IDependency> = [];
+    let fileArr = <IFile>{};
+
+    for (const dependency of dependencies.files){
+      for (const purl of dependency.purls) {
+        depArr.push({
+          component: null,
+          purl: purl.purl,
+          version: null,
+          licenses: [{name: null}]
+        });
+      }
+
+      fileArr = ({
+        file: dependency.file,
+        id: "dependency",
+        status: "pending",
+        dependencies: depArr
+      });
+
+      results.files.push(fileArr);
+    }
+    return results
+  }
+}

package/src/lib/dependencies/DependencyTypes.ts

@@ -0,0 +1,22 @@
+
+interface ILicense {
+  name: string;
+}
+
+export interface IDependency {
+  component: string;
+  purl: string;
+  version: string;
+  licenses: Array<ILicense>;
+}
+
+export interface IFile {
+  file: string;
+  id: string;
+  status: string;
+  dependencies: Array<IDependency>;
+}
+
+export interface IDependencyResponse {
+  files: Array<IFile>;
+}

package/src/lib/dependencies/PurlGenerator.ts

@@ -0,0 +1,41 @@
+import fs from 'fs';
+import path from 'path';
+import { FileListDependency, ParserDefinitions } from './parsers/types';
+
+
+import { pomParser } from './parsers/mavenParser';
+import { packagelockParser, packageParser } from './parsers/npmParser';
+import { requirementsParser } from './parsers/pyParser';
+import { gemfilelockParser, gemfileParser } from './parsers/rubyParser';
+
+
+
+/*
+ This is a hash map that connect a filename with it's own parser function
+ Any parser function must return a FileDependency object (See type.ts file in src/parser
+*/
+const Parser: ParserDefinitions = {
+    'requirements.txt': requirementsParser,
+    'pom.xml': pomParser,
+    'package.json': packageParser,
+    'package-lock.json': packagelockParser,
+    'Gemfile': gemfileParser,
+    'Gemfile.lock': gemfilelockParser
+};
+
+
+export async function generateDependenciesPurls(files: Array<string>): Promise<FileListDependency> {
+    let results: FileListDependency = {files: []};
+
+    for (const filePath of files) {
+        const fileName = path.basename(filePath);
+        if(Parser[fileName] != null) {
+            const fileContent = await fs.promises.readFile(filePath, 'utf8');
+            const dependency = Parser[fileName](fileContent, filePath);
+            if(dependency.purls.length != 0)
+                results.files.push(dependency);
+        }
+    }
+    return results;
+}
+

package/src/lib/dependencies/parsers/golangParser.ts

@@ -0,0 +1,4 @@
+
+
+// Reference https://go.dev/ref/mod#go-mod-file
+// 

package/src/lib/dependencies/parsers/mavenParser.ts

@@ -0,0 +1,48 @@
+import path from "path";
+import { PackageURL } from "packageurl-js";
+import { FileDependency } from "./types";
+import { isValidPath, isValidUrl } from './utils';
+
+const PURL_TYPE = 'maven';
+
+
+// Parse a pom.txt file from maven manifest file
+// See reference on: https://maven.apache.org/guides/introduction/introduction-to-the-pom.html
+// and https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html
+const MANIFEST_FILE = 'pom.xml';
+export function pomParser(fileContent: string, filePath: string): FileDependency {
+    
+    // If the file is not a python manifest file, return an empty results
+    const results: FileDependency = {file: filePath, purls: []};
+    if(path.basename(filePath) != MANIFEST_FILE)
+        return results;
+
+    const dependencies = fileContent.match(/<dependency>((?:.|\n)*?)<\/dependency>/gm);
+    if(dependencies) {
+        // TODO: classifier are not supported yet
+        dependencies.forEach(dependency => {
+            // Extract groupId. It's the purl namespace
+            const groupId = dependency.match(/<groupId>([^<]*)<\/groupId>/);
+            const namespace = groupId ? groupId[1] : '';
+
+            // Extract artifact id. It's the purl name
+            const artifactId = dependency.match(/<artifactId>([^<]*)<\/artifactId>/);
+            const name = artifactId ? artifactId[1] : '';
+
+            const versionReg = dependency.match(/<version>([^<]*)<\/version>/);
+            const version = versionReg ? versionReg[1] : '';
+
+            
+            let purlQualifiers;
+            const type = dependency.match(/<type>([^<]*)<\/type>/);
+            if(type) {
+                purlQualifiers = {};
+                purlQualifiers['type'] = type[1]
+            }
+            
+            const purlString = new PackageURL(PURL_TYPE, namespace, name, version, purlQualifiers, undefined).toString();
+            results.purls.push({purl: purlString});
+        });
+    }
+    return results;
+}

package/src/lib/dependencies/parsers/npmParser.ts

@@ -0,0 +1,44 @@
+import path from "path";
+import { PackageURL } from "packageurl-js";
+import { FileDependency } from "./types";
+import { isValidPath, isValidUrl } from './utils';
+
+const PURL_TYPE = 'npm';
+
+
+// Parse a package.json file from node projects
+// See reference on: https://docs.npmjs.com/cli/v8/configuring-npm/package-json
+const MANIFEST_FILE = 'package.json';
+export function packageParser(fileContent: string, filePath: string): FileDependency {
+    // If the file is not manifest file, return an empty results
+    const results: FileDependency = {file: filePath, purls: []};
+    if(path.basename(filePath) != MANIFEST_FILE)
+        return results;
+    const o = JSON.parse(fileContent);
+    let devDeps = Object.keys(o.devDependencies || {});
+    let deps = Object.keys(o.dependencies || {});
+    let listDeps = [...deps, ...devDeps];
+    for(const name of listDeps){
+        const purlString = new PackageURL(PURL_TYPE, undefined, name, undefined, undefined, undefined).toString();
+        results.purls.push({purl: purlString});
+    }
+    return results;
+}
+
+
+// Parse a package-lock.json file from node projects
+// See reference on: https://docs.npmjs.com/cli/v8/configuring-npm/package-json
+const MANIFEST_FILE_1 = 'package-loc.json';
+export function packagelockParser(fileContent: string, filePath: string): FileDependency {
+
+    const results: FileDependency = {file: filePath, purls: []};
+    if(path.basename(filePath) != MANIFEST_FILE_1)
+        return results;
+
+    const o = JSON.parse(fileContent).dependencies;
+    for (const [key, value] of Object.entries(o)) {
+        if(!key) continue;
+        let purl = new PackageURL(PURL_TYPE, undefined, key,value['version'], undefined, undefined).toString();
+        results.purls.push({purl});
+    }
+}

package/src/lib/dependencies/parsers/pyParser.ts

@@ -0,0 +1,46 @@
+import path from "path";
+import { PackageURL } from "packageurl-js";
+import { FileDependency } from "./types";
+import { isValidPath, isValidUrl } from './utils';
+
+const PURL_TYPE = 'pypi';
+
+// Parse a requirements.txt file from python projects
+// See reference on: https://pip.pypa.io/en/stable/reference/requirements-file-format/
+const MANIFEST_FILE = 'requirements.txt';
+export function requirementsParser(fileContent: string, filePath: string): FileDependency {
+    
+    // If the file is not a python manifest file, return an empty results
+    const results: FileDependency = {file: filePath, purls: []};
+    if(path.basename(filePath) != MANIFEST_FILE)
+        return results;
+
+
+    const lines: Array<string> = fileContent.split('\n');
+    let compName: string = '';
+    let compVer: string = '';
+
+    for (let line of lines) {
+        line = line.trim();
+        if (!line.startsWith('#') && line.length>0) { // Avoid comments and new lines
+            if(isValidUrl(line)) {
+                // For reference about the regex see https://www.rfc-editor.org/rfc/rfc3986#appendix-B
+                const res = line.match(/^(([^:\/?#]+):)?(\/\/([^\/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?/);
+                continue;
+            } 
+            else if(isValidPath(line)) {continue;}   
+            else if(line.startsWith('-r')) {continue;} // recursive dependencies (NOT SUPPORTED YET)
+            else {
+                // Line contains a package name and/or version.
+                const res = line.match(/^([-\w]+)\s*(?:[>=~!]*)\s*([\d\.]*)/);    //Extract name and version
+                if (res) {
+                    compName = res.length > 1 ? res[1] : ' ';
+                    compVer = undefined;
+                }              
+            }
+            const purlString = new PackageURL(PURL_TYPE, undefined, compName, compVer, undefined, undefined).toString();
+            results.purls.push({purl: purlString});
+        }
+    }
+    return results;
+}

package/src/lib/dependencies/parsers/rubyParser.ts

@@ -0,0 +1,181 @@
+import path from "path";
+import { PackageURL } from "packageurl-js";
+import { FileDependency } from "./types";
+import { isValidPath, isValidUrl } from './utils';
+import { stringify } from "querystring";
+
+const PURL_TYPE = 'gem';
+
+
+// Parse a gemfile file from ruby
+// See reference on: https://bundler.io/gemfile.html
+// and https://bundler.io/man/gemfile.5.html
+const MANIFEST_FILE = 'Gemfile';
+export function gemfileParser(fileContent: string, filePath: string): FileDependency {
+    
+    // If the file is not a manifest file, return an empty results
+    const results: FileDependency = {file: filePath, purls: []};
+    if(path.basename(filePath) != MANIFEST_FILE)
+        return results;
+
+
+    const lines: Array<string> = fileContent.split('\n');
+    let compName: string = '';
+
+    for (let line of lines) {
+        line = line.trim();
+        if (!line.startsWith('#') && line.length>0) { // Avoid comments and empty lines
+            // Line contains a package name and/or version.
+            const res = line.match(/^gem\s*(["']\w+["'])/);    //Extract name
+            if (res) {
+                compName = res.length > 1 ? res[1] : ' ';
+                compName = compName.replace(/['"]/g, '');
+                const purlString = new PackageURL(PURL_TYPE, undefined, compName, undefined, undefined, undefined).toString();
+                results.purls.push({purl: purlString});
+            }              
+        }
+    }
+    return results;
+}
+
+
+const MANIFEST_FILE_1 = 'Gemfile.lock';
+export function gemfilelockParser(fileContent: string, filePath: string): FileDependency {
+    
+    // If the file is not a manifest file, return an empty results
+    const results: FileDependency = {file: filePath, purls: []};
+    if(path.basename(filePath) != MANIFEST_FILE_1)
+        return results;
+
+    const gemlockParser = new GemfileLockParser();
+    const purls = gemlockParser.getDependencies(fileContent);
+    for (const purl of purls) {
+        results.purls.push({purl});
+    }
+    return results;
+}
+
+
+// Section headings: these are also used as switches to track a parsing state
+
+const PATH = 'PATH';
+const GIT = 'GIT';
+const SVN = 'SVN';
+const GEM = 'GEM';
+const PLATFORMS = 'PLATFORMS';
+const DEPENDENCIES = 'DEPENDENCIES';
+const SPECS = '  specs:';
+
+// types of Gems, which is really where they are provisioned from
+// RubyGems repo, local path or VCS
+const GEM_TYPES = [GEM, PATH, GIT, SVN];
+
+
+const specRegex = /(?<name>[^ \)\(,!:]+)?(?: \((?<version>[^-]*?)(?:-(?<platform>[^!]*))?\))?/;
+const firstDepLevelRegex = /^ {4}(?! )/;
+/*
+    The parsing use a simple state machine, switching states based on sections
+    headings. The result is a list of purls
+*/
+class GemfileLockParser {
+
+    private statesMap;
+    
+    private state;  
+    
+    private current_options: Record<string, string>; 
+    
+    private current_gem; 
+    
+    private purlList;
+
+    constructor () {
+
+        this.purlList = [];
+
+        // map of a line start string to the next parsing state function
+        this.statesMap = {};
+        this.statesMap[DEPENDENCIES] = this.parseDependency;
+        this.statesMap[PLATFORMS] = this.parsePlatform;
+        this.statesMap[GIT] = this.parseOptions;
+        this.statesMap[PATH] = this.parseOptions;
+        this.statesMap[SVN] = this.parseOptions;
+        this.statesMap[GEM] = this.parseOptions;
+        this.statesMap[SPECS] = this.parseSpec;
+    }
+
+    public getDependencies(filecontent: string) {
+        this.resetState();
+        let file = filecontent.split('\n');
+        for (let line of file) {
+            line = line.trimEnd();
+
+            //reset state on empty lines
+            if (!line.length) {
+                this.resetState();
+                continue;
+            }
+
+            //switch to new state
+            if (line in this.statesMap) {
+                if (GEM_TYPES.includes(line))
+                    this.current_gem = line;
+                this.state=this.statesMap[line];
+                continue;
+            }
+
+            // process the line
+            if (this.state) this.state(line);
+        }
+        this.refine();
+        return this.purlList;
+    }
+
+    private resetState() {
+        this.current_options = {};
+        this.state = null;
+    }
+
+    private parseOptions(line: string) {
+        const match = line.match(/\s*(\w+):\s*(.*)/);
+        const key = match.length>=1 ? match[1] : null;
+        const value = match.length>=2 ? match[2] : null;
+        if(key) this.current_options[key] = value;
+    }   
+
+    private parseDependency(line: string) {}
+    private parsePlatform(line: string){}
+    
+
+    private parseSpec(line: string) {
+        
+        if(this.current_gem == GEM) {
+            if(firstDepLevelRegex.test(line)) { 
+                line = line.trimStart();
+                const match = line.match(specRegex);
+
+                const purl = new PackageURL( PURL_TYPE,
+                                undefined, 
+                                match.groups.name,
+                                match.groups.version,
+                                undefined,
+                                undefined ).toString();
+                this.purlList.push(purl);
+
+            } else {    // Second level of dependence
+
+            }
+        }
+
+        if(this.current_gem == GIT) {}
+
+        // Purl from local dependencies are not generated
+        if(this.current_gem == PATH){}
+
+    }
+
+    private refine() {}
+
+
+
+}