scanoss 0.2.20 → 0.2.23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scanoss",
-  "version": "0.2.20",
+  "version": "0.2.23",
   "description": "The SCANOSS JS package provides a simple, easy to consume module for interacting with SCANOSS APIs/Engine.",
   "main": "build/main/index.js",
   "typings": "build/main/index.d.ts",

package/src/bin/cli-bin.ts

@@ -15,13 +15,13 @@ function CLIErrorHandler(e: Error) {
 
 async function main() {
   program
-    .version("0.2.20")
+    .version("0.2.23")
     .description('The SCANOSS JS package provides a simple, easy to consume module for interacting with SCANOSS APIs/Engine.')
 
   program
     .command('scan <source>')
     .description('Scan a folder/file')
-    //.option('-w, --wfp', 'Scan a .wfp file instead of a folder') // In progress
+    .option('-w, --wfp', 'Scan a .wfp file instead of a folder')
     .option('-c, --concurrency <number>', 'Number of concurrent connections to use while scanning (optional -default 10)')
     .option('--filter <path>', 'Loads an user defined filter (optional)')
     .option('-o, --output <filename>', 'Output result file name (optional - default stdout)')
@@ -40,7 +40,7 @@ async function main() {
 
     program
     .command('dep <source>')
-    .description('Scan for dependencies [ BETA ]')
+    .description('Scan for dependencies')
     .option('-o, --output <filename>', 'Output result file name (optional - default stdout)')
     .option('-a, --grpc-host <host>', 'SCANOSS GRPC HOST (optional - default: scanoss.com)')
     .option('-p, --grpc-port <port>', 'SCANOSS GRPC PORT  (optional - default: 443)')

package/src/commands/scan.ts

@@ -56,7 +56,7 @@ export async function scanHandler(rootPath: string, options: any): Promise<void>
     }
   } else {
     const winnowing = fs.readFileSync(rootPath, {encoding: 'utf-8'});
-    //filesCounter = [...winnowing.matchAll(/file=/g)].length;
+    scannerInput.fileList.length = [...winnowing.matchAll(/file=/g)].length;
   }
 
   if (!options.verbose) {
@@ -80,8 +80,8 @@ export async function scanHandler(rootPath: string, options: any): Promise<void>
       console.log(await fs.promises.readFile(resultPath, 'utf8'));
   });
 
-  if (options.wfp) await scanner.scanFromWinnowingFile(rootPath);
-  else await scanner.scan([scannerInput]);
+  if (options.wfp) scannerInput.wfpPath = rootPath;
+  await scanner.scan([scannerInput]);
 
 }
 

package/src/index.js

@@ -0,0 +1 @@
+alert("Hello world")

package/src/index.ts

@@ -1,5 +1,9 @@
 export * from './lib/scanner/ScannerTypes';
 export * from './lib/scanner/ScannerCfg'
 export * from './lib/scanner/Scanner'
+export * from './lib/dependencies/DependencyTypes';
+export * from './lib/dependencies/DependencyScannerCfg';
 export * from './lib/dependencies/DependencyScanner';
 
+
+

package/src/lib/dependencies/DependencyScanner.ts

@@ -19,11 +19,12 @@ export class DependencyScanner {
 
   public async scan(files: Array<string>): Promise<IDependencyResponse> {
     const localDependencies = await this.localDependency.search(files);
-    if (localDependencies.files.length === 0) return null;
+    if (localDependencies.files.length === 0) return {filesList: []};
     const request = this.buildRequest(localDependencies);
     const grpcResponse = await this.grpcDependencyService.get(request);
     const response = grpcResponse.toObject();
 
+
     // Extract scope from localDependencies and add it to response
     this.mergeScopeField(localDependencies, response);
     return response;
@@ -33,13 +34,14 @@ export class DependencyScanner {
   private buildRequest(localDependencies: ILocalDependencies): DependencyRequest {
     try {
       const depRequest = new DependencyRequest();
+      depRequest.setDepth(1);
       for (const file of localDependencies.files) {
         const fileMsg = new DependencyRequest.Files();
         fileMsg.setFile(file.file);
         for (const purl of file.purls) {
           const purlMsg = new DependencyRequest.Purls();
           purlMsg.setPurl(purl.purl);
-          purlMsg.setRequirement(purl?.requirements);
+          if(purl?.requirement) purlMsg.setRequirement(purl.requirement);
           fileMsg.addPurls(purlMsg);
         }
         depRequest.addFiles(fileMsg);

package/src/lib/dependencies/LocalDependency/DependencyTypes.ts

@@ -1,6 +1,6 @@
 export interface ILocalPurl {
   purl: string;
-  requirements?: string;
+  requirement?: string;
   scope?: string;
 }
 

package/src/lib/dependencies/LocalDependency/parsers/npmParser.ts

@@ -20,12 +20,12 @@ export function packageParser(fileContent: string, filePath: string): ILocalDepe
 
     for(const name of deps){
         const purlString = new PackageURL(PURL_TYPE, undefined, name, undefined, undefined, undefined).toString();
-        results.purls.push({purl: purlString, scope: "dependencies", requirements: o.dependencies[name]});
+        results.purls.push({purl: purlString, scope: "dependencies", requirement: o.dependencies[name]});
     }
 
     for(const name of devDeps){
       const purlString = new PackageURL(PURL_TYPE, undefined, name, undefined, undefined, undefined).toString();
-      results.purls.push({purl: purlString, scope: "devDependencies", requirements: o.devDependencies[name]});
+      results.purls.push({purl: purlString, scope: "devDependencies", requirement: o.devDependencies[name]});
     }
 
     return results;

package/src/lib/dependencies/LocalDependency/parsers/pyParser.ts

@@ -41,12 +41,15 @@ export function requirementsParser(fileContent: string, filePath: string): ILoca
             else {
 
                 const dep = parseDep(line);
-                if (dep.sym === '==') {
+                if (!dep.sym) {
+                  const purlString = new PackageURL(PURL_TYPE, undefined, dep.name, undefined, undefined, undefined).toString();
+                  results.purls.push({purl: purlString});
+                } else if (dep.sym === '==') {
                   const purlString = new PackageURL(PURL_TYPE, undefined, dep.name, dep.version, undefined, undefined).toString();
                   results.purls.push({purl: purlString});
                 } else {
                   const purlString = new PackageURL(PURL_TYPE, undefined, dep.name, undefined, undefined, undefined).toString();
-                  results.purls.push({purl: purlString, requirements: dep.sym+dep.version});
+                  results.purls.push({purl: purlString, requirement: dep.sym+dep.version});
                 }
             }
         }

package/src/lib/grpc/GrpcDependencyService.ts

@@ -7,8 +7,14 @@ export class GrpcDependencyService {
 
   private client: DependenciesClient;
 
-  constructor(endpoint: string, port: string) {
+  private metadata;
+
+  constructor(endpoint: string, port: string, apiKey='') {
     this.client = new DependenciesClient(endpoint + ':' + port, grpc.credentials.createSsl());
+    if(apiKey) {
+      this.metadata = new grpc.Metadata();
+      this.metadata.add('x-api-key', apiKey);
+    }
   }
 
   public async get(req: DependenciesMessages.DependencyRequest): Promise<DependenciesMessages.DependencyResponse> {
@@ -20,7 +26,7 @@ export class GrpcDependencyService {
     });
   }
 
-  public buildDependencyRequestMsg(plainObj: any): DependenciesMessages.DependencyRequest {
+  public buildDependencyRequestMsg(plainObj: DependenciesMessages.DependencyRequest.AsObject): DependenciesMessages.DependencyRequest {
     try {
       const depMessage = new DependenciesMessages.DependencyRequest();
       for (const dependency of plainObj.filesList) {
@@ -52,7 +58,7 @@ export class GrpcDependencyService {
   }
 
 
-  public buildEchoRequestMsg(plainObj: any): CommonMessages.EchoRequest {
+  public buildEchoRequestMsg(plainObj: CommonMessages.EchoRequest.AsObject): CommonMessages.EchoRequest {
     try {
       const echoMessage = new CommonMessages.EchoRequest();
       echoMessage.setMessage(plainObj.message);

package/src/lib/scanner/Dispatcher/DispatchableItem.ts

@@ -1,12 +1,11 @@
-import { WinnowerResponse } from "../Winnower/WinnowerResponse";
-
+import { FingerprintPacket } from "../WfpProvider/FingerprintPacket";
 export class DispatchableItem {
-  private winnowerResponse: WinnowerResponse;
+  private fingerprintPacket: FingerprintPacket;
 
   private errorCounter: number;
 
-  constructor(winnowerResponse) {
-    this.winnowerResponse = winnowerResponse;
+  constructor(fingerprintPacket: FingerprintPacket) {
+    this.fingerprintPacket = fingerprintPacket;
     this.errorCounter = 0;
   }
 
@@ -14,12 +13,12 @@ export class DispatchableItem {
     this.errorCounter += 1;
   }
 
-  public getWinnowerResponse(): WinnowerResponse {
-    return this.winnowerResponse;
+  public getWinnowerResponse(): FingerprintPacket {
+    return this.fingerprintPacket;
   }
 
   getContent() {
-    return this.winnowerResponse.getContent();
+    return this.fingerprintPacket.getContent();
   }
 
   getErrorCounter() {

package/src/lib/scanner/Scanner.ts

@@ -4,7 +4,7 @@ import EventEmitter from 'eventemitter3';
 import os from 'os';
 import fs from 'fs';
 
-import { Winnower } from './Winnower/Winnower';
+
 import { Dispatcher } from './Dispatcher/Dispatcher';
 
 import { DispatchableItem } from './Dispatcher/DispatchableItem';
@@ -13,8 +13,11 @@ import { ScannerCfg } from './ScannerCfg';
 import { ScannerEvents, ScannerInput } from './ScannerTypes';
 
 import sortPaths from 'sort-paths';
-import { WinnowerResponse } from './Winnower/WinnowerResponse';
 
+import { WfpProvider } from './WfpProvider/WfpProvider';
+import { FingerprintPacket } from './WfpProvider/FingerprintPacket';
+import { WfpCalculator } from './WfpProvider/WfpCalculator/WfpCalculator';
+import { WfpSplitter } from './WfpProvider/WfpSplitter/WfpSplitter';
 
 let finishPromiseResolve;
 let finishPromiseReject;
@@ -22,44 +25,44 @@ let finishPromiseReject;
 
 
 export class Scanner extends EventEmitter {
-  scannerCfg;
+  private scannerCfg: ScannerCfg;
 
-  workDirectory;
+  private workDirectory: string;
 
-  scanRoot;
+  private resultFilePath: string;
 
-  scannerId;
+  private wfpFilePath: string;
 
-  private winnower: Winnower;
+  private scanRoot: string;
 
-  private dispatcher: Dispatcher;
+  private scannerId: string;
 
-  resultFilePath;
+  private wfpProvider: WfpProvider;
 
-  wfpFilePath;
+  private dispatcher: Dispatcher;
 
-  scanFinished; // Both flags are used to prevent a race condition between DISPATCHER.NEW_DATA and DISPATCHER_FINISHED
+  private finishPromise: Promise<void>;
 
-  processingNewData; // Both flags are used to prevent a race condition between DISPATCHER.NEW_DATA and DISPATCHER_FINISHED
+  private scannerInput: Array<ScannerInput>;
 
-  responseBuffer;
+  private scanFinished: boolean; // Both flags are used to prevent a race condition between DISPATCHER.NEW_DATA and DISPATCHER_FINISHED
 
-  processedFiles;
+  private processingNewData: boolean; // Both flags are used to prevent a race condition between DISPATCHER.NEW_DATA and DISPATCHER_FINISHED
 
-  running;
+  private processedFiles: number;
 
-  filesToScan;
+  private running: boolean;
 
-  filesNotScanned;
+  private filesToScan;
 
-  finishPromise: Promise<void>;
+  private responseBuffer;
 
-  private scannerInput: Array<ScannerInput>;
+  private filesNotScanned;
 
   constructor(scannerCfg = new ScannerCfg()) {
     super();
     this.scannerCfg = scannerCfg;
-    this.scannerId = new Date().getTime();
+    this.scannerId = new Date().getTime().toString();
   }
 
   init() {
@@ -70,7 +73,7 @@ export class Scanner extends EventEmitter {
     this.responseBuffer = [];
     this.filesToScan = {};
     this.filesNotScanned = {};
-    this.winnower = new Winnower(this.scannerCfg);
+    this.wfpProvider = new WfpCalculator(this.scannerCfg);
     this.dispatcher = new Dispatcher(this.scannerCfg);
 
     this.finishPromise = new Promise((resolve, reject) =>{
@@ -85,19 +88,19 @@ export class Scanner extends EventEmitter {
   }
 
   setWinnowerListeners() {
-    this.winnower.on(ScannerEvents.WINNOWING_NEW_CONTENT, (winnowerResponse: WinnowerResponse) => {
-      this.emit(ScannerEvents.WINNOWING_NEW_CONTENT, winnowerResponse);
+    this.wfpProvider.on(ScannerEvents.WINNOWING_NEW_CONTENT, (fingerprintPacket: FingerprintPacket) => {
+      this.emit(ScannerEvents.WINNOWING_NEW_CONTENT, fingerprintPacket);
       this.reportLog(`[ SCANNER ]: New WFP content`);
-      winnowerResponse.setEngineFlags(this.scannerInput[0].engineFlags);
-      const disptItem = new DispatchableItem(winnowerResponse);
+      fingerprintPacket.setEngineFlags(this.scannerInput[0].engineFlags);
+      const disptItem = new DispatchableItem(fingerprintPacket);
       this.dispatcher.dispatchItem(disptItem);
     });
 
-    this.winnower.on(ScannerEvents.WINNOWER_LOG, (msg) => {
+    this.wfpProvider.on(ScannerEvents.WINNOWER_LOG, (msg) => {
       this.reportLog(msg);
     });
 
-    this.winnower.on(ScannerEvents.ERROR, (error) => {
+    this.wfpProvider.on(ScannerEvents.ERROR, (error) => {
       this.errorHandler(error, ScannerEvents.MODULE_WINNOWER);
     });
   }
@@ -105,12 +108,12 @@ export class Scanner extends EventEmitter {
   setDispatcherListeners() {
     this.dispatcher.on(ScannerEvents.DISPATCHER_QUEUE_SIZE_MAX_LIMIT, () => {
       this.reportLog(`[ SCANNER ]: Maximum queue size reached. Winnower will be paused`);
-      this.winnower.pause();
+      this.wfpProvider.pause();
     });
 
     this.dispatcher.on(ScannerEvents.DISPATCHER_QUEUE_SIZE_MIN_LIMIT, () => {
       this.reportLog(`[ SCANNER ]: Minimum queue size reached. Winnower will be resumed`);
-      this.winnower.resume();
+      this.wfpProvider.resume();
     });
 
     this.dispatcher.on(ScannerEvents.DISPATCHER_NEW_DATA, async (response) => {
@@ -125,7 +128,7 @@ export class Scanner extends EventEmitter {
     });
 
     this.dispatcher.on(ScannerEvents.DISPATCHER_FINISHED, async () => {
-      if (!this.winnower.hasPendingFiles()) {
+      if (!this.wfpProvider.hasPendingFiles()) {
         if (this.processingNewData) this.scanFinished = true;
         else await this.finishJob();
       }
@@ -136,7 +139,7 @@ export class Scanner extends EventEmitter {
       this.appendFilesToNotScanned(filesNotScanned);
     });
 
-    this.winnower.on(ScannerEvents.DISPATCHER_LOG, (msg) => {
+    this.wfpProvider.on(ScannerEvents.DISPATCHER_LOG, (msg) => {
       this.reportLog(msg);
     });
 
@@ -206,8 +209,18 @@ export class Scanner extends EventEmitter {
     this.scannerInput.shift();
     this.reportLog(`[ SCANNER ]: Job finished. ${this.scannerInput.length} pendings`);
 
-    if(this.scannerInput.length) this.winnower.startWinnowing(this.scannerInput[0]);
-    else await this.finishScan();
+    if(this.scannerInput.length) {
+      if (this.scannerInput[0].wfpPath) {
+        this.wfpProvider = new WfpSplitter();
+        this.setWinnowerListeners();
+        this.wfpProvider.start({wfpPath: this.scannerInput[0].wfpPath});
+      } else {
+        const folderRoot = this.scannerInput[0].folderRoot;
+        const winnowingMode = this.scannerInput[0].winnowingMode;
+        const fileList = this.scannerInput[0].fileList;
+        this.wfpProvider.start({folderRoot, winnowingMode, fileList});
+      }
+     } else await this.finishScan();
   }
 
   private async finishScan() {
@@ -264,7 +277,6 @@ export class Scanner extends EventEmitter {
   public scanFromWinnowingFile(wfpFilePath: string): Promise<void> {
     this.init();
     this.createOutputFiles();
-    this.winnower.startWinnowingFromFile(wfpFilePath);
     return this.finishPromise;
   }
 
@@ -274,16 +286,27 @@ export class Scanner extends EventEmitter {
     this.createOutputFiles();
     this.scannerInput = scannerInput;
 
-    if (!this.isValidInput(scannerInput)) {
-      this.finishScan();
-      return this.finishPromise;
+    // if (!this.isValidInput(scannerInput)) {
+    //   this.finishScan();
+    //   return this.finishPromise;
+    // }
+
+    if (scannerInput[0].wfpPath) {
+      this.wfpProvider = new WfpSplitter();
+      this.setWinnowerListeners();
+      this.wfpProvider.start({wfpPath: scannerInput[0].wfpPath});
+    } else {
+      const folderRoot = this.scannerInput[0].folderRoot;
+      const winnowingMode = this.scannerInput[0].winnowingMode;
+      const fileList = this.scannerInput[0].fileList;
+      this.wfpProvider.start({folderRoot, winnowingMode, fileList});
     }
-
-    this.winnower.startWinnowing(this.scannerInput[0]);
     return this.finishPromise;
   }
 
 
+
+
   private isValidInput(scannerInput: Array<ScannerInput>): boolean {
     if (!scannerInput) {
       this.reportLog('[ SCANNER ]: No input provided', 'warning');
@@ -313,25 +336,13 @@ export class Scanner extends EventEmitter {
     return this.scannerId;
   }
 
-  pause() {
-    this.running = false;
-    this.winnower.pause();
-    // this.dispatcher.pause();
-  }
-
-  resume() {
-    this.running = true;
-    this.winnower.resume();
-    // this.dispatcher.resume();
-  }
-
   stop() {
     this.reportLog(`[ SCANNER ]: Stopping scanner`);
     this.running = false;
-    this.winnower.removeAllListeners();
+    this.wfpProvider.removeAllListeners();
     this.dispatcher.removeAllListeners();
     this.dispatcher.stop();
-    this.winnower.stop();
+    this.wfpProvider.stop();
   }
 
   isRunning() {

package/src/lib/scanner/ScannerCfg.ts

@@ -23,7 +23,7 @@ export class ScannerCfg {
   // Persist results after [ X ] number of server responses.
   MAX_RESPONSES_IN_BUFFER = 300;
 
-  DISPATCHER_QUEUE_SIZE_MAX_LIMIT = 50;
+  DISPATCHER_QUEUE_SIZE_MAX_LIMIT = 300;
 
-  DISPATCHER_QUEUE_SIZE_MIN_LIMIT = 20;
+  DISPATCHER_QUEUE_SIZE_MIN_LIMIT = 200;
 };

package/src/lib/scanner/ScannerTypes.ts

@@ -38,4 +38,5 @@ export interface ScannerInput {
   folderRoot?: string;
   fileList: Array<string>;
   winnowingMode?: WinnowingMode;  // Enable winnowing algorithm, otherwise is scanned only MD5
+  wfpPath?: string;
 };

package/src/lib/scanner/{Winnower/WinnowerResponse.ts → WfpProvider/FingerprintPacket.ts}

@@ -1,17 +1,17 @@
-export class WinnowerResponse {
+export class FingerprintPacket {
   private wfpContent: string;
 
   private engineFlags: number;
 
   private scanRoot: string;
 
-  constructor(wfpContent, scanRoot = '') {
+  constructor(wfpContent: string, scanRoot = '') {
     this.wfpContent = wfpContent;
     this.scanRoot = scanRoot;
   }
 
-  public isEqual(winnowerResponse) {
-    return this.getContent() === winnowerResponse.getContent();
+  public isEqual(fingerprintPacket: FingerprintPacket): boolean {
+    return this.getContent() === fingerprintPacket.getContent();
   }
 
   public getContent() {
@@ -26,7 +26,7 @@ export class WinnowerResponse {
     return this.engineFlags;
   }
 
-  public getFilesWinnowed() {
+  public getFilesFingerprinted() {
     const files = [];
     const regExp = new RegExp(/,(\/.*)/g);
     let result;