scanoss 0.7.4 → 0.7.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scanoss",
-  "version": "0.7.4",
+  "version": "0.7.8",
   "description": "The SCANOSS JS package provides a simple, easy to consume module for interacting with SCANOSS APIs/Engine.",
   "main": "build/main/index.js",
   "typings": "build/main/index.d.ts",

package/src/cli/bin/cli-bin.ts

@@ -4,6 +4,12 @@ import { program } from 'commander';
 import { depHandler } from '../commands/dep';
 import { wfpHandler } from '../commands/wfp';
 import { scanHandler } from '../commands/scan';
+import path from 'path';
+import fs from 'fs';
+
+
+export const PackageJSONPath = path.join(__dirname,"../../../../package.json");
+export const PackageJSON = JSON.parse(fs.readFileSync(PackageJSONPath, 'utf-8'));
 
 function CLIErrorHandler(e: Error) {
   console.error(' ');
@@ -11,12 +17,9 @@ function CLIErrorHandler(e: Error) {
   process.exit(1);
 }
 
-
-
-
 async function main() {
   program
-    .version("0.7.4")
+    .version(PackageJSON.version)
     .description('The SCANOSS JS package provides a simple, easy to consume module for interacting with SCANOSS APIs/Engine.')
 
   program

package/src/sdk/Dependencies/LocalDependency/parsers/rubyParser.ts

@@ -1,6 +1,6 @@
 import path from "path";
 import { PackageURL } from "packageurl-js";
-import { ILocalDependency } from "../DependencyTypes";
+import { ILocalDependency, ILocalPurl } from '../DependencyTypes';
 
 
 const PURL_TYPE = 'gem';
@@ -49,8 +49,9 @@ export function gemfilelockParser(fileContent: string, filePath: string): Promis
     const gemlockParser = new GemfileLockParser();
     const purls = gemlockParser.getDependencies(fileContent);
     for (const purl of purls) {
-        results.purls.push({purl});
+        results.purls.push(purl);
     }
+
     return Promise.resolve(results);
 }
 
@@ -86,7 +87,7 @@ class GemfileLockParser {
 
     private current_gem;
 
-    private purlList;
+    private purlList: Array<ILocalPurl>;
 
     constructor () {
 
@@ -103,7 +104,7 @@ class GemfileLockParser {
         this.statesMap[SPECS] = this.parseSpec;
     }
 
-    public getDependencies(filecontent: string) {
+    public getDependencies(filecontent: string): Array<ILocalPurl> {
         this.resetState();
         let file = filecontent.split('\n');
         for (let line of file) {
@@ -153,12 +154,12 @@ class GemfileLockParser {
                 const match = line.match(specRegex);
 
                 const purl = new PackageURL( PURL_TYPE,
-                                undefined,
+                              null,
                                 match.groups.name,
-                                match.groups.version,
-                                undefined,
-                                undefined ).toString();
-                this.purlList.push(purl);
+                                  null,
+                                null,
+                                  null ).toString();
+                this.purlList.push({purl: purl, requirement: match.groups.version});
 
             } else {    // Second level of dependence
 

package/src/sdk/scanner/Dispatcher/Dispatcher.ts

@@ -4,7 +4,6 @@ import EventEmitter from 'eventemitter3';
 import fetch from 'node-fetch';
 import PQueue from "p-queue";
 
-
 import { ScannerEvents } from "../ScannerTypes";
 import { DispatcherResponse } from "./DispatcherResponse";
 import { ScannerCfg } from "../ScannerCfg";
@@ -13,6 +12,8 @@ import { DispatchableItem } from './DispatchableItem';
 import { HttpsProxyAgent } from 'https-proxy-agent';
 import * as syswideCa from "syswide-cas";
 
+import { PackageJSON } from '../../../cli/bin/cli-bin';
+
 const MAX_CONCURRENT_REQUEST = 30;
 
 export class Dispatcher extends EventEmitter {
@@ -139,7 +140,9 @@ export class Dispatcher extends EventEmitter {
         agent: this.proxy,
         method: 'post',
         body: item.getForm(),
-        headers: { 'User-Agent': this.scannerCfg.CLIENT_TIMESTAMP, 'X-Session': this.scannerCfg.API_KEY },
+        headers: {  'User-Agent': this.scannerCfg.CLIENT_TIMESTAMP ? this.scannerCfg.CLIENT_TIMESTAMP : `scanoss-js/v${PackageJSON.version}`,
+                    'X-Session': this.scannerCfg.API_KEY,
+        },
         signal: timeoutController.signal,
       });
 

package/src/sdk/scanner/Fingerprint.ts

@@ -14,11 +14,12 @@ export class Fingerprint extends EventEmitter {
   private finishPromise: Promise<void>;
   private finishPromiseResolve: (value?: (void | PromiseLike<void>)) => void;
   private finishPromiseReject: (value?: (void | PromiseLike<void>)) => void;
-  private totalFileCounter: number;
+  private reportStatusAfter: number;
   private queue: Array<IWfpProviderInput>;
-  constructor() {
+  constructor(reportStatusAfter = 10) {
     super();
 
+    this.reportStatusAfter = reportStatusAfter;
     this.finishPromise = new Promise((resolve, reject) =>{
       this.finishPromiseResolve = resolve;
       this.finishPromiseReject = reject;
@@ -34,7 +35,7 @@ export class Fingerprint extends EventEmitter {
     // TODO: Maybe the fingerprints shouldn't be truncated. Determine this optimal value.
     const cfg = new ScannerCfg();
     cfg.WFP_FILE_MAX_SIZE = 10 * 1024 *1024;
-
+    cfg.WINNOWING_REPORT_STATUS_AFTER_X = this.reportStatusAfter;
     this.wfpCalculator = new WfpCalculator(cfg);
 
 
@@ -43,8 +44,7 @@ export class Fingerprint extends EventEmitter {
     });
 
     this.wfpCalculator.on(ScannerEvents.WINNOWING_STATUS, (filesProcessed: number) => {
-      this.totalFileCounter += filesProcessed
-      this.emit(ScannerEvents.WINNOWING_STATUS, this.totalFileCounter);
+      this.emit(ScannerEvents.WINNOWING_STATUS, filesProcessed);
     });
 
 
@@ -58,11 +58,9 @@ export class Fingerprint extends EventEmitter {
     const pathFolder = path.dirname(this.fingerprintPath);
     if(!fs.existsSync(pathFolder)) fs.mkdirSync(pathFolder, { recursive: true });
 
-
-    this.totalFileCounter = 0;
     this.queue = jobs;
 
-    for (const job of this.queue) {
+    for await (const job of this.queue) {
       this.configureWfpCalculator();
       await this.wfpCalculator.start(job);
     }

package/src/sdk/scanner/Scanner.ts

@@ -100,6 +100,10 @@ export class Scanner extends EventEmitter {
       this.dispatcher.dispatchItem(item);
     });
 
+    this.wfpProvider.on(ScannerEvents.WINNOWING_STATUS, (newFilesProcessed)=>{
+      this.emit(ScannerEvents.WINNOWING_STATUS, newFilesProcessed)
+    });
+
     this.wfpProvider.on(ScannerEvents.WINNOWER_LOG, (msg) => {
       this.reportLog(msg);
     });

package/src/sdk/scanner/ScannerCfg.ts

@@ -1,6 +1,6 @@
 export class ScannerCfg {
-  // Client Timestamp
-  CLIENT_TIMESTAMP = "SCANNER JS"
+  // Client Timestamp, default value is scanoss-js/${version}
+  CLIENT_TIMESTAMP = ""
 
   // API URL
   API_URL = 'https://osskb.org/api/scan/direct';
@@ -23,6 +23,10 @@ export class ScannerCfg {
   // The maximum size for each .wfp file
   WFP_FILE_MAX_SIZE = 64 * 1000;
 
+  //After processing #WINNOWING_AFTER_X_REPORT_STATUS files,
+  // the winnowing algorithm will report a ScannerEvents.WINNOWING_STATUS event.
+  WINNOWING_REPORT_STATUS_AFTER_X = 10;
+
   MAX_RETRIES_FOR_RECOVERABLES_ERRORS = 5;
 
   ABORT_ON_MAX_RETRIES = true;
@@ -34,4 +38,6 @@ export class ScannerCfg {
 
   DISPATCHER_QUEUE_SIZE_MIN_LIMIT = 200;
 
+
+
 };

package/src/sdk/scanner/WfpProvider/WfpCalculator/WfpCalculator.ts

@@ -446,12 +446,17 @@ export class WfpCalculator extends WfpProvider {
   }
 
   async getNextScannableItem() {
-    if (this.fileListIndex >= this.fileList.length) return null;
+    if (this.fileListIndex >= this.fileList.length) {
+      this.emit(ScannerEvents.WINNOWING_STATUS, (this.fileListIndex) % this.scannerCfg.WINNOWING_REPORT_STATUS_AFTER_X)
+      return null;
+    }
     const path = this.fileList[this.fileListIndex];
     const contentSource = path.replace(`${this.folderRoot}`, '');
     const content = await fs.promises.readFile(path);
     this.fileListIndex += 1;
-    this.emit(ScannerEvents.WINNOWING_STATUS, this.fileListIndex);
+    if (!(this.fileListIndex % this.scannerCfg.WINNOWING_REPORT_STATUS_AFTER_X))
+      this.emit(ScannerEvents.WINNOWING_STATUS, this.scannerCfg.WINNOWING_REPORT_STATUS_AFTER_X);
+
     const scannable = new ScannableItem(content, contentSource, this.winnowingMode, this.scannerCfg.WFP_FILE_MAX_SIZE);
     return scannable;
   }