scanoss 0.2.8 → 0.2.12

package/src/lib/dependencies/parsers/types.ts

@@ -0,0 +1,50 @@
+export interface Purl {
+    purl: string;
+}
+
+export interface FileDependency {
+  file: string;
+  purls: Array<Purl>;
+}
+
+export interface FileListDependency {
+  files: Array<FileDependency>;
+}
+
+type ParserFuncType = (fileContent: string, filePath: string) => FileDependency;
+
+export interface ParserDefinitions {
+    [key: string]: ParserFuncType;
+}
+  
+
+/*
+    EXAMPLE
+{
+  "files": [
+    {
+      "file": "./test/data/requirements.txt",
+      "purls": [
+        {
+          "purl": "pkg:pypi/requests"
+        },
+        {
+          "purl": "pkg:pypi/crc32c@2.2"
+        },
+        {
+          "purl": "pkg:pypi/binaryornot"
+        },
+        {
+          "purl": "pkg:pypi/progress"
+        },
+        {
+          "purl": "pkg:pypi/grpcio"
+        },
+        {
+          "purl": "pkg:pypi/protobuf"
+        }
+      ]
+    }
+  ]
+}
+*/

package/src/lib/dependencies/parsers/utils.ts

@@ -0,0 +1,16 @@
+import path from "path";
+import { URL } from "url";
+
+export function isValidUrl(string: string): boolean {
+    let url: URL;
+    try {
+      new URL(string);
+    } catch (_) {
+      return false;
+    }
+    return true;
+  }
+
+export function isValidPath(string: string): boolean {
+  return /^((?:\.\.?)|(?:[a-zA-Z]:\\)|(?:\/))/gm.test(string);
+}

package/src/lib/scanner/Dispatcher/DispatchableItem.ts

@@ -14,7 +14,7 @@ export class DispatchableItem {
     this.errorCounter += 1;
   }
 
-  getWinnowerResponse() {
+  public getWinnowerResponse(): WinnowerResponse {
     return this.winnowerResponse;
   }
 

package/src/lib/scanner/Dispatcher/Dispatcher.ts

@@ -5,10 +5,11 @@ import fetch from 'node-fetch';
 import PQueue from "p-queue";
 import FormData from "form-data";
 
-import { ScannerEvents } from "../ScannerEvents";
+import { ScannerEvents } from "../ScannerTypes";
 import { DispatcherResponse } from "./DispatcherResponse";
 import { ScannerCfg } from "../ScannerCfg";
 import { GlobalControllerAborter } from "./GlobalControllerAborter";
+import { DispatchableItem } from './DispatchableItem';
 
 export class Dispatcher extends EventEmitter {
   private scannerCfg: ScannerCfg;
@@ -63,7 +64,7 @@ export class Dispatcher extends EventEmitter {
     this.globalAbortController.abortAll();
   }
 
-  dispatchItem(disptItem) {
+  public dispatchItem(disptItem: DispatchableItem): void {
     this.pQueue.add(() => this.dispatch(disptItem));
 
     if (
@@ -104,13 +105,17 @@ export class Dispatcher extends EventEmitter {
     }
   }
 
-  async dispatch(disptItem) {
+  async dispatch(disptItem: DispatchableItem) {
     const timeoutController = this.globalAbortController.getAbortController();
     const timeoutId = setTimeout(() => timeoutController.abort(), this.scannerCfg.TIMEOUT);
     try {
       const form = new FormData();
-      const wfpContent = disptItem.getContent();
-      form.append('filename', Buffer.from(wfpContent), 'data.wfp');
+
+      form.append('filename', Buffer.from(disptItem.getContent()), 'data.wfp');
+
+      const engineFlag = disptItem.getWinnowerResponse().getEngineFlags();
+      if(engineFlag) form.append('flags', engineFlag);
+
       this.emit(ScannerEvents.DISPATCHER_WFP_SENDED);
       const response = await fetch(this.scannerCfg.API_URL, {
         method: 'post',
@@ -141,7 +146,6 @@ export class Dispatcher extends EventEmitter {
         this.globalAbortController.removeAbortController(timeoutController);
         this.errorHandler(e, disptItem);
         return Promise.resolve();
-
     }
   }
 }

package/src/lib/scanner/Scannable/ScannableItem.ts

@@ -1,18 +1,21 @@
+import { WinnowingMode } from "../ScannerTypes";
+
+
 export class ScannableItem {
   private contentSource: string;
 
   private content: Buffer;
 
-  private scanMode: any;
+  private winnowingMode: WinnowingMode;
 
   private fingerprint: any;
 
   private maxSizeWfp: any;
 
-  constructor(content: Buffer, contentSource: string, scanMode: any, maxSizeWfp: number) {
+  constructor(content: Buffer, contentSource: string, winnowingMode: any, maxSizeWfp: number) {
     this.contentSource = contentSource;
     this.content = content;
-    this.scanMode = scanMode;
+    this.winnowingMode = winnowingMode;
     this.maxSizeWfp = maxSizeWfp;
   }
 
@@ -24,14 +27,14 @@ export class ScannableItem {
     return this.contentSource;
   }
 
-  public getScanMode() {
-    return this.scanMode;
-  }
-
   public getFingerprint() {
     return this.fingerprint;
   }
 
+  public getWinnowingMode(): WinnowingMode {
+    return this.winnowingMode;
+  }
+
   public getMaxSizeWfp() {
     return this.maxSizeWfp;
   }

package/src/lib/scanner/Scanner.ts

@@ -10,17 +10,13 @@ import { Dispatcher } from './Dispatcher/Dispatcher';
 import { DispatchableItem } from './Dispatcher/DispatchableItem';
 import { DispatcherResponse } from './Dispatcher/DispatcherResponse';
 import { ScannerCfg } from './ScannerCfg';
-import { ScannerEvents } from './ScannerEvents';
+import { ScannerEvents, ScannerInput } from './ScannerTypes';
 
 
 import sortPaths from 'sort-paths';
 import { WinnowerResponse } from './Winnower/WinnowerResponse';
 
 
-// TO DO:
-// - Split ScannerEvents into ExternalEvents and InternalEvents
-// - Implement a static atribute to keep track of the scannerId
-
 let finishPromiseResolve;
 let finishPromiseReject;
 
@@ -33,9 +29,9 @@ export class Scanner extends EventEmitter {
 
   scannerId;
 
-  winnower: Winnower;
+  private winnower: Winnower;
 
-  dispatcher;
+  private dispatcher: Dispatcher;
 
   resultFilePath;
 
@@ -57,6 +53,8 @@ export class Scanner extends EventEmitter {
 
   finishPromise: Promise<void>;
 
+  private scannerInput: Array<ScannerInput>;
+
   constructor(scannerCfg = new ScannerCfg()) {
     super();
     this.scannerCfg = scannerCfg;
@@ -89,6 +87,7 @@ export class Scanner extends EventEmitter {
     this.winnower.on(ScannerEvents.WINNOWING_NEW_CONTENT, (winnowerResponse: WinnowerResponse) => {
       this.emit(ScannerEvents.WINNOWING_NEW_CONTENT, winnowerResponse);
       this.reportLog(`[ SCANNER ]: New WFP content`);
+      winnowerResponse.setEngineFlags(this.scannerInput[0].engineFlags);
       const disptItem = new DispatchableItem(winnowerResponse);
       this.dispatcher.dispatchItem(disptItem);
     });
@@ -121,13 +120,13 @@ export class Scanner extends EventEmitter {
       this.insertIntoBuffer(response);
       if (this.bufferReachedLimit()) this.bufferToFiles();
       this.processingNewData = false;
-      if (this.scanFinished) await this.finishScan();
+      if (this.scanFinished) await this.finishJob();
     });
 
     this.dispatcher.on(ScannerEvents.DISPATCHER_FINISHED, async () => {
       if (!this.winnower.hasPendingFiles()) {
         if (this.processingNewData) this.scanFinished = true;
-        else await this.finishScan();
+        else await this.finishJob();
       }
     });
 
@@ -197,12 +196,20 @@ export class Scanner extends EventEmitter {
     return this.workDirectory;
   }
 
-  cleanWorkDirectory() {
+  public cleanWorkDirectory(): void {
     if (fs.existsSync(this.resultFilePath)) fs.unlinkSync(this.resultFilePath);
     if (fs.existsSync(this.wfpFilePath)) fs.unlinkSync(this.wfpFilePath);
   }
 
-  async finishScan() {
+  private async finishJob() {
+    this.scannerInput.shift();
+    this.reportLog(`[ SCANNER ]: Job finished. ${this.scannerInput.length} pendings`);
+
+    if(this.scannerInput.length) this.winnower.startWinnowing(this.scannerInput[0]);
+    else await this.finishScan();
+  }
+
+  private async finishScan() {
     if (!this.isBufferEmpty()) this.bufferToFiles();
     const results = JSON.parse(await fs.promises.readFile(this.resultFilePath, 'utf8'));
     const sortedPaths = sortPaths(Object.keys(results), '/');
@@ -253,30 +260,26 @@ export class Scanner extends EventEmitter {
   }
 
 
-  public scanList(files, scanRoot = ''): Promise<void> {
+  public scanFromWinnowingFile(wfpFilePath: string): Promise<void> {
     this.init();
-    this.filesToScan = files;
-    this.scanRoot = scanRoot;
     this.createOutputFiles();
-    if (!Object.entries(files).length) {
-      this.finishScan();
-      return this.finishPromise;
-    }
-    this.winnower.startWinnowing(this.filesToScan, scanRoot);
+    this.winnower.startWinnowingFromFile(wfpFilePath);
     return this.finishPromise;
   }
 
 
-  public scanFromWinnowingFile(wfpFilePath: string): Promise<void> {
+  public scan(scannerInput: Array<ScannerInput>): Promise<void> {
     this.init();
     this.createOutputFiles();
-    this.winnower.startWinnowingFromFile(wfpFilePath);
-    return this.finishPromise;
-  }
-
+    this.scannerInput = scannerInput;
+    // If some jobs have no files to scan, we return
+    if (this.scannerInput.some((input) => !input.fileList.length)){
+      this.finishScan();
+      return this.finishPromise;
+    }
 
-  public scan(scanInput: Array<any>): Promise<void> {
-    return new Promise((resolve, reject) => {});
+    this.winnower.startWinnowing(this.scannerInput[0]);
+    return this.finishPromise;
   }
 
 
@@ -289,13 +292,13 @@ export class Scanner extends EventEmitter {
   pause() {
     this.running = false;
     this.winnower.pause();
-    this.dispatcher.pause();
+    // this.dispatcher.pause();
   }
 
   resume() {
     this.running = true;
     this.winnower.resume();
-    this.dispatcher.resume();
+    // this.dispatcher.resume();
   }
 
   stop() {

package/src/lib/scanner/{ScannerEvents.ts → ScannerTypes.ts}

@@ -27,3 +27,15 @@ export enum ScannerEvents {
 
   ERROR = 'error',
 };
+
+export enum WinnowingMode {
+  FULL_WINNOWING = 'FULL_WINNOWING',
+  WINNOWING_ONLY_MD5 = 'WINNOWING_ONLY_MD5',
+};
+
+export interface ScannerInput {
+  engineFlags?: number;
+  folderRoot?: string;
+  fileList: Array<string>;
+  winnowingMode?: WinnowingMode;  // Enable winnowing algorithm, otherwise is scanned only MD5
+};

package/src/lib/scanner/Winnower/Winnower.ts

@@ -4,7 +4,7 @@ import { Worker } from 'worker_threads';
 
 import { ScannableItem } from '../Scannable/ScannableItem';
 import { ScannerCfg } from '../ScannerCfg';
-import { ScannerEvents } from '../ScannerEvents';
+import { ScannerEvents, ScannerInput, WinnowingMode } from '../ScannerTypes';
 import { WinnowerExtractor } from './WinnowerExtractor';
 
 import { WinnowerResponse } from './WinnowerResponse';
@@ -17,7 +17,7 @@ const { parentPort } = require('worker_threads');
 parentPort.on('message', async (scannableItem) => {
 
   let fingerprint;
-  if ( scannableItem.scanMode === "FULL_SCAN") {
+  if ( scannableItem.winnowingMode === "FULL_WINNOWING") {
     fingerprint = wfp_for_content(
       scannableItem.content,
       scannableItem.contentSource,
@@ -240,7 +240,7 @@ export class Winnower extends EventEmitter {
 
   private fileListIndex: number;
 
-  private scanRoot: string;
+  private folderRoot: string;
 
   private wfp: string;
 
@@ -250,6 +250,8 @@ export class Winnower extends EventEmitter {
 
   private isRunning: boolean;
 
+  private winnowingMode: WinnowingMode;
+
   private readingFromFile: boolean;
 
   private winnowingExtractor: WinnowerExtractor;
@@ -257,18 +259,17 @@ export class Winnower extends EventEmitter {
   constructor(scannerCfg = new ScannerCfg()) {
     super();
     this.scannerCfg = scannerCfg;
-    this.init();
-    this.prepareWorker();
   }
 
   init() {
     this.wfp = '';
-    this.scanRoot = '';
+    this.folderRoot = '';
     this.continue = true;
     this.isRunning = false;
     this.readingFromFile = false;
     this.fileList = [];
     this.fileListIndex = 0;
+    this.winnowingMode = WinnowingMode.FULL_WINNOWING;
   }
 
   prepareWorker() {
@@ -281,11 +282,11 @@ export class Winnower extends EventEmitter {
 
   recoveryIndex() {
     // Files: contains all files winnowed but not packed yet
-    const files = new WinnowerResponse(this.wfp, this.scanRoot).getFilesWinnowed();
+    const files = new WinnowerResponse(this.wfp, this.folderRoot).getFilesWinnowed();
     if (files.length) {
       const lastFileWinnowed = files[files.length - 1];
       let i = 0;
-      while (i <= files.length && lastFileWinnowed !== this.fileList[this.fileListIndex - i][0]) {
+      while (i <= files.length && lastFileWinnowed !== this.fileList[this.fileListIndex - i]) {
         i += 1;
       }
       // If file already winnowed cannot be found in fileList emit an error.
@@ -294,7 +295,7 @@ export class Winnower extends EventEmitter {
         return -1;
       }
       this.fileListIndex -= i;
-      if (this.fileList[this.fileListIndex][0] === lastFileWinnowed) this.fileListIndex += 1;
+      if (this.fileList[this.fileListIndex] === lastFileWinnowed) this.fileListIndex += 1;
     }
     return 0;
   }
@@ -332,18 +333,17 @@ export class Winnower extends EventEmitter {
   }
 
   processPackedWfp(content) {
-    const wnRsp = new WinnowerResponse(content, this.scanRoot);
+    const wnRsp = new WinnowerResponse(content, this.folderRoot);
     this.emit(ScannerEvents.WINNOWING_NEW_CONTENT, wnRsp);
   }
 
   async getNextScannableItem() {
     if (this.fileListIndex >= this.fileList.length) return null;
-    const path = this.fileList[this.fileListIndex][0];
-    const scanMode = this.fileList[this.fileListIndex][1];
-    const contentSource = path.replace(`${this.scanRoot}`, '');
+    const path = this.fileList[this.fileListIndex];
+    const contentSource = path.replace(`${this.folderRoot}`, '');
     const content = await fs.promises.readFile(path);
     this.fileListIndex += 1;
-    const scannable = new ScannableItem(content, contentSource, scanMode, this.scannerCfg.WFP_FILE_MAX_SIZE);
+    const scannable = new ScannableItem(content, contentSource, this.winnowingMode, this.scannerCfg.WFP_FILE_MAX_SIZE);
     return scannable;
   }
 
@@ -377,12 +377,17 @@ export class Winnower extends EventEmitter {
     if(winBlock === '') this.finishWinnowing();
   }
 
-  public async startWinnowing(files, scanRoot) {
+  public async startWinnowing(scanInput: ScannerInput): Promise<void> {
     this.emit(ScannerEvents.WINNOWER_LOG, '[ SCANNER ]: Starting Winnowing...');
+
+    this.init();
+    this.prepareWorker();
+
+    if(scanInput.winnowingMode) this.setWinnowingMode(scanInput.winnowingMode);
     this.readingFromFile = false;
-    this.scanRoot = scanRoot;
     this.isRunning = true;
-    this.fileList = Object.entries(files);
+    this.folderRoot = scanInput.folderRoot;
+    this.fileList = scanInput.fileList;
     this.nextStepMachine();
   }
 
@@ -428,4 +433,7 @@ export class Winnower extends EventEmitter {
     return this.isRunning;
   }
 
+  public setWinnowingMode(mode: WinnowingMode): void {
+    this.winnowingMode = mode;
+  }
 }

package/src/lib/scanner/Winnower/WinnowerResponse.ts

@@ -1,7 +1,7 @@
 export class WinnowerResponse {
   private wfpContent: string;
 
-  // private date: any;
+  private engineFlags: number;
 
   private scanRoot: string;
 
@@ -18,6 +18,14 @@ export class WinnowerResponse {
     return this.wfpContent;
   }
 
+  public setEngineFlags(engineFlags: number): void {
+    this.engineFlags = engineFlags;
+  }
+
+  public getEngineFlags(): number {
+    return this.engineFlags;
+  }
+
   public getFilesWinnowed() {
     const files = [];
     const regExp = new RegExp(/,(\/.*)/g);

package/src/lib/tree/Tree.ts

@@ -21,7 +21,7 @@ export class Tree {
 
   constructor(path: string) {
     const pathParts = path.split(pathLib.sep);
-    this.rootName = pathParts[pathParts.length - 1];
+    this.rootName = pathLib.basename(path);
     this.rootPath = path;
     this.rootFolder = new Folder('/', this.rootName);
   }
@@ -44,7 +44,7 @@ export class Tree {
     for (const dirEntry of dirEntries) {
       const fullPath = `${path}/${dirEntry.name}`;
       const relativePath = `${path}/${dirEntry.name}`.replace(this.rootPath, '');
-      if (this.filter && this.filter.include(fullPath))
+      if (!this.filter || this.filter.include(fullPath))
         if (dirEntry.isDirectory()) {
           const f: Folder = new Folder(relativePath, dirEntry.name);
           const subTree = this.buildTreeRec(`${path}/${dirEntry.name}`, f);
@@ -82,7 +82,7 @@ export class Tree {
   }
 
   public getFileList(): Array<string> {
-    const rootPath = this.rootPath.substring(0, this.rootPath.length - 1);
+    const rootPath = this.rootPath.substring(0, this.rootPath.length);
     const fList = this.rootFolder.getFiles();
     return fList.map((fileRelativePath: string) => {return (rootPath + fileRelativePath)});
   }