scanoss 0.2.10 → 0.2.11

package/src/lib/scanner/Dispatcher/Dispatcher.ts

@@ -1,151 +0,0 @@
-/* eslint-disable guard-for-in */
-/* eslint-disable no-restricted-syntax */
-import EventEmitter from 'eventemitter3';
-import fetch from 'node-fetch';
-import PQueue from "p-queue";
-import FormData from "form-data";
-
-import { ScannerEvents } from "../ScannerTypes";
-import { DispatcherResponse } from "./DispatcherResponse";
-import { ScannerCfg } from "../ScannerCfg";
-import { GlobalControllerAborter } from "./GlobalControllerAborter";
-import { DispatchableItem } from './DispatchableItem';
-
-export class Dispatcher extends EventEmitter {
-  private scannerCfg: ScannerCfg;
-
-  private pQueue;
-
-  private globalAbortController: GlobalControllerAborter;
-
-  private queueMaxLimitReached: boolean;
-
-  private queueMinLimitReached: boolean;
-
-  private recoverableErrors;
-
-  constructor(scannerCfg = new ScannerCfg()) {
-    super();
-    this.scannerCfg = scannerCfg;
-    this.init();
-  }
-
-  init() {
-    this.pQueue = new PQueue({
-      concurrency: this.scannerCfg.CONCURRENCY_LIMIT,
-    });
-    this.pQueue.clear();
-
-    this.pQueue.on('idle', () => {
-      this.emit(ScannerEvents.DISPATCHER_FINISHED);
-    });
-
-    this.pQueue.on('next', () => {
-      if ((this.pQueue.size + this.pQueue.pending) < this.scannerCfg.DISPATCHER_QUEUE_SIZE_MIN_LIMIT && !this.queueMinLimitReached) {
-        this.emit(ScannerEvents.DISPATCHER_QUEUE_SIZE_MIN_LIMIT);
-        this.queueMinLimitReached = true;
-        this.queueMaxLimitReached = false;
-      }
-    });
-
-    this.queueMaxLimitReached = false;
-    this.queueMinLimitReached = true;
-
-    this.globalAbortController = new GlobalControllerAborter();
-
-    this.recoverableErrors = new Set();
-    this.recoverableErrors.add('ECONNRESET');
-    this.recoverableErrors.add('TIMEOUT');
-  }
-
-  stop() {
-    this.pQueue.clear();
-    this.pQueue.pause();
-    this.globalAbortController.abortAll();
-  }
-
-  public dispatchItem(disptItem: DispatchableItem): void {
-    this.pQueue.add(() => this.dispatch(disptItem));
-
-    if (
-      this.pQueue.size + this.pQueue.pending >= this.scannerCfg.DISPATCHER_QUEUE_SIZE_MAX_LIMIT &&
-      !this.queueMaxLimitReached
-    ) {
-      this.emit(ScannerEvents.DISPATCHER_QUEUE_SIZE_MAX_LIMIT);
-      this.queueMaxLimitReached = true;
-      this.queueMinLimitReached = false;
-    }
-  }
-
-  handleUnrecoverableError(error,disptItem) {
-    this.emit('error', error, disptItem);
-  }
-
-  emitNoDispatchedItem(disptItem) {
-    this.emit(ScannerEvents.DISPATCHER_LOG, `[ SCANNER ]: WFP content sended to many times. Some files won't be scanned`);
-    this.emit(ScannerEvents.DISPATCHER_ITEM_NO_DISPATCHED, disptItem);
-  }
-
-  errorHandler(error, disptItem) {
-    if (!this.globalAbortController.isAborting()) {
-      if (error.name === 'AbortError') error.name = 'TIMEOUT';
-      if (this.recoverableErrors.has(error.code) || this.recoverableErrors.has(error.name)) {
-        disptItem.increaseErrorCounter();
-        if (disptItem.getErrorCounter() >= this.scannerCfg.MAX_RETRIES_FOR_RECOVERABLES_ERRORS) {
-          this.emitNoDispatchedItem(disptItem);
-          if (this.scannerCfg.ABORT_ON_MAX_RETRIES) this.handleUnrecoverableError(error, disptItem);
-          return;
-        }
-       // const leftRetry = this.scannerCfg.MAX_RETRIES_FOR_RECOVERABLES_ERRORS - disptItem.getErrorCounter();
-        this.emit(ScannerEvents.DISPATCHER_LOG,`[ SCANNER ]: Recoverable error happened sending WFP content to server. Reason: ${error.code || error.name}`);
-        this.dispatchItem(disptItem);
-        return;
-      }
-      this.handleUnrecoverableError(error, disptItem);
-    }
-  }
-
-  async dispatch(disptItem: DispatchableItem) {
-    const timeoutController = this.globalAbortController.getAbortController();
-    const timeoutId = setTimeout(() => timeoutController.abort(), this.scannerCfg.TIMEOUT);
-    try {
-      const form = new FormData();
-
-      form.append('filename', Buffer.from(disptItem.getContent()), 'data.wfp');
-
-      const engineFlag = disptItem.getWinnowerResponse().getEngineFlags();
-      if(engineFlag) form.append('flags', engineFlag);
-
-      this.emit(ScannerEvents.DISPATCHER_WFP_SENDED);
-      const response = await fetch(this.scannerCfg.API_URL, {
-        method: 'post',
-        body: form,
-        headers: { 'User-Agent': this.scannerCfg.CLIENT_TIMESTAMP, 'X-Session': this.scannerCfg.API_KEY },
-        signal: timeoutController.signal,
-      });
-
-      clearTimeout(timeoutId);
-      this.globalAbortController.removeAbortController(timeoutController);
-
-      if (!response.ok) {
-        const msg = await response.text();
-        const err = new Error(msg);
-        err["code"] = response.status;
-        err.name = ScannerEvents.ERROR_SERVER_SIDE;
-        throw err;
-      }
-
-      const dataAsText = await response.text();
-      const dataAsObj = JSON.parse(dataAsText);
-
-      const dispatcherResponse = new DispatcherResponse(dataAsObj, disptItem.getContent());
-      this.emit(ScannerEvents.DISPATCHER_NEW_DATA, dispatcherResponse);
-      return Promise.resolve();
-    } catch (e) {
-        clearTimeout(timeoutId);
-        this.globalAbortController.removeAbortController(timeoutController);
-        this.errorHandler(e, disptItem);
-        return Promise.resolve();
-    }
-  }
-}

package/src/lib/scanner/Dispatcher/DispatcherResponse.ts

@@ -1,40 +0,0 @@
-export class DispatcherResponse {
-  serverResponse: any;
-
-  wfpContent: string;
-
-  filesScanned: Array<string>;
-
-  constructor(serverResponse, wfpContent) {
-    this.serverResponse = serverResponse;
-    this.wfpContent = wfpContent;
-    this.filesScanned = Object.keys(this.serverResponse);
-    // this.verifyResponse();
-  }
-
-  getServerResponse() {
-    return this.serverResponse;
-  }
-
-  getWfpContent() {
-    return this.wfpContent;
-  }
-
-  matchRegex(str, re = /file=/g) {
-    return ((str || '').match(re) || []).length;
-  }
-
-  verifyResponse() {
-    const wfpNumFiles = this.matchRegex(this.wfpContent, /file=/g);
-    const serverResponseNumFiles = Object.keys(this.serverResponse).length;
-    if (wfpNumFiles !== serverResponseNumFiles) throw new Error(`The numbers of files in the wfp sended does not match with the server response`);
-  }
-
-  getFilesScanned() {
-    return this.filesScanned;
-  }
-
-  getNumberOfFilesScanned() {
-    return this.filesScanned.length;
-  }
-}

package/src/lib/scanner/Dispatcher/GlobalControllerAborter.ts

@@ -1,33 +0,0 @@
-
-import AbortController from 'abort-controller';
-
-export class GlobalControllerAborter {
-  private abortControllerList: Array<AbortController>;
-
-  private abortFlag: Boolean;
-
-  constructor() {
-    this.abortControllerList = [];
-    this.abortFlag = false;
-  }
-
-  abortAll() {
-    this.abortFlag = true;
-    for (const c of this.abortControllerList) c.abort();
-  }
-
-  isAborting() {
-    return this.abortFlag;
-  }
-
-  getAbortController() {
-    const c = new AbortController();
-    this.abortControllerList.push(c);
-    return c;
-  }
-
-  removeAbortController(c) {
-    const index = this.abortControllerList.findIndex((controller) => controller === c);
-    if (index > -1) this.abortControllerList.splice(index, 1);
-  }
-}

package/src/lib/scanner/Scannable/ScannableItem.ts

@@ -1,43 +0,0 @@
-import { WinnowingMode } from "../ScannerTypes";
-
-
-export class ScannableItem {
-  private contentSource: string;
-
-  private content: Buffer;
-
-  private winnowingMode: WinnowingMode;
-
-  private fingerprint: any;
-
-  private maxSizeWfp: any;
-
-  constructor(content: Buffer, contentSource: string, winnowingMode: any, maxSizeWfp: number) {
-    this.contentSource = contentSource;
-    this.content = content;
-    this.winnowingMode = winnowingMode;
-    this.maxSizeWfp = maxSizeWfp;
-  }
-
-  public getContent() {
-    return this.content;
-  }
-
-  public getContentSource() {
-    return this.contentSource;
-  }
-
-  public getFingerprint() {
-    return this.fingerprint;
-  }
-
-  public getWinnowingMode(): WinnowingMode {
-    return this.winnowingMode;
-  }
-
-  public getMaxSizeWfp() {
-    return this.maxSizeWfp;
-  }
-
-
-}

package/src/lib/scanner/Scanner.ts

@@ -1,319 +0,0 @@
-/* eslint-disable no-console */
-// 2.0
-import EventEmitter from 'eventemitter3';
-import os from 'os';
-import fs from 'fs';
-
-import { Winnower } from './Winnower/Winnower';
-import { Dispatcher } from './Dispatcher/Dispatcher';
-
-import { DispatchableItem } from './Dispatcher/DispatchableItem';
-import { DispatcherResponse } from './Dispatcher/DispatcherResponse';
-import { ScannerCfg } from './ScannerCfg';
-import { ScannerEvents, ScannerInput } from './ScannerTypes';
-
-
-import sortPaths from 'sort-paths';
-import { WinnowerResponse } from './Winnower/WinnowerResponse';
-
-
-let finishPromiseResolve;
-let finishPromiseReject;
-
-export class Scanner extends EventEmitter {
-  scannerCfg;
-
-  workDirectory;
-
-  scanRoot;
-
-  scannerId;
-
-  private winnower: Winnower;
-
-  private dispatcher: Dispatcher;
-
-  resultFilePath;
-
-  wfpFilePath;
-
-  scanFinished; // Both flags are used to prevent a race condition between DISPATCHER.NEW_DATA and DISPATCHER_FINISHED
-
-  processingNewData; // Both flags are used to prevent a race condition between DISPATCHER.NEW_DATA and DISPATCHER_FINISHED
-
-  responseBuffer;
-
-  processedFiles;
-
-  running;
-
-  filesToScan;
-
-  filesNotScanned;
-
-  finishPromise: Promise<void>;
-
-  private scannerInput: Array<ScannerInput>;
-
-  constructor(scannerCfg = new ScannerCfg()) {
-    super();
-    this.scannerCfg = scannerCfg;
-    this.scannerId = new Date().getTime();
-  }
-
-  init() {
-    this.scanFinished = false;
-    this.processingNewData = false;
-    this.running = true;
-    this.processedFiles = 0;
-    this.responseBuffer = [];
-    this.filesToScan = {};
-    this.filesNotScanned = {};
-    this.winnower = new Winnower(this.scannerCfg);
-    this.dispatcher = new Dispatcher(this.scannerCfg);
-
-    this.finishPromise = new Promise((resolve, reject) =>{
-      finishPromiseResolve = resolve;
-      finishPromiseReject = reject;
-    });
-
-    this.setWinnowerListeners();
-    this.setDispatcherListeners();
-
-    if (this.workDirectory === undefined) this.setWorkDirectory(`${os.tmpdir()}/scanner-${this.getScannerId()}`);
-  }
-
-  setWinnowerListeners() {
-    this.winnower.on(ScannerEvents.WINNOWING_NEW_CONTENT, (winnowerResponse: WinnowerResponse) => {
-      this.emit(ScannerEvents.WINNOWING_NEW_CONTENT, winnowerResponse);
-      this.reportLog(`[ SCANNER ]: New WFP content`);
-      winnowerResponse.setEngineFlags(this.scannerInput[0].engineFlags);
-      const disptItem = new DispatchableItem(winnowerResponse);
-      this.dispatcher.dispatchItem(disptItem);
-    });
-
-    this.winnower.on(ScannerEvents.WINNOWER_LOG, (msg) => {
-      this.reportLog(msg);
-    });
-
-    this.winnower.on(ScannerEvents.ERROR, (error) => {
-      this.errorHandler(error, ScannerEvents.MODULE_WINNOWER);
-    });
-  }
-
-  setDispatcherListeners() {
-    this.dispatcher.on(ScannerEvents.DISPATCHER_QUEUE_SIZE_MAX_LIMIT, () => {
-      this.reportLog(`[ SCANNER ]: Maximum queue size reached. Winnower will be paused`);
-      this.winnower.pause();
-    });
-
-    this.dispatcher.on(ScannerEvents.DISPATCHER_QUEUE_SIZE_MIN_LIMIT, () => {
-      this.reportLog(`[ SCANNER ]: Minimum queue size reached. Winnower will be resumed`);
-      this.winnower.resume();
-    });
-
-    this.dispatcher.on(ScannerEvents.DISPATCHER_NEW_DATA, async (response) => {
-      this.processingNewData = true;
-      this.processedFiles += response.getNumberOfFilesScanned();
-      this.reportLog(`[ SCANNER ]: Received results of ${response.getNumberOfFilesScanned()} files`);
-      this.emit(ScannerEvents.DISPATCHER_NEW_DATA, response);
-      this.insertIntoBuffer(response);
-      if (this.bufferReachedLimit()) this.bufferToFiles();
-      this.processingNewData = false;
-      if (this.scanFinished) await this.finishJob();
-    });
-
-    this.dispatcher.on(ScannerEvents.DISPATCHER_FINISHED, async () => {
-      if (!this.winnower.hasPendingFiles()) {
-        if (this.processingNewData) this.scanFinished = true;
-        else await this.finishJob();
-      }
-    });
-
-    this.dispatcher.on(ScannerEvents.DISPATCHER_ITEM_NO_DISPATCHED, (disptItem) => {
-      const filesNotScanned = disptItem.getWinnowerResponse().getFilesWinnowed();
-      this.appendFilesToNotScanned(filesNotScanned);
-    });
-
-    this.winnower.on(ScannerEvents.DISPATCHER_LOG, (msg) => {
-      this.reportLog(msg);
-    });
-
-    this.dispatcher.on(ScannerEvents.ERROR, (error, disptItem) => {
-      const wfpContent = disptItem.getWinnowerResponse().getContent();
-      fs.writeFileSync(`${this.workDirectory}/failed.wfp`, wfpContent, 'utf8');
-      this.errorHandler(error, ScannerEvents.MODULE_DISPATCHER);
-    });
-  }
-
-  appendFilesToNotScanned(fileList) {
-    const obj = {};
-    // eslint-disable-next-line no-restricted-syntax
-    for (const file of fileList) obj[file] = this.filesToScan[file];
-    Object.assign(this.filesNotScanned, obj);
-    return this.filesNotScanned;
-  }
-
-  insertIntoBuffer(dispatcherResponse) {
-    this.responseBuffer.push(dispatcherResponse);
-  }
-
-  isBufferEmpty() {
-    return this.responseBuffer.length === 0;
-  }
-
-  bufferReachedLimit() {
-    if (this.responseBuffer.length >= this.scannerCfg.MAX_RESPONSES_IN_BUFFER) return true;
-    return false;
-  }
-
-  bufferToFiles() {
-    let wfpContent = '';
-    const serverResponse = {};
-    // eslint-disable-next-line no-restricted-syntax
-    for (const dispatcherResponse of this.responseBuffer) {
-      wfpContent += dispatcherResponse.getWfpContent();
-      const serverResponseToAppend = dispatcherResponse.getServerResponse();
-      Object.assign(serverResponse, serverResponseToAppend);
-    }
-    this.appendOutputFiles(wfpContent, serverResponse);
-    this.responseBuffer = [];
-    const responses = new DispatcherResponse(serverResponse, wfpContent);
-    this.reportLog(`[ SCANNER ]: Persisted results of ${responses.getNumberOfFilesScanned()} files...`);
-    this.emit(ScannerEvents.RESULTS_APPENDED, responses, this.filesNotScanned);
-    return responses;
-  }
-
-  public setWorkDirectory(workDirectory: string) {
-    this.workDirectory = workDirectory;
-    this.resultFilePath = `${this.workDirectory}/result.json`;
-    this.wfpFilePath = `${this.workDirectory}/winnowing.wfp`;
-
-    if (!fs.existsSync(this.workDirectory)) fs.mkdirSync(this.workDirectory);
-  }
-
-  public getWorkDirectory(): string {
-    return this.workDirectory;
-  }
-
-  public cleanWorkDirectory(): void {
-    if (fs.existsSync(this.resultFilePath)) fs.unlinkSync(this.resultFilePath);
-    if (fs.existsSync(this.wfpFilePath)) fs.unlinkSync(this.wfpFilePath);
-  }
-
-  private async finishJob() {
-    this.scannerInput.shift();
-    this.reportLog(`[ SCANNER ]: Job finished. ${this.scannerInput.length} pendings`);
-
-    if(this.scannerInput.length) this.winnower.startWinnowing(this.scannerInput[0]);
-    else await this.finishScan();
-  }
-
-  private async finishScan() {
-    if (!this.isBufferEmpty()) this.bufferToFiles();
-    const results = JSON.parse(await fs.promises.readFile(this.resultFilePath, 'utf8'));
-    const sortedPaths = sortPaths(Object.keys(results), '/');
-    const resultSorted = {};
-    // eslint-disable-next-line no-restricted-syntax
-    for (const key of sortedPaths) resultSorted[key] = results[key];
-    await fs.promises.writeFile(this.resultFilePath, JSON.stringify(resultSorted, null, 2));
-    this.reportLog(
-      `[ SCANNER ]: Scan finished (Scanned: ${this.processedFiles}, Not Scanned: ${
-        Object.keys(this.filesNotScanned).length
-      })`
-    );
-    this.reportLog(`[ SCANNER ]: Results on: ${this.resultFilePath}`);
-    this.running = false;
-    this.emit(ScannerEvents.SCAN_DONE, this.resultFilePath, this.filesNotScanned);
-    finishPromiseResolve();
-  }
-
-  reportLog(txt, level = 'info') {
-    this.emit(ScannerEvents.SCANNER_LOG, txt, level);
-  }
-
-  errorHandler(error, origin) {
-    this.stop();
-    if (origin === ScannerEvents.MODULE_DISPATCHER) {
-    }
-    if (origin === ScannerEvents.MODULE_WINNOWER) {
-    }
-
-    this.reportLog(`[ SCANNER ]: Error reason ${error}`);
-
-    this.emit(ScannerEvents.ERROR, error);
-    if(this.finishPromise) finishPromiseReject(error);
-  }
-
-  createOutputFiles() {
-    if (!fs.existsSync(this.wfpFilePath)) fs.writeFileSync(this.wfpFilePath, '');
-    if (!fs.existsSync(this.resultFilePath)) fs.writeFileSync(this.resultFilePath, JSON.stringify({}));
-  }
-
-  appendOutputFiles(wfpContent, serverResponse) {
-    fs.appendFileSync(this.wfpFilePath, wfpContent);
-    const storedResultStr = fs.readFileSync(this.resultFilePath, 'utf-8');
-    const storedResultObj = JSON.parse(storedResultStr);
-    Object.assign(storedResultObj, serverResponse);
-    const newResultStr = JSON.stringify(storedResultObj);
-    fs.writeFileSync(this.resultFilePath, newResultStr);
-  }
-
-
-  public scanFromWinnowingFile(wfpFilePath: string): Promise<void> {
-    this.init();
-    this.createOutputFiles();
-    this.winnower.startWinnowingFromFile(wfpFilePath);
-    return this.finishPromise;
-  }
-
-
-  public scan(scannerInput: Array<ScannerInput>): Promise<void> {
-    this.init();
-    this.createOutputFiles();
-    this.scannerInput = scannerInput;
-    // If some jobs have no files to scan, we return
-    if (this.scannerInput.some((input) => !input.fileList.length)){
-      this.finishScan();
-      return this.finishPromise;
-    }
-
-    this.winnower.startWinnowing(this.scannerInput[0]);
-    return this.finishPromise;
-  }
-
-
-
-
-  getScannerId() {
-    return this.scannerId;
-  }
-
-  pause() {
-    this.running = false;
-    this.winnower.pause();
-    // this.dispatcher.pause();
-  }
-
-  resume() {
-    this.running = true;
-    this.winnower.resume();
-    // this.dispatcher.resume();
-  }
-
-  stop() {
-    this.reportLog(`[ SCANNER ]: Stopping scanner`);
-    this.running = false;
-    this.winnower.removeAllListeners();
-    this.dispatcher.removeAllListeners();
-    this.dispatcher.stop();
-    this.winnower.stop();
-  }
-
-  isRunning() {
-    return this.running;
-  }
-
-}
-
-

package/src/lib/scanner/ScannerCfg.ts

@@ -1,29 +0,0 @@
-export class ScannerCfg {
-  // Client Timestamp
-  CLIENT_TIMESTAMP = "SCANNER JS"
-
-  // API URL
-  API_URL = 'https://osskb.org/api/scan/direct';
-
-  API_KEY = '';
-
-  // Level of concurrency
-  CONCURRENCY_LIMIT = 15;
-
-  // Timeout for each transaction
-  TIMEOUT = 60000;
-
-  // The maximum size for each .wfp file
-  WFP_FILE_MAX_SIZE = 64 * 1000;
-
-  MAX_RETRIES_FOR_RECOVERABLES_ERRORS = 5;
-
-  ABORT_ON_MAX_RETRIES = true;
-
-  // Persist results after [ X ] number of server responses.
-  MAX_RESPONSES_IN_BUFFER = 300;
-
-  DISPATCHER_QUEUE_SIZE_MAX_LIMIT = 50;
-
-  DISPATCHER_QUEUE_SIZE_MIN_LIMIT = 20;
-};

package/src/lib/scanner/ScannerTypes.ts

@@ -1,41 +0,0 @@
-export enum ScannerEvents {
-  WINNOWING_STARTING = 'WINNOWING_STARTING',
-  WINNOWING_NEW_CONTENT = 'WINNOWING_NEW_CONTENT',
-  WINNOWING_FINISHED = 'WINNOWING_FINISHED',
-  WINNOWER_LOG = 'WINNOWER_LOG',
-
-  DISPATCHER_WFP_SENDED = 'DISPATCHER_WFP_SENDED',
-  DISPATCHER_NEW_DATA = 'DISPATCHER_NEW_DATA',
-  DISPATCHER_FINISHED = 'DISPATCHER_FINISHED',
-  DISPATCHER_ITEM_NO_DISPATCHED = 'DISPATCHER_ITEM_NO_DISPATCHED',
-  DISPATCHER_QUEUE_SIZE_MAX_LIMIT = 'DISPATCHER_QUEUE_FULL',
-  DISPATCHER_QUEUE_SIZE_MIN_LIMIT = 'DISPATCHER_QUEUE_SIZE_MIN_LIMIT',
-  DISPATCHER_LOG = 'DISPATCHER_LOG',
-
-  ERROR_SCANNER_ABORTED = 'ERROR_SCANNER_ABORTED',
-
-  ERROR_SERVER_SIDE = 'ERROR_SERVER_SIDE',
-  ERROR_CLIENT_SIDE = 'ERROR_CLIENT_SIDE',
-
-  MODULE_DISPATCHER = 'MODULE_DISPATCHER',
-  MODULE_WINNOWER = 'MODULE_WINNOWER',
-
-  SCAN_DONE = 'SCAN_DONE',
-  RESULTS_APPENDED = 'RESULTS_APPENDED',
-
-  SCANNER_LOG = 'SCANNER_LOG',
-
-  ERROR = 'error',
-};
-
-export enum WinnowingMode {
-  FULL_WINNOWING = 'FULL_WINNOWING',
-  WINNOWING_ONLY_MD5 = 'WINNOWING_ONLY_MD5',
-};
-
-export interface ScannerInput {
-  engineFlags?: number;
-  folderRoot?: string;
-  fileList: Array<string>;
-  winnowingMode?: WinnowingMode;  // Enable winnowing algorithm, otherwise is scanned only MD5
-};