scanoss 0.7.9 → 0.7.11

package/src/sdk/scanner/Scanner.ts

@@ -1,360 +0,0 @@
-/* eslint-disable no-console */
-// 2.0
-import EventEmitter from 'eventemitter3';
-import os from 'os';
-import fs from 'fs';
-
-
-import { Dispatcher } from './Dispatcher/Dispatcher';
-
-import { DispatchableItem } from './Dispatcher/DispatchableItem';
-import { DispatcherResponse } from './Dispatcher/DispatcherResponse';
-import { ScannerCfg } from './ScannerCfg';
-import { ScannerEvents, ScannerInput } from './ScannerTypes';
-
-import sortPaths from 'sort-paths';
-
-import { WfpProvider } from './WfpProvider/WfpProvider';
-import { FingerprintPackage } from './WfpProvider/FingerprintPackage';
-import { WfpCalculator } from './WfpProvider/WfpCalculator/WfpCalculator';
-import { WfpSplitter } from './WfpProvider/WfpSplitter/WfpSplitter';
-
-let finishPromiseResolve;
-let finishPromiseReject;
-
-export class Scanner extends EventEmitter {
-  private scannerCfg: ScannerCfg;
-
-  private workDirectory: string;
-
-  private resultFilePath: string;
-
-  private wfpFilePath: string;
-
-  private scanRoot: string;
-
-  private scannerId: string;
-
-  private wfpProvider: WfpProvider;
-
-  private dispatcher: Dispatcher;
-
-  private finishPromise: Promise<string>;
-
-  private scannerInput: Array<ScannerInput>;
-
-  private scanFinished: boolean; // Both flags are used to prevent a race condition between DISPATCHER.NEW_DATA and DISPATCHER_FINISHED
-
-  private processingNewData: boolean; // Both flags are used to prevent a race condition between DISPATCHER.NEW_DATA and DISPATCHER_FINISHED
-
-  private processedFiles: number;
-
-  private running: boolean;
-
-  private filesToScan;
-
-  private responseBuffer;
-
-  private filesNotScanned;
-
-  constructor(scannerCfg = new ScannerCfg()) {
-    super();
-    this.scannerCfg = scannerCfg;
-    this.scannerId = new Date().getTime().toString();
-  }
-
-  public init() {
-    this.scanFinished = false;
-    this.processingNewData = false;
-    this.running = true;
-    this.processedFiles = 0;
-    this.responseBuffer = [];
-    this.filesToScan = {};
-    this.filesNotScanned = {};
-    this.wfpProvider = new WfpCalculator(this.scannerCfg);
-    this.dispatcher = new Dispatcher(this.scannerCfg);
-
-    this.finishPromise = new Promise((resolve, reject) =>{
-      finishPromiseResolve = resolve;
-      finishPromiseReject = reject;
-    });
-
-    this.setWinnowerListeners();
-    this.setDispatcherListeners();
-
-    if (this.workDirectory === undefined) this.setWorkDirectory(`${os.tmpdir()}/scanner-${this.getScannerId()}`);
-  }
-
-  private setWinnowerListeners() {
-    this.wfpProvider.on(ScannerEvents.WINNOWING_NEW_CONTENT, (fingerprintPackage: FingerprintPackage) => {
-      this.emit(ScannerEvents.WINNOWING_NEW_CONTENT, fingerprintPackage);
-      this.reportLog(`[ SCANNER ]: New WFP content`);
-
-      const item = new DispatchableItem();
-      item.setFingerprintPackage(fingerprintPackage);
-      if(this.scannerInput[0]?.engineFlags) item.setEngineFlags(this.scannerInput[0]?.engineFlags);
-
-      if(this.scannerInput[0]?.sbom && this.scannerInput[0]?.sbomMode)
-        item.setSbom(this.scannerInput[0]?.sbom, this.scannerInput[0]?.sbomMode);
-
-      this.dispatcher.dispatchItem(item);
-    });
-
-    this.wfpProvider.on(ScannerEvents.WINNOWING_STATUS, (newFilesProcessed)=>{
-      this.emit(ScannerEvents.WINNOWING_STATUS, newFilesProcessed)
-    });
-
-    this.wfpProvider.on(ScannerEvents.WINNOWER_LOG, (msg) => {
-      this.reportLog(msg);
-    });
-
-    this.wfpProvider.on(ScannerEvents.ERROR, (error) => {
-      this.errorHandler(error, ScannerEvents.MODULE_WINNOWER);
-    });
-  }
-
-  private setDispatcherListeners() {
-    this.dispatcher.on(ScannerEvents.DISPATCHER_QUEUE_SIZE_MAX_LIMIT, () => {
-      this.reportLog(`[ SCANNER ]: Maximum queue size reached. Winnower will be paused`);
-      this.wfpProvider.pause();
-    });
-
-    this.dispatcher.on(ScannerEvents.DISPATCHER_QUEUE_SIZE_MIN_LIMIT, () => {
-      this.reportLog(`[ SCANNER ]: Minimum queue size reached. Winnower will be resumed`);
-      this.wfpProvider.resume();
-    });
-
-    this.dispatcher.on(ScannerEvents.DISPATCHER_NEW_DATA, async (response: DispatcherResponse) => {
-      this.processingNewData = true;
-      this.processedFiles += response.getNumberOfFilesScanned();
-      this.reportLog(`[ SCANNER ]: Received results of ${response.getNumberOfFilesScanned()} files`);
-      this.emit(ScannerEvents.DISPATCHER_NEW_DATA, response);
-      this.insertIntoBuffer(response);
-      if (this.bufferReachedLimit()) this.bufferToFiles();  //Uses sync to ensure no new data is appended to the buffer
-      this.processingNewData = false;
-      if (this.scanFinished) await this.finishJob();
-    });
-
-    this.dispatcher.on(ScannerEvents.DISPATCHER_FINISHED, async () => {
-      if (!this.wfpProvider.hasPendingFiles()) {
-        if (this.processingNewData) this.scanFinished = true;
-        else await this.finishJob();
-      }
-    });
-
-    this.dispatcher.on(ScannerEvents.DISPATCHER_ITEM_NO_DISPATCHED, (disptItem: DispatchableItem) => {
-      const filesNotScanned = disptItem.getFingerprintPackage().getFilesFingerprinted();
-      this.appendFilesToNotScanned(filesNotScanned);
-    });
-
-    this.wfpProvider.on(ScannerEvents.DISPATCHER_LOG, (msg) => {
-      this.reportLog(msg);
-    });
-
-    this.dispatcher.on(ScannerEvents.ERROR, (error: Error, disptItem: DispatchableItem) => {
-      const wfpContent = disptItem.getFingerprintPackage().getContent();
-      fs.writeFileSync(`${this.workDirectory}/failed.wfp`, wfpContent, 'utf8');
-      this.errorHandler(error, ScannerEvents.MODULE_DISPATCHER);
-    });
-  }
-
-  private appendFilesToNotScanned(fileList) {
-    const obj = {};
-    // eslint-disable-next-line no-restricted-syntax
-    for (const file of fileList) obj[file] = this.filesToScan[file];
-    Object.assign(this.filesNotScanned, obj);
-    return this.filesNotScanned;
-  }
-
-  private insertIntoBuffer(dispatcherResponse) {
-    this.responseBuffer.push(dispatcherResponse);
-  }
-
-  private isBufferEmpty() {
-    return this.responseBuffer.length === 0;
-  }
-
-  private bufferReachedLimit() {
-    if (this.responseBuffer.length >= this.scannerCfg.MAX_RESPONSES_IN_BUFFER) return true;
-    return false;
-  }
-
-  private bufferToFiles() {
-    let wfpContent = '';
-    const serverResponse = {};
-    // eslint-disable-next-line no-restricted-syntax
-    for (const dispatcherResponse of this.responseBuffer) {
-      wfpContent += dispatcherResponse.getWfpContent();
-      const serverResponseToAppend = dispatcherResponse.getServerResponse();
-      Object.assign(serverResponse, serverResponseToAppend);
-    }
-    this.appendOutputFiles(wfpContent, serverResponse);
-    this.responseBuffer = [];
-    const responses = new DispatcherResponse(serverResponse, wfpContent);
-    this.reportLog(`[ SCANNER ]: Persisted results of ${responses.getNumberOfFilesScanned()} files...`);
-    this.emit(ScannerEvents.RESULTS_APPENDED, responses, this.filesNotScanned);
-    return responses;
-  }
-
-  public setWorkDirectory(workDirectory: string) {
-    this.workDirectory = workDirectory;
-    this.resultFilePath = `${this.workDirectory}/result.json`;
-    this.wfpFilePath = `${this.workDirectory}/winnowing.wfp`;
-
-    if (!fs.existsSync(this.workDirectory)) fs.mkdirSync(this.workDirectory);
-  }
-
-  public getWorkDirectory(): string {
-    return this.workDirectory;
-  }
-
-  public cleanWorkDirectory(): void {
-    if (fs.existsSync(this.resultFilePath)) fs.unlinkSync(this.resultFilePath);
-    if (fs.existsSync(this.wfpFilePath)) fs.unlinkSync(this.wfpFilePath);
-  }
-
-  private async finishJob() {
-    this.scannerInput.shift();
-    this.reportLog(`[ SCANNER ]: Job finished. ${this.scannerInput.length} pendings`);
-
-    if(this.scannerInput.length) {
-      if (this.scannerInput[0].wfpPath) {
-        this.wfpProvider = new WfpSplitter();
-        this.setWinnowerListeners();
-        this.wfpProvider.start(this.scannerInput[0]);
-      } else {
-        const folderRoot = this.scannerInput[0].folderRoot;
-        const winnowingMode = this.scannerInput[0].winnowingMode;
-        const fileList = this.scannerInput[0].fileList;
-        this.wfpProvider.start({folderRoot, winnowingMode, fileList});
-      }
-     } else await this.finishScan();
-  }
-
-  private async finishScan() {
-    if (!this.isBufferEmpty()) this.bufferToFiles();
-    const results = JSON.parse(await fs.promises.readFile(this.resultFilePath, 'utf8'));
-    const sortedPaths = sortPaths(Object.keys(results), '/');
-    const resultSorted = {};
-    // eslint-disable-next-line no-restricted-syntax
-    for (const key of sortedPaths) resultSorted[key] = results[key];
-    await fs.promises.writeFile(this.resultFilePath, JSON.stringify(resultSorted, null, 2));
-    this.reportLog(
-      `[ SCANNER ]: Scan finished (Scanned: ${this.processedFiles}, Not Scanned: ${
-        Object.keys(this.filesNotScanned).length
-      })`
-    );
-    this.reportLog(`[ SCANNER ]: Results on: ${this.resultFilePath}`);
-    this.running = false;
-    this.emit(ScannerEvents.SCAN_DONE, this.resultFilePath, this.filesNotScanned);
-    finishPromiseResolve(this.resultFilePath);
-  }
-
-  private reportLog(txt, level = 'info') {
-    this.emit(ScannerEvents.SCANNER_LOG, txt, level);
-  }
-
-  private errorHandler(error, origin) {
-    this.abort();
-    if (origin === ScannerEvents.MODULE_DISPATCHER) {
-    }
-    if (origin === ScannerEvents.MODULE_WINNOWER) {
-    }
-
-    this.reportLog(`[ SCANNER ]: Error reason ${error}`);
-
-    this.emit(ScannerEvents.ERROR, error);
-    if(this.finishPromise) finishPromiseReject(error);
-  }
-
-  private createOutputFiles() {
-    if (!fs.existsSync(this.wfpFilePath)) fs.writeFileSync(this.wfpFilePath, '');
-    if (!fs.existsSync(this.resultFilePath)) fs.writeFileSync(this.resultFilePath, JSON.stringify({}));
-  }
-
-  private appendOutputFiles(wfpContent, serverResponse) {
-    fs.appendFileSync(this.wfpFilePath, wfpContent);
-    const storedResultStr = fs.readFileSync(this.resultFilePath, 'utf-8');
-    const storedResultObj = JSON.parse(storedResultStr);
-    Object.assign(storedResultObj, serverResponse);
-    const newResultStr = JSON.stringify(storedResultObj);
-    fs.writeFileSync(this.resultFilePath, newResultStr);
-  }
-
-
-  public scan(scannerInput: Array<ScannerInput>): Promise<string> {
-    this.init();
-    this.createOutputFiles();
-    this.scannerInput = scannerInput;
-
-    if (!this.isValidInput(scannerInput)) {
-      this.finishScan();
-      return this.finishPromise;
-    }
-
-    if (scannerInput[0]?.wfpPath) {
-      this.wfpProvider = new WfpSplitter();
-      this.setWinnowerListeners();
-      this.wfpProvider.start(scannerInput[0]);
-    } else {
-      const folderRoot = this.scannerInput[0].folderRoot;
-      const winnowingMode = this.scannerInput[0].winnowingMode;
-      const fileList = this.scannerInput[0].fileList;
-      this.wfpProvider.start({folderRoot, winnowingMode, fileList});
-    }
-    return this.finishPromise;
-  }
-
-
-
-
-  private isValidInput(scannerInput: Array<ScannerInput>): boolean {
-    if (!scannerInput) {
-      this.reportLog('[ SCANNER ]: No input provided', 'warning');
-      return false;
-    }
-
-    if (!Array.isArray(scannerInput)) {
-      this.reportLog('[ SCANNER ]: Input must be an array','warning');
-      return false;
-    }
-
-    if (!scannerInput.length) {
-      this.reportLog('[ SCANNER ]: Input array is empty', 'warning');
-      return false;
-    }
-
-    if (scannerInput.some((input) => !input.fileList.length && !input.wfpPath)) {
-      this.reportLog('[ SCANNER ]: Input array contains an element with no file list','warning');
-      return false;
-    }
-
-    return true;
-  }
-
-
-  public getScannerId() {
-    return this.scannerId;
-  }
-
-  private abort() {
-    this.running = false;
-    this.wfpProvider.removeAllListeners();
-    this.dispatcher.removeAllListeners();
-    this.dispatcher.stop();
-    this.wfpProvider.stop();
-  }
-
-  public stop() {
-    this.reportLog(`[ SCANNER ]: Stopping scanner`);
-    this.abort();
-    finishPromiseResolve();
-  }
-
-  public isRunning() {
-    return this.running;
-  }
-
-}
-
-

package/src/sdk/scanner/ScannerCfg.ts

@@ -1,43 +0,0 @@
-export class ScannerCfg {
-  // Client Timestamp, default value is scanoss-js/${version}
-  CLIENT_TIMESTAMP = ""
-
-  // API URL
-  API_URL = 'https://osskb.org/api/scan/direct';
-
-  API_KEY = '';
-
-  CA_CERT = '';
-
-  PROXY = '';
-
-  //Set to true to ignore self certificates issues
-  IGNORE_CERT_ERRORS = false;
-
-  // Level of concurrency
-  CONCURRENCY_LIMIT = 15;
-
-  // Timeout for each transaction
-  TIMEOUT = 60000;
-
-  // The maximum size for each .wfp file
-  WFP_FILE_MAX_SIZE = 64 * 1000;
-
-  //After processing #WINNOWING_AFTER_X_REPORT_STATUS files,
-  // the winnowing algorithm will report a ScannerEvents.WINNOWING_STATUS event.
-  WINNOWING_REPORT_STATUS_AFTER_X = 10;
-
-  MAX_RETRIES_FOR_RECOVERABLES_ERRORS = 5;
-
-  ABORT_ON_MAX_RETRIES = true;
-
-  // Persist results after [ X ] server responses
-  MAX_RESPONSES_IN_BUFFER = 300;
-
-  DISPATCHER_QUEUE_SIZE_MAX_LIMIT = 300;
-
-  DISPATCHER_QUEUE_SIZE_MIN_LIMIT = 200;
-
-
-
-};

package/src/sdk/scanner/ScannerTypes.ts

@@ -1,127 +0,0 @@
-import path from 'path';
-
-export enum ScannerEvents {
-  WINNOWING_STARTING = 'WINNOWING_STARTING',
-  WINNOWING_NEW_CONTENT = 'WINNOWING_NEW_CONTENT',
-  WINNOWING_STATUS = 'WINNOWING_STATUS',
-  WINNOWING_FINISHED = 'WINNOWING_FINISHED',
-  WINNOWER_LOG = 'WINNOWER_LOG',
-
-  DISPATCHER_WFP_SENDED = 'DISPATCHER_WFP_SENDED',
-  DISPATCHER_NEW_DATA = 'DISPATCHER_NEW_DATA',
-  DISPATCHER_FINISHED = 'DISPATCHER_FINISHED',
-  DISPATCHER_ITEM_NO_DISPATCHED = 'DISPATCHER_ITEM_NO_DISPATCHED',
-  DISPATCHER_QUEUE_SIZE_MAX_LIMIT = 'DISPATCHER_QUEUE_FULL',
-  DISPATCHER_QUEUE_SIZE_MIN_LIMIT = 'DISPATCHER_QUEUE_SIZE_MIN_LIMIT',
-  DISPATCHER_LOG = 'DISPATCHER_LOG',
-
-  ERROR_SCANNER_ABORTED = 'ERROR_SCANNER_ABORTED',
-
-  ERROR_SERVER_SIDE = 'ERROR_SERVER_SIDE',
-  ERROR_CLIENT_SIDE = 'ERROR_CLIENT_SIDE',
-
-  MODULE_DISPATCHER = 'MODULE_DISPATCHER',
-  MODULE_WINNOWER = 'MODULE_WINNOWER',
-
-  SCAN_DONE = 'SCAN_DONE',
-  RESULTS_APPENDED = 'RESULTS_APPENDED',
-
-  SCANNER_LOG = 'SCANNER_LOG',
-
-  ERROR = 'error',
-};
-
-export enum WinnowingMode {
-  FULL_WINNOWING = 'FULL_WINNOWING',
-  FULL_WINNOWING_HPSM = 'FULL_WINNOWING_HPSM',
-  WINNOWING_ONLY_MD5 = 'WINNOWING_ONLY_MD5',
-};
-
-export enum SbomMode {
-  SBOM_IGNORE = 'blacklist',
-  SBOM_IDENTIFY = 'identify'
-}
-
-export interface ScannerInput {
-  fileList: Array<string>;
-  folderRoot?: string;
-  engineFlags?: number;
-  winnowingMode?: WinnowingMode;  // Enable winnowing algorithm, otherwise is scanned only MD5
-  wfpPath?: string;
-  sbom?: string;
-  sbomMode?: SbomMode;
-};
-
-/********************** Scanner results types **********************/
-
-export type ScannerResults = Record<string , ScannerComponent[]>;
-
-export enum ScannerComponentId {NONE = 'none', FILE = 'file' , SNIPPET = 'snippet'};
-
-export interface ScannerComponent {
-  id: ScannerComponentId;
-  status: string;
-  lines: string;
-  oss_lines: string;
-  matched: string;
-  purl: string[];
-  vendor: string
-  component: string;
-  version: string;
-  latest: string;
-  url: string;
-  release_date: string;
-  file: string;
-  url_hash: string;
-  file_hash: string;
-  source_hash: string;
-  file_url: string;
-  licenses: {
-    name: string;
-    patent_hints: string;
-    copyleft: string;
-    checklist_url: string;
-    osadl_updated: string;
-    source: string;
-    incompatible_with?: string;
-  }[];
-  dependencies: {
-    vendor: string;
-    component: string;
-    version: string;
-    source: string; }[];
-  copyrights: {
-    name: string;
-    source: string; }[];
-  vulnerabilities: {
-    ID: string;
-    CVE: string;
-    severity: string;
-    reported: string;
-    introduced: string;
-    patched: string;
-    summary: string;
-    source: string; }[];
-  quality: {
-    score: string;
-    source: string; }[];
-  cryptography: any[];
-  health: {
-    creation_date: string;
-    issues: number;
-    last_push: string;
-    last_update: string;
-    watchers: number;
-    country: string;
-    stars: number;
-    forks: number;
-  };
-  server: {
-    version: string;
-    kb_version: { monthly: string; daily: string; }
-    hostname: string;
-    flags: string;
-    elapsed: string;
-  };
-}
-/********************** Scanner results types **********************/

package/src/sdk/scanner/WfpProvider/FingerprintPackage.ts

@@ -1,34 +0,0 @@
-export class FingerprintPackage {
-  private wfpContent: string;
-
-  private scanRoot: string;
-
-  constructor(wfpContent: string, scanRoot = '') {
-    this.wfpContent = wfpContent;
-    this.scanRoot = scanRoot;
-  }
-
-  public isEqual(fingerprintPackage: FingerprintPackage): boolean {
-    return this.getContent() === fingerprintPackage.getContent();
-  }
-
-  public getContent(): string {
-    return this.wfpContent;
-  }
-
-  public getNumberFilesFingerprinted() {
-    const match = this.getContent().match(/file=/g);
-    if(!match) return 0;
-    return match.length;
-  }
-
-  public getFilesFingerprinted() {
-    const files = [];
-    const regExp = new RegExp(/,(.?\/.*)/g);
-    let result;
-    // eslint-disable-next-line no-cond-assign
-    while ((result = regExp.exec(this.wfpContent))) files.push(this.scanRoot + result[1]);
-    return files || '';
-  }
-}
-