scanoss 0.7.9 → 0.7.10

package/src/sdk/scanner/WfpProvider/WfpSplitter/WfpSplitter.ts

@@ -1,184 +0,0 @@
-import { IWfpProviderInput, WfpProvider } from "../WfpProvider";
-import fs from "fs";
-import { ScannerCfg } from "../../ScannerCfg";
-import { Readable } from "stream";
-
-//Regex find paths from wfp string
-
-export class WfpSplitter extends WfpProvider {
-
-  private continue: boolean;
-
-  private wfpStream: Readable;
-
-  private chunkDataRead: string;
-
-  private fingerprints: Array<string>;
-
-  private fingerprintIndex: number;
-
-  private timer: ReturnType<typeof setTimeout>;
-
-  private ignoreFiles: Set<string>;
-
-  constructor(scannerCfg = new ScannerCfg()) {
-    super();
-    this.scannerCfg = scannerCfg;
-  }
-
-  public start(params: IWfpProviderInput): Promise<void> {
-
-    this.sendLog('[ SCANNER ]: WFP Splitter starting...');
-
-    this.init();
-    this.pendingFiles = true;
-    this.chunkDataRead = '';
-    this.fingerprints = [];
-    this.continue = true;
-    this.fingerprintIndex = 0;
-    this.ignoreFiles = new Set(params?.fileList);
-
-    const wfpPath = params.wfpPath;
-
-    if (!wfpPath) this.sendError('WFP path is not defined');
-
-    this.wfpStream = fs.createReadStream(wfpPath, { encoding: 'utf8' });
-    this.setStreamListeners();
-
-    return this.finishPromise;
-  }
-
-  public stop(): void {
-    this.continue = false;
-  }
-
-  public pause(): void {
-    this.sendLog('[ SCANNER ]: WFP Splitter paused...')
-    this.continue = false;
-  }
-
-  public resume(): void {
-    this.sendLog('[ SCANNER ]: WFP Splitter resumed...')
-    this.continue = true;
-    this.sendFingerprints();
-    this.streamBufferFlush();
-  }
-
-
-  private sendFingerprints() {
-    //The timer allows the main thread not to be blocked while sending wfps
-    if(this.timer === undefined) {
-      this.timer = setInterval(() => {
-        if(this.fingerprintIndex < this.fingerprints.length && this.continue) {
-          this.fingerprintPacker(this.fingerprints[this.fingerprintIndex]);
-          this.fingerprintIndex++;
-        } else {
-          this.stopSendFingerprints();
-        }
-      });
-    }
-
-
-  }
-
-  private stopSendFingerprints() {
-    clearInterval(this.timer);
-    this.timer = undefined;
-    if(this.fingerprintIndex === this.fingerprints.length){
-      this.finishWinnowing();
-      this.sendLog('[ SCANNER ]: WFP Splitter finished...');
-    }
-  }
-
-  private streamBufferFlush(): void {
-
-    let ignoreFirstChunkOfFingerprint = false;
-
-
-    //Use a loop to make sure we read all currently available data
-    while (this.continue && null !== (this.chunkDataRead = this.wfpStream.read(1 * 1024 * 1024))) {  // Read chunks of 1MB 1*1024*1024
-
-      /**** This part removes all the wfp that includes the paths inside this.ignoreFiles ****/
-      if (this.ignoreFiles.size > 0) {
-
-        // Removes fingerprints that are loose because the file=...... was removed in previous iteration
-        if (ignoreFirstChunkOfFingerprint) { //TODO: Test this scenario
-          //If there is no file= delete everything then
-          if (this.chunkDataRead.indexOf("file=") >= 0) {
-            this.chunkDataRead = this.chunkDataRead.substring(this.chunkDataRead.indexOf('file='));
-            ignoreFirstChunkOfFingerprint = false;
-          } else this.chunkDataRead = "";
-        }
-
-
-        const rWfpPath = new RegExp(/^file=\w+,\d+,(?<path>.+)$/gm)
-        //Search for paths in the wfp and compares with the ignorefiles set
-        //When there is a match the matched fingerprint is deleted on the fly
-        let result;
-        while ((result = rWfpPath.exec(this.chunkDataRead)) !== null) {
-          if (this.ignoreFiles.has(result?.groups?.path)) {
-            const indexDeleteFrom = result.index
-
-            //TODO: Verify this condition
-            //If there is no next file= in the string, remove until end.
-            let indexDeleteTo = this.chunkDataRead.indexOf('file=', indexDeleteFrom + 1)
-            if (indexDeleteTo < 0) {
-              indexDeleteTo = this.chunkDataRead.length;
-
-              //After the deletion of a wfp there are no other file=, so then set ignoreFirstChunkOfFingerprint to true.
-              //In the next iteration, the next chunk of data will be fingerprints without a file=. So, this first part will be discarded.
-              ignoreFirstChunkOfFingerprint = true;
-            }
-
-            const first = this.chunkDataRead.substring(0, indexDeleteFrom);
-            const second = this.chunkDataRead.substring(indexDeleteTo, this.chunkDataRead.length);
-            this.chunkDataRead = first + second;
-            rWfpPath.lastIndex = 0; //Make sure we reset the state of the regex.
-          }
-        }
-      }
-      /**** This part removes all the wfp that includes the paths inside this.ignoreFiles ****/
-
-
-
-      /**** This part process a chunk of wfp and send the packages to the subscriber ****/
-      if(this.chunkDataRead.length){
-
-        // Extract the first portion of the chunk until the first file=
-        // Then append the chunk to the last fingerprint.
-        if (!this.chunkDataRead.startsWith('file=') ) {
-          const chunkOfData = this.chunkDataRead.substring(0, this.chunkDataRead.indexOf('file='));
-          this.fingerprints[this.fingerprints.length-1] += chunkOfData;
-        }
-
-        this.fingerprints = [...this.fingerprints, ...this.splitFingerprints(this.chunkDataRead)];
-        this.sendFingerprints();
-      }
-      /**** This part process a chunk of wfp and send the packages to the subscriber ****/
-
-    }
-  }
-
-  private setStreamListeners() {
-    this.wfpStream.on('readable', () => {
-      this.streamBufferFlush();
-    });
-
-    this.wfpStream.on('end', () => {});
-  }
-
-  private splitFingerprints(chunkOfData:string): Array<string> {
-    const reg = /file=/g;
-    const blocksIndex = [...((chunkOfData).matchAll(reg))].map(x => x.index);
-    const fingerprints = [];
-    for (let i = 0; i < blocksIndex.length ; i++) {
-      const fingerprint = chunkOfData.substring(blocksIndex[i], blocksIndex[i + 1]);
-      fingerprints.push(fingerprint);
-    }
-    return fingerprints;
-  }
-
-
-}
-
-

package/src/sdk/tree/File.ts

@@ -1,22 +0,0 @@
-import { NodeType } from './Node';
-import Node from './Node';
-import { Filter } from './Filters/Filter';
-
-
-export default class File extends Node {
-  constructor(name: string, path: string) {
-    super(name, path);
-    this.type = NodeType.FILE;
-  }
-
-  public getNode(path: string): Node {
-    if (path === this.getPath()) return this;
-    return null;
-  }
-
-  public getFiles(f: Filter): Array<string> {
-    if (!f || !f.evaluate(this)) return [];
-    return [this.getPath()];
-  }
-
-}

package/src/sdk/tree/Filters/DecompressionFilter.ts

@@ -1,22 +0,0 @@
-import { Filter } from './Filter';
-import Node, { NodeType } from '../Node';
-import { DecompressionManager } from '../../Decompress/DecompressionManager';
-import path from 'path';
-
-export class DecompressionFilter extends Filter {
-
-  private supportedFileExtension: Array<string>;
-
-  public constructor(filterName: string) {
-    super();
-    this.supportedFileExtension = new DecompressionManager().getSupportedFormats();
-  }
-
-  //Returns true if you want the file
-  public evaluate(node: Node): boolean {
-    if(node.getType() == NodeType.FOLDER) return true;
-    if (this.supportedFileExtension.some(supportedFormat => node.getName().endsWith(supportedFormat))) return true;
-    return false;
-  }
-
-}

package/src/sdk/tree/Filters/DependencyFilter.ts

@@ -1,22 +0,0 @@
-import { Filter } from './Filter';
-import Node from '../Node'
-import { FilterList, IFilter} from '../../Filtering/Filtering';
-import {
-  defaultFilterForDependencies
-} from '../../Filtering/DefaultFilterForDependencies';
-
-export class DependencyFilter extends Filter {
-
-  private filter: FilterList;
-
-  public constructor(filterName: string) {
-    super();
-    this.filter = new FilterList();
-    this.filter.load(defaultFilterForDependencies)
-  }
-
-  public evaluate(node: Node): boolean {
-    return this.filter.include(node.getPath());
-  }
-
-}

package/src/sdk/tree/Filters/Filter.ts

@@ -1,5 +0,0 @@
-import Node from '../Node'
-
-export abstract class Filter {
-  public abstract evaluate(node: Node): boolean
-}

package/src/sdk/tree/Filters/ScanFilter.ts

@@ -1,22 +0,0 @@
-import { Filter } from './Filter';
-import Node from '../Node'
-import { FilterList } from '../../Filtering/Filtering';
-import {
-  defaultFilterForScanning
-} from '../../Filtering/DefaultFilterForScanning';
-
-export class ScanFilter extends Filter {
-
-  private filter: FilterList;
-
-  public constructor(filterName: string) {
-    super();
-    this.filter = new FilterList();
-    this.filter.load(defaultFilterForScanning)
-  }
-
-  public evaluate(node: Node): boolean {
-    return this.filter.include(node.getPath());
-  }
-
-}

package/src/sdk/tree/Folder.ts

@@ -1,57 +0,0 @@
-import { NodeType } from "./Node";
-import Node from './Node';
-import { Filter } from './Filters/Filter';
-
-export default class Folder extends Node {
-  private children: Node[];
-
-  constructor(path: string, label: string) {
-    super(path, label);
-    this.type = NodeType.FOLDER;
-    this.children = [];
-  }
-
-  public addChild(node: Node): void {
-    this.children.push(node);
-  }
-
-  public getChildren(): Node[] {
-    return this.children;
-  }
-
-  public getNode(path: string): Node {
-    if (!this.checkMyPath(path)) return null;
-    if (path === this.getPath()) return this;
-    for (const child of this.children) {
-      const node = child.getNode(path);
-      if (node !== null) return node;
-    }
-    return null;
-  }
-
-  // Returns true only if my path is contained in the path (parameter)
-  private checkMyPath(path: string): boolean {
-    if (!path.includes(this.getPath())) return false;
-    // Only if first filter is passed.
-    const myPathExploded = this.getPath().split('/');
-    const pathExploded = path.split('/');
-    for (let i = 0; i < myPathExploded.length; i += 1) {
-      if (myPathExploded[i] !== pathExploded[i]) return false;
-    }
-    return true;
-  }
-
-  public getFiles(f?: Filter): Array<string> {
-    if (!f || !f.evaluate(this)) return [];
-
-    const files: Array<string> = [];
-    this.children.forEach((child) => {
-      files.push(...child.getFiles(f));
-    });
-    return files;
-  }
-
-  public removeChildren(){
-    this.children = [];
-  }
-}

package/src/sdk/tree/Node.ts

@@ -1,39 +0,0 @@
-import { FilterList } from '../Filtering/Filtering';
-import { Filter } from './Filters/Filter';
-
-export default abstract class Node {
-  protected type: NodeType;
-
-  protected path: string; // Relative path to the folder or file
-
-  protected label: string;
-
-  protected action: string;
-
-  constructor(path: string, label: string) {
-    this.path = path;
-    this.label = label;
-  }
-
-  public abstract getNode(path: string): Node;
-
-  public abstract getFiles(f: Filter): Array<string>;
-
-  public getName(): string {
-    return this.label;
-  }
-
-  public getPath(): string {
-    return this.path;
-  }
-
-  public getType(): NodeType {
-    return this.type;
-  }
-
-}
-
-export enum NodeType {
-  FOLDER = 'FOLDER',
-  FILE = 'FILE',
-};

package/src/sdk/tree/Tree.ts

@@ -1,82 +0,0 @@
-import fs from 'fs';
-import pathLib from 'path';
-
-
-import { NodeType } from './Node';
-import Node from './Node';
-import File from './File';
-import Folder from './Folder';
-import { FilterList } from '../Filtering/Filtering';
-import { Filter } from './Filters/Filter';
-
-export class Tree {
-  private rootFolder: Folder;
-
-  private rootName: string;
-
-  private rootPath: string;
-
-  private filter: FilterList;
-
-  constructor(path: string) {
-    const pathParts = path.split(pathLib.sep);
-    this.rootName = pathLib.basename(path);
-    this.rootPath = path;
-    this.rootFolder = new Folder('/', this.rootName);
-  }
-
-  public build(): Node {
-    this.rootFolder.removeChildren();
-    this.buildRec(this.rootPath, this.rootFolder);
-    return this.rootFolder;
-  }
-
-  private buildRec(path: string, root: Folder): Node {
-    const dirEntries = fs
-      .readdirSync(path, { withFileTypes: true }) // Returns a list of files and folders
-      .sort(this.dirFirstFileAfter)
-      .filter((dirent: any) => !dirent.isSymbolicLink());
-
-    for (const dirEntry of dirEntries) {
-      const fullPath = `${path}/${dirEntry.name}`;
-      const relativePath = fullPath.replace(this.rootPath, '');
-      if (dirEntry.isDirectory()) {
-        const f: Folder = new Folder(fullPath, dirEntry.name);
-        const subTree = this.buildRec(`${path}/${dirEntry.name}`, f);
-        root.addChild(subTree);
-      } else root.addChild(new File(fullPath, dirEntry.name));
-    }
-    return root;
-  }
-
-  // This is a sorter that will sort folders before files in alphabetical order.
-  private dirFirstFileAfter(a: any, b: any) {
-    if (!a.isDirectory() && b.isDirectory()) return 1;
-    if (a.isDirectory() && !b.isDirectory()) return -1;
-    return 0;
-  }
-
-  public loadTree(data: any): void {
-    this.rootFolder = this.deserialize(data) as Folder;
-  }
-
-  private deserialize(data: any): Node {
-    if (data.type === NodeType.FILE) {
-      return Object.assign(Object.create(File.prototype), data);
-    }
-    const children = data.children.map((child: any) => this.deserialize(child));
-    return Object.assign(Object.create(Folder.prototype), { ...data, children });
-  }
-
-  public getRootFolder(): Folder {
-    return this.rootFolder;
-  }
-
-  public getRootPath(): string {
-    return this.rootPath;
-  }
-
-  public getFileList(f?: Filter): Array<string> {
-    return  this.rootFolder.getFiles(f);
-  }
-}

package/tests/scanoss.sh

@@ -1,56 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Print help info
-#
-help() {
-  echo "Usage: $0 [-h] -w winnowing.wfp -r result.json
-               [-r file to result.json]
-               [-w file to winnowing]"
-  exit 2
-}
-
-cmd_compare_wfp_result() {
-  result_json_path=$1
-  winnowing_path=$2
-  if [ ! -e "$result_json_path" ] || [ ! -e "$winnowing_path" ]; then
-    echo "Error: Please, specify a valid path"
-    exit 1
-  fi
-
-  diff <(sed -n -E 's/file=\w+,[[:digit:]]+,//p' "$winnowing_path" | sort) <(jq -r '.scanner | keys | .[]' "$result_json_path" | sort)
-
-  exit 1
-}
-
-
-#
-# Parse command options and take action
-#
-force=false
-while getopts ":r:w:,h" o; do
-  case "${o}" in
-  r)
-    r=${OPTARG}
-    ;;
-  w)
-    w=${OPTARG}
-    ;;
-  h)
-    help
-    ;;
-  *)
-    echo "Unknown option: $o"
-    help
-    ;;
-  esac
-done
-shift $((OPTIND - 1))
-
-if [ -z "${r}" ] || [ -z "${w}" ]; then
-  echo "Please specify a result json and a winnowing file"
-  help
-fi
-
-
-cmd_compare_wfp_result "$r" "$w"

package/tests/sdk/DataLayer/DataProviders/ComponentDataProvider.spec.ts

@@ -1,176 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-
-import { ComponentDataProvider } from '../../../../src/sdk/DataLayer/DataProviders/ComponentDataProvider';
-import { ComponentDataLayer } from '../../../../src/sdk/DataLayer/DataLayerTypes';
-
-const deepEqualInAnyOrder = require('deep-equal-in-any-order');
-const chai = require('chai');
-chai.use(deepEqualInAnyOrder);
-const { expect } = chai;
-
-
-describe('Suit test for DataProvider', () => {
-
-  it('Test ComponentDataLayer',  function () {
-    const result = JSON.parse(fs.readFileSync(path.join(__dirname, '/samples/results-with-dep.json'), 'utf-8'));
-    const componentDataProvider = new ComponentDataProvider(result.scanner, result.dependencies);
-    const componentData = componentDataProvider.getData();
-
-    const expectedOutput: ComponentDataLayer[] = [
-
-      //First component
-      {
-        key : "pkg:npm/url-parse",
-        purls: [
-          "pkg:npm/url-parse",
-          "pkg:github/unshiftio/url-parse",
-          "pkg:deb/debian/node-url-parse",
-          "pkg:maven/org.webjars.bowergithub.unshiftio/url-parse"
-        ],
-        name: "url-parse",
-        vendor: "Arnout Kazemier",
-        url: "https://www.npmjs.com/package/url-parse",
-        versions: [
-          {
-            version: "1.5.10",
-            licenses: ["MIT"],
-            copyrights: [
-              {
-                name: "Copyright (c) 2015 Unshift.io; Arnout Kazemier;  the Contributors.",
-                source: "license_file"
-              }
-            ],
-            cryptography: null,
-            quality: null
-          }
-        ],
-        health: null,
-      },
-
-      //Second component with multiple versions detected
-      {
-        key: "pkg:github/unshiftio/url-parse",
-        purls: [
-          "pkg:github/unshiftio/url-parse",
-          "pkg:deb/debian/node-url-parse",
-          "pkg:maven/org.webjars.bowergithub.unshiftio/url-parse",
-          "pkg:npm/url-parse"
-        ],
-        name: "url-parse",
-        vendor: "unshiftio",
-        url: "https://github.com/unshiftio/url-parse",
-        versions: [
-          {
-            version: "1.5.3",
-            licenses: ["MIT"],
-            copyrights: [
-              {
-                name: "Copyright (c) 2015 Unshift.io; Arnout Kazemier;  the Contributors.",
-                source: "license_file"
-              }
-            ],
-            cryptography: null,
-            quality: {
-              sum: 4,
-              scoreAvg: 4,
-              count: 1
-            }
-          },
-          {
-            version: "1.5.0",
-            licenses: ["MIT"],
-            copyrights: [
-              {
-                name: "Copyright (c) 2015 Unshift.io; Arnout Kazemier;  the Contributors.",
-                source: "license_file"
-              }
-            ],
-            cryptography: null,
-            quality: {
-              sum: 4,
-              scoreAvg: 4,
-              count: 1
-            }
-          },
-          {
-            version: "1.5.10",
-            licenses: ["MIT"],
-            copyrights: [
-              {
-                name: "Copyright (c) 2015 Unshift.io; Arnout Kazemier;  the Contributors.",
-                source: "license_file"
-              }
-            ],
-            cryptography: null,
-            quality: null
-          }
-        ],
-        health: null,
-      },{
-        key: "pkg:npm/querystringify",
-        purls: ["pkg:npm/querystringify"],
-        name: "querystringify",
-        url: null,
-        versions: [{
-          version: "2.2.0",
-          licenses: ["MIT"],
-          copyrights: null,
-          cryptography: null,
-          quality: null
-        }],
-        vendor: null,
-        health: null,
-      },{
-        key: "pkg:npm/requires-port",
-        purls: ["pkg:npm/requires-port"],
-        name: "requires-port",
-        url: null,
-        versions: [{
-          version: "1.0.0",
-          licenses: ["MIT"],
-          copyrights: null,
-          cryptography: null,
-          quality: null
-        }],
-        vendor: null,
-        health: null,
-      },
-      {
-        key: "pkg:npm/assume",
-        purls: ["pkg:npm/assume"],
-        name: "assume",
-        url: null,
-        versions: [{
-          version: "2.3.0",
-          licenses: ["MIT"],
-          copyrights: null,
-          cryptography: null,
-          quality: null
-        }],
-        health: null,
-        vendor: null,
-      },
-      {
-        key: "pkg:npm/browserify",
-        purls: ["pkg:npm/browserify"],
-        name: "browserify",
-        url: null,
-        versions: [{
-          version: "17.0.0",
-          licenses: ["MIT"],
-          copyrights: null,
-          cryptography: null,
-          quality: null
-        }],
-        vendor: null,
-        health: null,
-      }
-      ];
-
-    expect(componentData.component).to.deep.equalInAnyOrder(expectedOutput);
-  });
-
-});
-
-