scanoss 0.7.9 → 0.7.11

package/src/cli/bin/cli-bin.ts

@@ -1,81 +0,0 @@
-#!/usr/bin/env node
-
-import { program } from 'commander';
-import { depHandler } from '../commands/dep';
-import { wfpHandler } from '../commands/wfp';
-import { scanHandler } from '../commands/scan';
-import path from 'path';
-import fs from 'fs';
-
-
-export const PackageJSONPath = path.join(__dirname,"../../../../package.json");
-export const PackageJSON = JSON.parse(fs.readFileSync(PackageJSONPath, 'utf-8'));
-
-function CLIErrorHandler(e: Error) {
-  console.error(' ');
-  console.error(e);
-  process.exit(1);
-}
-
-async function main() {
-  program
-    .version(PackageJSON.version)
-    .description('The SCANOSS JS package provides a simple, easy to consume module for interacting with SCANOSS APIs/Engine.')
-
-  program
-    .command('scan <source>')
-    .description('Scan a folder/file')
-    .option('-w, --wfp', 'Scan a .wfp file instead of a folder')
-    .option('-H, --hpsm', 'Scan using winnowing high precision matching')
-    .option('-x, --extract', 'Extract compressed files before launch scan in folder <<zip_name>>-unzipped')
-    .option('   --extract-overwrite', 'Overwrite folder when decompressing if exists')
-    .option('   --extract-deep <number>', 'Sets uncompress recursion level')
-    .option('   --extract-suffix <suffix>', 'Sets suffix for the folder name')
-    .option('-c, --concurrency <number>', 'Number of concurrent connections to use while scanning (optional -default 10)')
-    .option('-n, --ignore <ignore>',  'Ignore components specified in the SBOM file')
-    .option('-o, --output <filename>', 'Output result file name (optional - default stdout)')
-    .option('-f, --format <format>', 'Result output format. {JSON, HTML} Default: JSON')
-    .option('-F, --flags <flags>', 'Scanning engine flags (1: disable snippet matching, 2 enable snippet ids, 4: disable dependencies, 8: disable licenses, 16: disable copyrights,32: disable vulnerabilities, 64: disable quality, 128: disable cryptography,256: disable best match, 512: Report identified files)')
-    .option('-P, --post-size <postsize>', 'Number of kilobytes to limit the post to while scanning (optional - default 64)')
-    .option('-R, --max-retry <retry>', 'Max number of retries for each POST (optional -default 5)')
-    .option('-M, --timeout <timeout>', 'Timeout (in seconds) for API communication (optional -default 120)')
-    .option('-D, --dependencies', 'Add dependency scanning')
-    .option('-a, --apiurl <apiurl>', 'SCANOSS API URL (optional - default: https://osskb.org/api/scan/direct)')
-    .option('-a, --api2url <api2url>', 'SCANOSS gRPC API 2.0 URL (optional - default: scanoss.com)')
-    .option('-k, --key <key>', 'SCANOSS API Key token (optional - not required for default OSSKB URL)')
-    .option('--ignore-cert-errors', 'Ignore self signed certificate errors')
-    .option('--ca-cert <cert>', 'Specify a path for a cert used in SSL/TLS connection')
-    .option('--proxy <proxy>', 'Use proxy')
-    .option('-v, --verbose', 'Makes scan operation verbose')
-    .action((source, options) => {scanHandler(source, options).catch((e) => {CLIErrorHandler(e)})})
-    .addHelpText('after', `
-  Examples:
-    $ scanoss-js scan -o scan-output.json <source-folder>`
-    );
-
-    program
-    .command('dep <source>')
-    .description('Scan for dependencies')
-    .option('-o, --output <filename>', 'Output result file name (optional - default stdout)')
-    .option('-a, --grpc-host <host>', 'SCANOSS GRPC HOST (optional - default: scanoss.com)')
-    .option('-p, --grpc-port <port>', 'SCANOSS GRPC PORT  (optional - default: 443)')
-    .action((source, options) => {depHandler(source, options).catch((e) => {CLIErrorHandler(e)})})
-
-    program
-    .command('wfp <source>')
-    .description('Generates fingerprints for a folder/file')
-    .option('-H, --hpsm', 'Scan using winnowing high precision matching')
-    .option('-o, --output <filename>', 'Output result file name (optional - default stdout)')
-    .option('-p, --block-size <size>', 'Maximum size in Kb for each fingerprint block (optional - default 64Kb)')
-    .action((source, options) => {wfpHandler(source, options).catch((e) => {CLIErrorHandler(e)})})
-
-    await program.parseAsync(process.argv);
-}
-
-
-try {
-  main();
-} catch (e) {
-  console.error(e);
-  process.exit(1);
-}

package/src/cli/commands/dep.ts

@@ -1,37 +0,0 @@
-import fs from "fs";
-import { DependencyScanner } from "../../sdk/Dependencies/DependencyScanner";
-import { DependencyScannerCfg } from "../../sdk/Dependencies/DependencyScannerCfg";
-import { Tree } from "../../sdk/tree/Tree";
-import { DependencyFilter } from '../../sdk/tree/Filters/DependencyFilter';
-import { isFolder } from "./helpers";
-
-
-export async function depHandler(rootPath: string, options: any): Promise<void> {
-
-  rootPath = rootPath.replace(/\/$/, '');  // Remove trailing slash if exists
-  rootPath = rootPath.replace(/^\./, process.env.PWD);  // Convert relative path to absolute path.
-  const pathIsFolder = await isFolder(rootPath);
-  const dependencyScannerCfg = new DependencyScannerCfg();
-  if(options.grpcHost) dependencyScannerCfg.DEFAULT_GRPC_HOST = options.grpcHost;
-  if(options.grpcPort) dependencyScannerCfg.DEFAULT_GRPC_PORT = options.grpcPort;
-
-  const dependencyScanner = new DependencyScanner(dependencyScannerCfg);
-
-  let fileList: Array<string> = [];
-  fileList.push(rootPath);
-
-  if (pathIsFolder) {
-    const tree = new Tree(rootPath);
-    tree.build();
-    fileList = tree.getFileList(new DependencyFilter(""));
-  }
-
-  const results = await dependencyScanner.scan(fileList);
-
-  if(options.output) {
-    fs.promises.writeFile(options.output, JSON.stringify(results, null, 2));
-  } else {
-    console.log(JSON.stringify(results, null, 2));
-  }
-
-}

package/src/cli/commands/helpers.ts

@@ -1,19 +0,0 @@
-import pathLib from 'path';
-import fs from 'fs';
-
-// Async function that verify if a path is a folder. If the path is not valid the promise will be rejected
-export const isFolder = (path: string): Promise<boolean> => {
-  return new Promise((resolve, reject) => {
-    fs.stat(path, (err, stats) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(stats.isDirectory());
-      }
-    });
-  });
-}
-
-export function getProjectNameFromPath(path: string): string {
-  return pathLib.basename(path,pathLib.extname(path))
-}

package/src/cli/commands/scan.ts

@@ -1,173 +0,0 @@
-import os from 'os';
-import fs from 'fs';
-
-import { Scanner } from '../../sdk/scanner/Scanner';
-import {
-  SbomMode,
-  ScannerEvents,
-  ScannerInput, ScannerComponent, ScannerResults,
-  WinnowingMode
-} from '../../sdk/scanner/ScannerTypes';
-import { ScannerCfg } from '../../sdk/scanner/ScannerCfg';
-import { Tree } from '../../sdk/tree/Tree';
-import cliProgress from 'cli-progress';
-import { DispatcherResponse } from '../../sdk/scanner/Dispatcher/DispatcherResponse';
-import { getProjectNameFromPath, isFolder } from './helpers';
-
-
-import { DependencyScannerCfg } from '../../sdk/Dependencies/DependencyScannerCfg';
-import { DependencyScanner } from '../../sdk/Dependencies/DependencyScanner';
-import { IDependencyResponse } from '../../sdk/Dependencies/DependencyTypes';
-import { ScanFilter } from '../../sdk/tree/Filters/ScanFilter';
-import { DependencyFilter } from '../../sdk/tree/Filters/DependencyFilter';
-import { Report } from '../../sdk/Report/Report';
-import { DataProviderManager } from '../../sdk/DataLayer/DataProviderManager';
-import {
-  ComponentDataProvider
-} from '../../sdk/DataLayer/DataProviders/ComponentDataProvider';
-import {
-  DependencyDataProvider
-} from '../../sdk/DataLayer/DataProviders/DependencyDataProvider';
-import {
-  LicenseDataProvider
-} from '../../sdk/DataLayer/DataProviders/LicenseDataProvider';
-import {
-  SummaryDataProvider
-} from '../../sdk/DataLayer/DataProviders/SummaryDataProvider';
-import {
-  DecompressionFilter
-} from '../../sdk/tree/Filters/DecompressionFilter';
-import {
-  DecompressionManager
-} from '../../sdk/Decompress/DecompressionManager';
-
-
-export async function scanHandler(rootPath: string, options: any): Promise<void> {
-
-  rootPath = rootPath.replace(/\/$/, '');  // Remove trailing slash if exists
-  rootPath = rootPath.replace(/^\./, process.env.PWD);  // Convert relative path to absolute path.
-  const pathIsFolder = await isFolder(rootPath);
-
-
-  const projectName = getProjectNameFromPath(rootPath)
-
-  // Create dependency scanner and set parameters
-  const dependencyScannerCfg = new DependencyScannerCfg();
-  if (options.api2url) dependencyScannerCfg.DEFAULT_GRPC_HOST = options.api2url;
-  const dependencyScanner = new DependencyScanner(dependencyScannerCfg);
-  let dependencyInput: Array<string> = [];
-
-
-  // Create scanner and set connections parameters
-  const scannerCfg = new ScannerCfg();
-  if(options.concurrency) scannerCfg.CONCURRENCY_LIMIT = parseInt(options.concurrency);
-  if(options.postSize) scannerCfg.WFP_FILE_MAX_SIZE = parseInt(options.postSize) * 1024;
-  if(options.apiurl) scannerCfg.API_URL = options.apiurl;
-  if(options.key) scannerCfg.API_KEY = options.key;
-  if(options.timeout) scannerCfg.TIMEOUT = options.timeout * 1000;
-  if(options.maxRetry) scannerCfg.MAX_RETRIES_FOR_RECOVERABLES_ERRORS = options.maxRetry;
-  if(options.proxy) scannerCfg.PROXY = options.proxy;
-  if(options.caCert) scannerCfg.CA_CERT = options.caCert;
-  if(options.ignoreCertErrors) scannerCfg.IGNORE_CERT_ERRORS=true;
-  const scanner = new Scanner(scannerCfg);
-
-  let scannerInput: ScannerInput = {fileList: []};
-  scannerInput.folderRoot = rootPath + '/'; // This will remove the project root path from the results.
-  if(options.flags) scannerInput.engineFlags = options.flags;
-
-
-
-  if(!options.wfp) {
-    if(pathIsFolder) {
-      console.error('Reading directory...  ');
-      const tree = new Tree(rootPath);
-      tree.build();
-
-      if(options.extract) {
-        const archives = tree.getFileList(new DecompressionFilter(""));
-        console.error("Searching archives files...")
-        if(archives.length) {
-          console.error("Extracting archives...")
-          const decompressionManager = new DecompressionManager(options.extractDeep,options.extractSuffix,options.extractOverwrite);
-          await decompressionManager.decompress(archives);
-          console.error("Reindexing files...")
-          tree.build();
-        } else console.error("No archives found.");
-      }
-
-      scannerInput.fileList = tree.getFileList(new ScanFilter(""));
-      dependencyInput = tree.getFileList(new DependencyFilter(""));
-
-
-    } else {
-      scannerInput.fileList = [rootPath];
-      dependencyInput = [rootPath];
-    }
-  } else {
-    const winnowing = fs.readFileSync(rootPath, {encoding: 'utf-8'});
-    scannerInput.fileList.length = [...winnowing.matchAll(/file=/g)].length;
-  }
-
-  if (!options.verbose) {
-    const optBar1 = { format: 'Scan Progress: [{bar}] {percentage}% | Scanned {value} files of {total}' };
-    const bar1 = new cliProgress.SingleBar(optBar1, cliProgress.Presets.shades_classic);
-    bar1.start(scannerInput.fileList.length, 0);
-
-    scanner.on(ScannerEvents.DISPATCHER_NEW_DATA, (dispResp: DispatcherResponse) => {
-      bar1.increment(dispResp.getFilesScanned().length);
-    });
-
-    scanner.on(ScannerEvents.SCAN_DONE, async (resultPath) => {bar1.stop();});
-  } else {
-    scanner.on(ScannerEvents.SCANNER_LOG, (logText) => console.error(logText));
-  }
-
-  if (options.wfp) scannerInput.wfpPath = rootPath;
-  if (options.hpsm) scannerInput.winnowingMode = WinnowingMode.FULL_WINNOWING_HPSM
-
-  if (options.ignore) {
-    scannerInput.sbom = fs.readFileSync(options.ignore, 'utf-8');
-    scannerInput.sbomMode = SbomMode.SBOM_IGNORE
-  }
-
-
-
-  // Dependency scanner
-  let pDependencyScanner = Promise.resolve(<IDependencyResponse>{});
-  if (options.dependencies) {
-    pDependencyScanner = dependencyScanner.scan(dependencyInput);
-  }
-
-  //Launch parallel scanners
-  const pScanner = scanner.scan([scannerInput]);
-
-  const [scannerResultPath, depResults] = await Promise.all([pScanner, pDependencyScanner])
-  const scannerResults = JSON.parse(await fs.promises.readFile(scannerResultPath, 'utf-8'));
-
-  //TODO Unify results.json and dependency.json. What happens with result.json that includes dependencies?
-  const scannersResults = {
-    scanner: scannerResults as ScannerResults,
-    ...(options.dependencies && {dependencies: depResults})
-  };
-
-  let scannerResultsString = JSON.stringify(scannersResults, null, 2);
-
-  if (options.format && options.format.toLowerCase() === "html") {
-
-    const dataProviderManager = new DataProviderManager();
-    dataProviderManager.addDataProvider(new ComponentDataProvider(scannersResults.scanner, scannersResults.dependencies))
-    dataProviderManager.addDataProvider(new DependencyDataProvider(scannersResults.dependencies))
-    dataProviderManager.addDataProvider(new LicenseDataProvider(scannersResults.scanner, scannersResults.dependencies));
-    dataProviderManager.addDataProvider(new SummaryDataProvider(projectName, new Date(), scannersResults.scanner));
-    const report = new Report(dataProviderManager);
-    scannerResultsString = await report.getHTML();
-
-  }
-
-  if(options.output)
-    await fs.promises.writeFile(options.output, scannerResultsString)
-  else
-    console.log(scannerResultsString);
-}
-
-

package/src/cli/commands/wfp.ts

@@ -1,60 +0,0 @@
-import fs from 'fs';
-
-import cliProgress from 'cli-progress';
-
-import { isFolder } from './helpers';
-import { ScannerEvents, WinnowingMode } from '../../sdk/scanner/ScannerTypes';
-import { IWfpProviderInput } from '../../sdk/scanner/WfpProvider/WfpProvider';
-import { WfpCalculator } from '../../sdk/scanner/WfpProvider/WfpCalculator/WfpCalculator';
-import { FingerprintPackage } from '../../sdk/scanner/WfpProvider/FingerprintPackage';
-import { Tree } from '../../sdk/tree/Tree';
-import { DependencyFilter } from '../../sdk/tree/Filters/DependencyFilter';
-
-
-export async function wfpHandler(rootPath: string, options: any): Promise<void> {
-
-  rootPath = rootPath.replace(/\/$/, '');  // Remove trailing slash if exists
-  rootPath = rootPath.replace(/^\./, process.env.PWD);  // Convert relative path to absolute path.
-  const pathIsFolder = await isFolder(rootPath);
-  const wfpCalculator = new WfpCalculator();
-
-  let filesToFingerprint: string[] = [];
-  if (pathIsFolder) {
-    const tree = new Tree(rootPath);
-    tree.build();
-    filesToFingerprint = tree.getFileList(new DependencyFilter(""));
-  } else {
-    filesToFingerprint.push(rootPath)
-  }
-
-
-  const optBar1 = { format: 'Fingerprinting Progress: [{bar}] {percentage}% | Fingerprinted {value} files of {total}' };
-  const bar1 = new cliProgress.SingleBar(optBar1, cliProgress.Presets.shades_classic);
-  bar1.start(filesToFingerprint.length, 0);
-
-  let fingerprints = '';
-  wfpCalculator.on(ScannerEvents.WINNOWING_NEW_CONTENT, (fingerprintPackage: FingerprintPackage) => {
-    bar1.increment(fingerprintPackage.getNumberFilesFingerprinted());
-    fingerprints = fingerprints.concat( fingerprintPackage.getContent() );
-  });
-
-  if (options.verbose)
-    wfpCalculator.on(ScannerEvents.WINNOWER_LOG, (log: string) => {
-      console.error(log);
-    });
-
-  wfpCalculator.on(ScannerEvents.WINNOWING_FINISHED, () => {
-    bar1.stop();
-    if(options.output) {
-      fs.writeFileSync(options.output, fingerprints);
-    } else {
-      console.log(fingerprints);
-    }
-  });
-
-  const wfpInput: IWfpProviderInput = {fileList: filesToFingerprint, folderRoot: rootPath}
-  if(options.hpsm) wfpInput.winnowingMode = WinnowingMode.FULL_WINNOWING_HPSM;
-  wfpCalculator.start(wfpInput);
-
-
-}

package/src/index.ts

@@ -1,31 +0,0 @@
-// *** Code scanner exports *** //
-export * from './sdk/scanner/Scanner';
-export * from './sdk/scanner/ScannerTypes';
-export * from './sdk/scanner/ScannerCfg';
-export * from './sdk/scanner/WfpProvider/WfpCalculator/WfpCalculator';
-
-// *** Fingerprint exports *** //
-export * from './sdk/scanner/Fingerprint';
-export {IWfpProviderInput} from './sdk/scanner/WfpProvider/WfpProvider';
-
-// *** Dependency scanner exports *** //
-export * from './sdk/Dependencies/DependencyTypes';
-export * from './sdk/Dependencies/DependencyScannerCfg';
-export * from './sdk/Dependencies/DependencyScanner';
-export * from './sdk/Dependencies/LocalDependency/LocalDependency'
-
-// *** Data layers export *** //
-export * from './sdk/DataLayer/DataLayerTypes';
-export * from './sdk/DataLayer/DataProviderManager';
-export * from './sdk/DataLayer/DataProviders/LicenseDataProvider';
-export * from './sdk/DataLayer/DataProviders/SummaryDataProvider';
-export * from './sdk/DataLayer/DataProviders/DependencyDataProvider';
-export * from './sdk/DataLayer/DataProviders/ComponentDataProvider';
-
-// *** Unzip *** //
-export * from './sdk/Decompress/DecompressionManager';
-
-
-// *** FileCount *** //
-export * from './sdk/FileCount/FileCount';
-export {IDirSummary} from './sdk/FileCount/Interfaces';

package/src/sdk/DataLayer/DataLayerTypes.ts

@@ -1,129 +0,0 @@
-/*************  Component interface definition  *************/
-export interface ComponentDataLayer {
-  key: string; // purl[0]
-  purls: string[];
-  name: string;
-  vendor: string;
-  url: string;
-  health: Health;
-  versions: Version[];
-}
-
-export interface Version {
-  version: string;
-  licenses:  string[]
-  copyrights: Copyright[]
-  cryptography: Cryptography[];
-  quality: Quality;
-}
-
-export interface Copyright {
-  name: string;
-  source: string;
-};
-
-export interface Cryptography {
-  algorithm: string;
-  strength: string;
-};
-
-export interface Quality {
-  scoreAvg: number;
-  count: number;
-  sum: number; //TODO remove
-};
-
-export interface Health {
-  creation_date: string;
-  issues: number;
-  last_push: string;
-  last_update: string;
-  watchers: number;
-  country: string;
-  stars: number;
-  forks: number;
-}
-/*************  Component interface definition  *************/
-
-
-/*************  Dependency interface definition  *************/
-export interface DependencyDataLayer {
-  file: string;
-  dependencies: Dependency[];
-}
-
-export interface License {
-  name: string;
-  spdxid: string;
-}
-
-export interface Dependency {
-  purl: string;
-  component: string;
-  version: string;
-  licenses: License[]
-}
-/*************  Dependency interface definition  *************/
-
-/*************  Vulnerability interface definition  *************/
-export interface Vulnerability {
-  id: string;
-  cve: string;
-  url: string;
-  summary: string;
-  severity: string;
-  published: string;
-  modified: string;
-  source: string;
-}
-
-export interface VulnerabilityDataLayer {
-  purl: string;
-  vulnerability: Vulnerability[];
-}
-/*************  Vulnerability interface definition  *************/
-
-
-/*************  License interface definition  *************/
-export interface LicenseDataLayer {
-  label: string;
-  value: number;  //Number of licenses found
-  components: Array<LicenseComponent>;
-  incompatibleWith: Array<string>;
-  hasIncompatibles: Array<string>;
-  copyleft:boolean;
-}
-
-
-export interface LicenseComponent {
-  purl: string;
-  name: string;
-  versions: Array<string>;
-  url:string;
-  vendor:string;
-}
-/*************  License interface definition  *************/
-
-export interface SummaryDataLayer {
-  projectName: string;
-  timestamp: Date;
-  matchedFiles: number;
-  noMatchFiles: number;
-  totalFiles: number;
-}
-
-// Each layer is created to group by differents criteria.
-export interface IDataLayers {
-  licenses: LicenseDataLayer[];
-  component: ComponentDataLayer[];
-  dependencies: DependencyDataLayer[];
-  vulnerabilities: VulnerabilityDataLayer[];
-  summary: SummaryDataLayer;
-}
-
-
-export interface DataProvider {
-  getData(): IDataLayers;
-  getLayerName(): string;
-}
-

package/src/sdk/DataLayer/DataProviderManager.ts

@@ -1,27 +0,0 @@
-import { DataProvider, IDataLayers } from './DataLayerTypes';
-
-export class DataProviderManager {
-  private dataLayersProviders: Array<DataProvider>;
-
-  constructor() {
-    this.dataLayersProviders = [];
-  }
-
-  public addDataProvider(l: DataProvider) {
-    this.dataLayersProviders.push(l)
-  }
-
-  public generateData(): IDataLayers {
-
-    let dataLayer: IDataLayers = {
-      component: null,
-      dependencies: null,
-      vulnerabilities: null,
-      summary: null,
-      licenses: null
-    };
-
-    for (const layer of this.dataLayersProviders) Object.assign(dataLayer, layer.getData());
-    return dataLayer;
-  }
-}