scanoss 0.7.9 → 0.7.10

package/src/sdk/grpc/scanoss/api/scanning/v2/scanoss-scanning_grpc_pb.d.ts

@@ -1,25 +0,0 @@
-// GENERATED CODE -- DO NOT EDIT!
-
-// package: scanoss.api.scanning.v2
-// file: scanoss/api/scanning/v2/scanoss-scanning.proto
-
-import * as scanoss_api_scanning_v2_scanoss_scanning_pb from "../../../../scanoss/api/scanning/v2/scanoss-scanning_pb";
-import * as scanoss_api_common_v2_scanoss_common_pb from "../../../../scanoss/api/common/v2/scanoss-common_pb";
-import * as grpc from "@grpc/grpc-js";
-
-interface IScanningService extends grpc.ServiceDefinition<grpc.UntypedServiceImplementation> {
-  echo: grpc.MethodDefinition<scanoss_api_common_v2_scanoss_common_pb.EchoRequest, scanoss_api_common_v2_scanoss_common_pb.EchoResponse>;
-}
-
-export const ScanningService: IScanningService;
-
-export interface IScanningServer extends grpc.UntypedServiceImplementation {
-  echo: grpc.handleUnaryCall<scanoss_api_common_v2_scanoss_common_pb.EchoRequest, scanoss_api_common_v2_scanoss_common_pb.EchoResponse>;
-}
-
-export class ScanningClient extends grpc.Client {
-  constructor(address: string, credentials: grpc.ChannelCredentials, options?: object);
-  echo(argument: scanoss_api_common_v2_scanoss_common_pb.EchoRequest, callback: grpc.requestCallback<scanoss_api_common_v2_scanoss_common_pb.EchoResponse>): grpc.ClientUnaryCall;
-  echo(argument: scanoss_api_common_v2_scanoss_common_pb.EchoRequest, metadataOrOptions: grpc.Metadata | grpc.CallOptions | null, callback: grpc.requestCallback<scanoss_api_common_v2_scanoss_common_pb.EchoResponse>): grpc.ClientUnaryCall;
-  echo(argument: scanoss_api_common_v2_scanoss_common_pb.EchoRequest, metadata: grpc.Metadata | null, options: grpc.CallOptions | null, callback: grpc.requestCallback<scanoss_api_common_v2_scanoss_common_pb.EchoResponse>): grpc.ClientUnaryCall;
-}

package/src/sdk/grpc/scanoss/api/scanning/v2/scanoss-scanning_grpc_pb.js

@@ -1,73 +0,0 @@
-// GENERATED CODE -- DO NOT EDIT!
-
-// Original file comments:
-//
-// SPDX-License-Identifier: MIT
-//
-// Copyright (c) 2021, SCANOSS
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in
-// all copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-// THE SOFTWARE.
-//
-// **
-// Scanning definition details
-// *
-'use strict';
-var grpc = require('@grpc/grpc-js');
-var scanoss_api_common_v2_scanoss$common_pb = require('../../../../scanoss/api/common/v2/scanoss-common_pb.js');
-
-function serialize_scanoss_api_common_v2_EchoRequest(arg) {
-  if (!(arg instanceof scanoss_api_common_v2_scanoss$common_pb.EchoRequest)) {
-    throw new Error('Expected argument of type scanoss.api.common.v2.EchoRequest');
-  }
-  return Buffer.from(arg.serializeBinary());
-}
-
-function deserialize_scanoss_api_common_v2_EchoRequest(buffer_arg) {
-  return scanoss_api_common_v2_scanoss$common_pb.EchoRequest.deserializeBinary(new Uint8Array(buffer_arg));
-}
-
-function serialize_scanoss_api_common_v2_EchoResponse(arg) {
-  if (!(arg instanceof scanoss_api_common_v2_scanoss$common_pb.EchoResponse)) {
-    throw new Error('Expected argument of type scanoss.api.common.v2.EchoResponse');
-  }
-  return Buffer.from(arg.serializeBinary());
-}
-
-function deserialize_scanoss_api_common_v2_EchoResponse(buffer_arg) {
-  return scanoss_api_common_v2_scanoss$common_pb.EchoResponse.deserializeBinary(new Uint8Array(buffer_arg));
-}
-
-
-// Expose all of the SCANOSS Scanning RPCs here
-var ScanningService = exports.ScanningService = {
-  // Standard echo
-echo: {
-    path: '/scanoss.api.scanning.v2.Scanning/Echo',
-    requestStream: false,
-    responseStream: false,
-    requestType: scanoss_api_common_v2_scanoss$common_pb.EchoRequest,
-    responseType: scanoss_api_common_v2_scanoss$common_pb.EchoResponse,
-    requestSerialize: serialize_scanoss_api_common_v2_EchoRequest,
-    requestDeserialize: deserialize_scanoss_api_common_v2_EchoRequest,
-    responseSerialize: serialize_scanoss_api_common_v2_EchoResponse,
-    responseDeserialize: deserialize_scanoss_api_common_v2_EchoResponse,
-  },
-};
-
-exports.ScanningClient = grpc.makeGenericClientConstructor(ScanningService);

package/src/sdk/grpc/scanoss/api/scanning/v2/scanoss-scanning_pb.d.ts

@@ -1,6 +0,0 @@
-// package: scanoss.api.scanning.v2
-// file: scanoss/api/scanning/v2/scanoss-scanning.proto
-
-import * as jspb from "google-protobuf";
-import * as scanoss_api_common_v2_scanoss_common_pb from "../../../../scanoss/api/common/v2/scanoss-common_pb";
-

package/src/sdk/grpc/scanoss/api/scanning/v2/scanoss-scanning_pb.js

@@ -1,15 +0,0 @@
-/**
- * @fileoverview
- * @enhanceable
- * @suppress {messageConventions} JS Compiler reports an error if a variable or
- *     field starts with 'MSG_' and isn't a translatable message.
- * @public
- */
-// GENERATED CODE -- DO NOT EDIT!
-
-var jspb = require('google-protobuf');
-var goog = jspb;
-var global = Function('return this')();
-
-var scanoss_api_common_v2_scanoss$common_pb = require('../../../../scanoss/api/common/v2/scanoss-common_pb.js');
-goog.object.extend(exports, proto.scanoss.api.scanning.v2);

package/src/sdk/scanner/Dispatcher/DispatchableItem.ts

@@ -1,62 +0,0 @@
-import { FingerprintPackage } from '../WfpProvider/FingerprintPackage';
-import FormData from 'form-data';
-import { SbomMode } from '../ScannerTypes';
-
-export class DispatchableItem {
-  private readonly form: FormData;
-
-  private errorCounter: number;
-
-  private fingerprintPackage: FingerprintPackage;
-
-  private engineFlags: number;
-
-  private sbom: string;
-
-  private sbomMode: SbomMode;
-
-  constructor() {
-    this.errorCounter = 0;
-    this.form = new FormData();
-  }
-
-  public getForm(): FormData {
-    this.form.append('file', Buffer.from(this.fingerprintPackage.getContent()), 'data.wfp');
-    if(this.engineFlags) this.form.append('flags', this.engineFlags);
-
-    if(this.sbomMode && this.sbom) {
-      this.form.append('assets', this.sbom);
-      this.form.append('type', this.sbomMode);
-    }
-
-    return this.form;
-  }
-
-  public increaseErrorCounter() {
-    this.errorCounter += 1;
-  }
-
-  public getErrorCounter() {
-    return this.errorCounter;
-  }
-
-  public setFingerprintPackage(fingerprintPackage: FingerprintPackage) {
-    this.fingerprintPackage = fingerprintPackage;
-  }
-
-  public getFingerprintPackage(): FingerprintPackage {
-    return this.fingerprintPackage;
-  }
-
-  public setEngineFlags(engineFlags: number) {
-    this.engineFlags = engineFlags;
-  }
-
-  public setSbom(sbom: string, sbomMode: SbomMode) {
-    this.sbom = sbom;
-    this.sbomMode = sbomMode;
-  }
-
-
-
-}

package/src/sdk/scanner/Dispatcher/Dispatcher.ts

@@ -1,173 +0,0 @@
-/* eslint-disable guard-for-in */
-/* eslint-disable no-restricted-syntax */
-import EventEmitter from 'eventemitter3';
-import fetch from 'node-fetch';
-import PQueue from "p-queue";
-
-import { ScannerEvents } from "../ScannerTypes";
-import { DispatcherResponse } from "./DispatcherResponse";
-import { ScannerCfg } from "../ScannerCfg";
-import { GlobalControllerAborter } from "./GlobalControllerAborter";
-import { DispatchableItem } from './DispatchableItem';
-import { HttpsProxyAgent } from 'https-proxy-agent';
-import * as syswideCa from "syswide-cas";
-
-import { PackageJSON } from '../../../cli/bin/cli-bin';
-
-const MAX_CONCURRENT_REQUEST = 30;
-
-export class Dispatcher extends EventEmitter {
-  private scannerCfg: ScannerCfg;
-
-  private pQueue;
-
-  private globalAbortController: GlobalControllerAborter;
-
-  private queueMaxLimitReached: boolean;
-
-  private queueMinLimitReached: boolean;
-
-  private recoverableErrors;
-
-  private proxy: HttpsProxyAgent;
-
-  constructor(scannerCfg = new ScannerCfg()) {
-    super();
-    this.scannerCfg = scannerCfg;
-    if(this.scannerCfg.CONCURRENCY_LIMIT > MAX_CONCURRENT_REQUEST)
-      this.scannerCfg.CONCURRENCY_LIMIT = MAX_CONCURRENT_REQUEST;
-
-    this.init();
-  }
-
-  init() {
-
-    //Loads proxy from SDK config, if not from env variables, if not leave empty
-    this.proxy = null;
-    const proxy = this.scannerCfg.PROXY || process.env.https_proxy || process.env.HTTPS_PROXY || process.env.http_proxy  || process.env.HTTP_PROXY || '';
-    if (proxy) this.proxy = new HttpsProxyAgent(proxy)
-
-
-
-    //Loads certs stuff from SDK config
-    const ca_cert = this.scannerCfg.CA_CERT  || process.env.NODE_EXTRA_CA_CERTS
-    if (ca_cert) {
-      syswideCa.addCAs(ca_cert)
-    } else {
-      if (this.scannerCfg.IGNORE_CERT_ERRORS || proxy)
-        process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
-    }
-
-    this.pQueue = new PQueue({
-      concurrency: this.scannerCfg.CONCURRENCY_LIMIT,
-    });
-    this.pQueue.clear();
-
-    this.pQueue.on('idle', () => {
-      this.emit(ScannerEvents.DISPATCHER_FINISHED);
-    });
-
-    this.pQueue.on('next', () => {
-      if ((this.pQueue.size + this.pQueue.pending) < this.scannerCfg.DISPATCHER_QUEUE_SIZE_MIN_LIMIT && !this.queueMinLimitReached) {
-        this.emit(ScannerEvents.DISPATCHER_QUEUE_SIZE_MIN_LIMIT);
-        this.queueMinLimitReached = true;
-        this.queueMaxLimitReached = false;
-      }
-    });
-
-    this.queueMaxLimitReached = false;
-    this.queueMinLimitReached = true;
-
-    this.globalAbortController = new GlobalControllerAborter();
-
-    this.recoverableErrors = new Set();
-    this.recoverableErrors.add('ECONNRESET');
-    this.recoverableErrors.add('TIMEOUT');
-  }
-
-  stop() {
-    this.pQueue.clear();
-    this.pQueue.pause();
-    this.globalAbortController.abortAll();
-  }
-
-  public dispatchItem(item: DispatchableItem): void {
-    this.pQueue.add(() => this.dispatch(item));
-
-    if (
-      this.pQueue.size + this.pQueue.pending >= this.scannerCfg.DISPATCHER_QUEUE_SIZE_MAX_LIMIT &&
-      !this.queueMaxLimitReached
-    ) {
-      this.emit(ScannerEvents.DISPATCHER_QUEUE_SIZE_MAX_LIMIT);
-      this.queueMaxLimitReached = true;
-      this.queueMinLimitReached = false;
-    }
-  }
-
-  handleUnrecoverableError(error,disptItem) {
-    this.emit('error', error, disptItem);
-  }
-
-  emitNoDispatchedItem(disptItem) {
-    this.emit(ScannerEvents.DISPATCHER_LOG, `[ SCANNER ]: WFP content sended to many times. Some files won't be scanned`);
-    this.emit(ScannerEvents.DISPATCHER_ITEM_NO_DISPATCHED, disptItem);
-  }
-
-  errorHandler(error, disptItem) {
-    if (!this.globalAbortController.isAborting()) {
-      if (error.name === 'AbortError') error.name = 'TIMEOUT';
-      if (this.recoverableErrors.has(error.code) || this.recoverableErrors.has(error.name)) {
-        disptItem.increaseErrorCounter();
-        if (disptItem.getErrorCounter() >= this.scannerCfg.MAX_RETRIES_FOR_RECOVERABLES_ERRORS) {
-          this.emitNoDispatchedItem(disptItem);
-          if (this.scannerCfg.ABORT_ON_MAX_RETRIES) this.handleUnrecoverableError(error, disptItem);
-          return;
-        }
-        this.emit(ScannerEvents.DISPATCHER_LOG,`[ SCANNER ]: Recoverable error happened sending WFP content to server. Reason: ${error.code || error.name}`);
-        this.dispatchItem(disptItem);
-        return;
-      }
-      this.handleUnrecoverableError(error, disptItem);
-    }
-  }
-
-  async dispatch(item: DispatchableItem) {
-    const timeoutController = this.globalAbortController.getAbortController();
-    const timeoutId = setTimeout(() => timeoutController.abort(), this.scannerCfg.TIMEOUT);
-    try {
-      this.emit(ScannerEvents.DISPATCHER_WFP_SENDED);
-      const response = await fetch(this.scannerCfg.API_URL, {
-        agent: this.proxy,
-        method: 'post',
-        body: item.getForm(),
-        headers: {  'User-Agent': this.scannerCfg.CLIENT_TIMESTAMP ? this.scannerCfg.CLIENT_TIMESTAMP : `scanoss-js/v${PackageJSON.version}`,
-                    'X-Session': this.scannerCfg.API_KEY,
-        },
-        signal: timeoutController.signal,
-      });
-
-      clearTimeout(timeoutId);
-      this.globalAbortController.removeAbortController(timeoutController);
-
-      if (!response.ok) {
-        const msg = await response.text();
-        const err = new Error(msg);
-        err["code"] = response.status;
-        err.name = ScannerEvents.ERROR_SERVER_SIDE;
-        throw err;
-      }
-
-      const dataAsText = await response.text();
-      const dataAsObj = JSON.parse(dataAsText);
-
-      const dispatcherResponse = new DispatcherResponse(dataAsObj, item.getFingerprintPackage().getContent());
-      this.emit(ScannerEvents.DISPATCHER_NEW_DATA, dispatcherResponse);
-      return Promise.resolve();
-    } catch (e) {
-        clearTimeout(timeoutId);
-        this.globalAbortController.removeAbortController(timeoutController);
-        this.errorHandler(e, item);
-        return Promise.resolve();
-    }
-  }
-}

package/src/sdk/scanner/Dispatcher/DispatcherResponse.ts

@@ -1,40 +0,0 @@
-export class DispatcherResponse {
-  serverResponse: any;
-
-  wfpContent: string;
-
-  filesScanned: Array<string>;
-
-  constructor(serverResponse, wfpContent) {
-    this.serverResponse = serverResponse;
-    this.wfpContent = wfpContent;
-    this.filesScanned = Object.keys(this.serverResponse);
-    // this.verifyResponse();
-  }
-
-  getServerResponse() {
-    return this.serverResponse;
-  }
-
-  getWfpContent() {
-    return this.wfpContent;
-  }
-
-  matchRegex(str, re = /file=/g) {
-    return ((str || '').match(re) || []).length;
-  }
-
-  verifyResponse() {
-    const wfpNumFiles = this.matchRegex(this.wfpContent, /file=/g);
-    const serverResponseNumFiles = Object.keys(this.serverResponse).length;
-    if (wfpNumFiles !== serverResponseNumFiles) throw new Error(`The numbers of files in the wfp sended does not match with the server response`);
-  }
-
-  getFilesScanned() {
-    return this.filesScanned;
-  }
-
-  getNumberOfFilesScanned() {
-    return this.filesScanned.length;
-  }
-}

package/src/sdk/scanner/Dispatcher/GlobalControllerAborter.ts

@@ -1,33 +0,0 @@
-
-import AbortController from 'abort-controller';
-
-export class GlobalControllerAborter {
-  private abortControllerList: Array<AbortController>;
-
-  private abortFlag: Boolean;
-
-  constructor() {
-    this.abortControllerList = [];
-    this.abortFlag = false;
-  }
-
-  abortAll() {
-    this.abortFlag = true;
-    for (const c of this.abortControllerList) c.abort();
-  }
-
-  isAborting() {
-    return this.abortFlag;
-  }
-
-  getAbortController() {
-    const c = new AbortController();
-    this.abortControllerList.push(c);
-    return c;
-  }
-
-  removeAbortController(c) {
-    const index = this.abortControllerList.findIndex((controller) => controller === c);
-    if (index > -1) this.abortControllerList.splice(index, 1);
-  }
-}

package/src/sdk/scanner/Fingerprint.ts

@@ -1,82 +0,0 @@
-import EventEmitter from 'eventemitter3';
-import os from 'os';
-import fs from 'fs';
-import { IWfpProviderInput, WfpProvider } from './WfpProvider/WfpProvider';
-import { WfpCalculator } from './WfpProvider/WfpCalculator/WfpCalculator';
-import path from 'path';
-import { ScannerEvents } from './ScannerTypes';
-import { FingerprintPackage } from './WfpProvider/FingerprintPackage';
-import { ScannerCfg } from './ScannerCfg';
-
-export class Fingerprint extends EventEmitter {
-  private wfpCalculator: WfpProvider;
-  private fingerprintPath: string;
-  private finishPromise: Promise<void>;
-  private finishPromiseResolve: (value?: (void | PromiseLike<void>)) => void;
-  private finishPromiseReject: (value?: (void | PromiseLike<void>)) => void;
-  private reportStatusAfter: number;
-  private queue: Array<IWfpProviderInput>;
-  constructor(reportStatusAfter = 10) {
-    super();
-
-    this.reportStatusAfter = reportStatusAfter;
-    this.finishPromise = new Promise((resolve, reject) =>{
-      this.finishPromiseResolve = resolve;
-      this.finishPromiseReject = reject;
-    });
-
-    this.configureWfpCalculator();
-  }
-
-
-
-  private configureWfpCalculator() {
-
-    // TODO: Maybe the fingerprints shouldn't be truncated. Determine this optimal value.
-    const cfg = new ScannerCfg();
-    cfg.WFP_FILE_MAX_SIZE = 10 * 1024 *1024;
-    cfg.WINNOWING_REPORT_STATUS_AFTER_X = this.reportStatusAfter;
-    this.wfpCalculator = new WfpCalculator(cfg);
-
-
-    this.wfpCalculator.on(ScannerEvents.WINNOWING_NEW_CONTENT, (fingerprintPackage: FingerprintPackage) => {
-      fs.appendFileSync(this.fingerprintPath, fingerprintPackage.getContent());
-    });
-
-    this.wfpCalculator.on(ScannerEvents.WINNOWING_STATUS, (filesProcessed: number) => {
-      this.emit(ScannerEvents.WINNOWING_STATUS, filesProcessed);
-    });
-
-
-  }
-
-  public async start(jobs: Array<IWfpProviderInput>): Promise<void> {
-    //If winnowing.wfp path is not set, set to a tmp folder
-    if (!this.fingerprintPath) this.fingerprintPath = path.join(os.tmpdir(), "fingerprint-" + new Date().getTime().toString(), 'winnowing.wfp');
-
-    //If folder does not exist, create
-    const pathFolder = path.dirname(this.fingerprintPath);
-    if(!fs.existsSync(pathFolder)) fs.mkdirSync(pathFolder, { recursive: true });
-
-    this.queue = jobs;
-
-    for await (const job of this.queue) {
-      this.configureWfpCalculator();
-      await this.wfpCalculator.start(job);
-    }
-
-    this.emit(ScannerEvents.WINNOWING_FINISHED);
-  }
-
-  public abort() {
-    this.wfpCalculator.stop();
-    this.wfpCalculator=null;
-  }
-
-  public setFingerprintPath(fingerprintsFilePath: string) {
-    this.fingerprintPath=fingerprintsFilePath;
-  }
-
-
-
-}

package/src/sdk/scanner/Scannable/ScannableItem.ts

@@ -1,43 +0,0 @@
-import { WinnowingMode } from "../ScannerTypes";
-
-
-export class ScannableItem {
-  private contentSource: string;
-
-  private content: Buffer;
-
-  private winnowingMode: WinnowingMode;
-
-  private fingerprint: any;
-
-  private maxSizeWfp: any;
-
-  constructor(content: Buffer, contentSource: string, winnowingMode: any, maxSizeWfp: number) {
-    this.contentSource = contentSource;
-    this.content = content;
-    this.winnowingMode = winnowingMode;
-    this.maxSizeWfp = maxSizeWfp;
-  }
-
-  public getContent() {
-    return this.content;
-  }
-
-  public getContentSource() {
-    return this.contentSource;
-  }
-
-  public getFingerprint() {
-    return this.fingerprint;
-  }
-
-  public getWinnowingMode(): WinnowingMode {
-    return this.winnowingMode;
-  }
-
-  public getMaxSizeWfp() {
-    return this.maxSizeWfp;
-  }
-
-
-}