scanoss 0.7.5 → 0.7.9

package/src/sdk/Filtering/Filtering.ts

@@ -0,0 +1,273 @@
+import fs from 'fs';
+import * as fpath from 'path';
+import { isBinaryFileSync } from 'isbinaryfile';
+
+
+
+export interface IFilter {
+  type: FilterListType.ALLOW | FilterListType.BANNED;
+  filters: Array<{
+    ftype: FilterType;
+    scope: FilterScope;
+    condition: string;
+    value: string;
+  }>;
+}
+
+export enum FilterType {
+  NAME = 'NAME',
+  CONTENT = 'CONTENT',
+  EXTENSION = 'EXTENSION',
+  SIZE = 'SIZE',
+  DATE = 'DATE',
+  NULL = 'NONE'
+};
+
+export enum FilterListType {
+  BANNED = 'BANNED',
+  ALLOW = 'ALLOW',
+};
+
+export enum FilterScope {
+  FOLDER = 'FOLDER',
+  FILE = 'FILE',
+  ALL = 'ALL',
+}
+
+class AbstractFilter {
+  condition: string;
+
+  value: string;
+
+  ftype: FilterType;
+
+  scope: FilterScope;
+
+  constructor(condition: string, value: string) {
+    this.condition = condition;
+    this.value = value;
+    this.ftype = FilterType.NULL;
+    this.scope = FilterScope.ALL;
+  }
+
+  evaluate(path: string): boolean {
+    return true;
+  }
+}
+
+export class NameFilter extends AbstractFilter {
+  static CONTAINS = 'contains';
+  static FULLMATCH = 'fullmatch';
+  static STARTS = 'starts';
+  static ENDS = 'ends';
+
+  constructor(condition: string, value: string, scope: FilterScope) {
+    super(condition, value);
+    this.ftype = FilterType.NAME;
+    this.scope = scope || super.scope;
+  }
+
+  evaluate(path: string): boolean {
+    this.value = this.value.toLowerCase();
+    path = path.toLowerCase();
+
+    if (this.condition === NameFilter.CONTAINS) {
+      return !(path.indexOf(this.value) >= 0);
+    }
+    if (this.condition === NameFilter.FULLMATCH) return fpath.basename(path) !== this.value;
+    if (this.condition === NameFilter.STARTS) {
+      let filename: string;
+      filename = fpath.basename(path);
+      return !filename.startsWith(this.value);
+    }
+    if (this.condition === NameFilter.ENDS) {
+      let filename: string;
+      filename = fpath.basename(path);
+      return !filename.endsWith(this.value);
+    }
+
+    return true;
+  }
+}
+
+export class ContentFilter extends AbstractFilter {
+  static VALUE_BINARY = 'BINARY';
+  static VALUE_TEXT = 'TEXT';
+  static EQUAL = '=';
+  static NOT_EQUAL = '!=';
+
+  constructor(condition: string, value: string, scope: FilterScope) {
+    super(condition, value);
+    this.ftype = FilterType.CONTENT;
+    this.scope = scope || super.scope; // Verificar
+  }
+
+  evaluate(path: string): boolean {
+    const binary = isBinaryFileSync(path);
+
+    if (this.condition === ContentFilter.EQUAL && this.value === ContentFilter.VALUE_BINARY && binary) return false;
+    if (this.condition === ContentFilter.NOT_EQUAL && this.value === ContentFilter.VALUE_TEXT && binary) return false;
+    if (this.condition === ContentFilter.EQUAL && this.value === ContentFilter.VALUE_TEXT && !binary) return false;
+    if (this.condition === ContentFilter.NOT_EQUAL && this.value === ContentFilter.VALUE_BINARY && !binary) return false;
+    return true;
+  }
+}
+
+export class ExtensionFilter extends AbstractFilter {
+
+  constructor(condition: string, value: string, scope: FilterScope) {
+    super(condition, value);
+    this.ftype = FilterType.EXTENSION;
+    this.scope = scope || super.scope; // Verificar
+  }
+
+  evaluate(path: string): boolean {
+    path = path.toLowerCase();
+    this.value = this.value.toLowerCase();
+    return !path.endsWith(this.value);
+  }
+}
+
+export class SizeFilter extends AbstractFilter {
+  static BIGGER = '>'
+  static SMALLER = '<'
+  static EQUAL = '='
+  constructor(condition: string, value: string, scope: FilterScope) {
+    super(condition, value);
+    this.ftype = FilterType.SIZE;
+    this.scope = scope || super.scope; // Verificar
+  }
+
+  evaluate(path: string): boolean {
+    const stat = fs.statSync(path);
+
+    if (this.condition === SizeFilter.BIGGER) {
+      if (stat.size > parseInt(this.value, 10)) {
+        //   console.log("NO aceptado por que NO es mayor");
+        return false;
+      }
+      return true;
+    }
+    if (this.condition === SizeFilter.SMALLER) {
+      if (stat.size < parseInt(this.value, 10)) {
+        return false;
+      }
+      return true;
+    }
+    if (this.condition === SizeFilter.EQUAL) {
+      if (stat.size === parseInt(this.value, 10)) {
+        return false;
+      }
+
+      return true;
+    }
+
+    return true;
+  }
+}
+export class DateFilter extends AbstractFilter {
+  static BIGGER = '>'
+  static SMALLER = '<'
+  constructor(condition: string, value: string, scope: FilterScope) {
+    super(condition, value);
+    this.ftype = FilterType.NAME;
+    this.scope = scope || super.scope; // Verificar
+  }
+
+  evaluate(path: string): boolean {
+    const stats = fs.statSync(path);
+
+    modified = stats.mtime;
+
+    const lDate = new Date(this.value);
+
+    const ms: number = stats.mtimeMs;
+    var modified = new Date(ms);
+    // console.log(lDate);
+    // console.log(modified);
+    if (this.condition === DateFilter.BIGGER) {
+      if (modified > lDate) {
+        return false;
+      }
+      return true;
+    }
+    if (this.condition === DateFilter.SMALLER) {
+      if (modified < lDate) {
+        return false;
+      }
+      return true;
+    }
+    return true;
+  }
+}
+
+export class FilterList {
+  type: FilterListType;
+
+  filters: AbstractFilter[];
+
+  constructor(fList?: IFilter) {
+    this.filters = [];
+    if (fList) this.load(fList);
+  }
+
+  public addFilter(filter: AbstractFilter): void {
+    this.filters.push(filter);
+  }
+
+  // Returns false if the path match with some filter in the list
+  // IMPORTANT: This method DOES NOT honor the FilterType (BANNED | ALLOW)
+  // Use the method include to honor the FilterType
+  public evaluate(path: string): boolean {
+    const pathStat = fs.lstatSync(path);
+
+    let i: number;
+    for (i = 0; i < this.filters.length; i += 1) {
+      const evaluation = this.filters[i].evaluate(path);
+
+      if (this.filters[i].scope === FilterScope.FOLDER && pathStat.isDirectory() && !evaluation) return false;
+
+      if (this.filters[i].scope === FilterScope.FILE && pathStat.isFile() && !evaluation) return false;
+
+      if (this.filters[i].scope === FilterScope.ALL && !evaluation) return false;
+    }
+    return true;
+  }
+
+  //
+  public include(path: string): boolean {
+    if(this.type === FilterListType.BANNED) return this.evaluate(path);
+    return !this.evaluate(path);
+  }
+
+  public save(path: string) {
+    fs.writeFileSync(path, JSON.stringify(this.filters).toString());
+  }
+
+  public loadFromFile(path: string) {
+    const json = fs.readFileSync(path, 'utf8');
+    const filters = JSON.parse(json);
+    this.load(filters);
+  }
+
+  public unload() {
+    this.filters = [];
+  }
+
+  public load(fList: IFilter) {
+    this.type = fList.type;
+    const a = fList.filters;
+
+    let i: number;
+    for (i = 0; i < a.length; i += 1) {
+      const scope = a[i].scope || FilterScope.ALL;
+      if (a[i].ftype === FilterType.NAME) this.addFilter(new NameFilter(a[i].condition, a[i].value, scope));
+      if (a[i].ftype === FilterType.DATE) this.addFilter(new DateFilter(a[i].condition, a[i].value, scope));
+      if (a[i].ftype === FilterType.SIZE) this.addFilter(new SizeFilter(a[i].condition, a[i].value, scope));
+      if (a[i].ftype === FilterType.EXTENSION) this.addFilter(new ExtensionFilter(a[i].condition, a[i].value, scope));
+      if (a[i].ftype === FilterType.CONTENT) this.addFilter(new ContentFilter(a[i].condition, a[i].value, scope));
+    }
+  }
+}
+
+

package/src/sdk/scanner/Dispatcher/Dispatcher.ts

@@ -4,7 +4,6 @@ import EventEmitter from 'eventemitter3';
 import fetch from 'node-fetch';
 import PQueue from "p-queue";
 
-
 import { ScannerEvents } from "../ScannerTypes";
 import { DispatcherResponse } from "./DispatcherResponse";
 import { ScannerCfg } from "../ScannerCfg";
@@ -13,6 +12,8 @@ import { DispatchableItem } from './DispatchableItem';
 import { HttpsProxyAgent } from 'https-proxy-agent';
 import * as syswideCa from "syswide-cas";
 
+import { PackageJSON } from '../../../cli/bin/cli-bin';
+
 const MAX_CONCURRENT_REQUEST = 30;
 
 export class Dispatcher extends EventEmitter {
@@ -139,7 +140,9 @@ export class Dispatcher extends EventEmitter {
         agent: this.proxy,
         method: 'post',
         body: item.getForm(),
-        headers: { 'User-Agent': this.scannerCfg.CLIENT_TIMESTAMP, 'X-Session': this.scannerCfg.API_KEY },
+        headers: {  'User-Agent': this.scannerCfg.CLIENT_TIMESTAMP ? this.scannerCfg.CLIENT_TIMESTAMP : `scanoss-js/v${PackageJSON.version}`,
+                    'X-Session': this.scannerCfg.API_KEY,
+        },
         signal: timeoutController.signal,
       });
 

package/src/sdk/scanner/ScannerCfg.ts

@@ -1,6 +1,6 @@
 export class ScannerCfg {
-  // Client Timestamp
-  CLIENT_TIMESTAMP = "SCANNER JS"
+  // Client Timestamp, default value is scanoss-js/${version}
+  CLIENT_TIMESTAMP = ""
 
   // API URL
   API_URL = 'https://osskb.org/api/scan/direct';

package/src/sdk/tree/Filters/DependencyFilter.ts

@@ -1,9 +1,9 @@
 import { Filter } from './Filter';
 import Node from '../Node'
-import { FilterList } from '../../filters/filtering';
+import { FilterList, IFilter} from '../../Filtering/Filtering';
 import {
   defaultFilterForDependencies
-} from '../../filters/DefaultFilterForDependencies';
+} from '../../Filtering/DefaultFilterForDependencies';
 
 export class DependencyFilter extends Filter {
 
@@ -11,8 +11,8 @@ export class DependencyFilter extends Filter {
 
   public constructor(filterName: string) {
     super();
-    this.filter = new FilterList(filterName);
-    this.filter.load(defaultFilterForDependencies as FilterList)
+    this.filter = new FilterList();
+    this.filter.load(defaultFilterForDependencies)
   }
 
   public evaluate(node: Node): boolean {

package/src/sdk/tree/Filters/ScanFilter.ts

@@ -1,9 +1,9 @@
 import { Filter } from './Filter';
 import Node from '../Node'
-import { FilterList } from '../../filters/filtering';
+import { FilterList } from '../../Filtering/Filtering';
 import {
   defaultFilterForScanning
-} from '../../filters/DefaultFilterForScanning';
+} from '../../Filtering/DefaultFilterForScanning';
 
 export class ScanFilter extends Filter {
 
@@ -11,8 +11,8 @@ export class ScanFilter extends Filter {
 
   public constructor(filterName: string) {
     super();
-    this.filter = new FilterList(filterName);
-    this.filter.load(defaultFilterForScanning as FilterList)
+    this.filter = new FilterList();
+    this.filter.load(defaultFilterForScanning)
   }
 
   public evaluate(node: Node): boolean {

package/src/sdk/tree/Node.ts

@@ -1,4 +1,4 @@
-import { FilterList } from '../filters/filtering';
+import { FilterList } from '../Filtering/Filtering';
 import { Filter } from './Filters/Filter';
 
 export default abstract class Node {

package/src/sdk/tree/Tree.ts

@@ -6,7 +6,7 @@ import { NodeType } from './Node';
 import Node from './Node';
 import File from './File';
 import Folder from './Folder';
-import { FilterList } from '../filters/filtering';
+import { FilterList } from '../Filtering/Filtering';
 import { Filter } from './Filters/Filter';
 
 export class Tree {

package/tests/sdk/FileCount/FileCount.spec.ts

@@ -0,0 +1,35 @@
+import { FileCount } from '../../../src/sdk/FileCount/FileCount';
+import { IDirSummary } from '../../../src/sdk/FileCount/Interfaces';
+import path from 'path';
+import { Format } from '../../../src/sdk/FileCount/Interfaces';
+import { expect } from 'chai';
+
+describe('Suit test for FileCount', () => {
+
+  it('Testing empty dir with CSV and RAW formats', async function() {
+
+    let data = await FileCount.walk(path.join(__dirname, './samples/0'), );
+    expect((data as IDirSummary).totalFileSize ).to.be.equal(0)
+    expect((data as IDirSummary).fileSummary.size ).to.be.equal(0)
+
+
+    data = await FileCount.walk(path.join(__dirname, './samples/0'), {output: Format.CSV});
+    expect(data).equal( "");
+  });
+
+
+  it('Testing dir with files', async function() {
+    const data = await FileCount.walk(path.join(__dirname, './samples/22036'), { output: Format.CSV});
+    expect(data).equal('.ts,6,1\n');
+  });
+
+
+  it('Testing dir with hidden files in CSV formats', async function() {
+    const data = await FileCount.walk(path.join(__dirname, './samples/16557'), { output: Format.CSV});
+    expect(data).equal('.ts,3,1\n');
+  });
+
+
+
+});
+

package/tests/sdk/FileCount/samples/16557/.y.ts

@@ -0,0 +1,27 @@
+// import { DataProvider, IDataLayers } from './DataLayerTypes';
+//
+// export class DataProviderManager {
+//   private dataLayersProviders: Array<DataProvider>;
+//
+//   constructor() {
+//     this.dataLayersProviders = [];
+//   }
+//
+//   public addDataProvider(l: DataProvider) {
+//     this.dataLayersProviders.push(l)
+//   }
+//
+//   public generateData(): IDataLayers {
+//
+//     let dataLayer: IDataLayers = {
+//       component: null,
+//       dependencies: null,
+//       vulnerabilities: null,
+//       summary: null,
+//       licenses: null
+//     };
+//
+//     for (const layer of this.dataLayersProviders) Object.assign(dataLayer, layer.getData());
+//     return dataLayer;
+//   }
+// }

package/tests/sdk/FileCount/samples/16557/.z.ts

@@ -0,0 +1,129 @@
+// /*************  Component interface definition  *************/
+// export interface ComponentDataLayer {
+//   key: string; // purl[0]
+//   purls: string[];
+//   name: string;
+//   vendor: string;
+//   url: string;
+//   health: Health;
+//   versions: Version[];
+// }
+//
+// export interface Version {
+//   version: string;
+//   licenses:  string[]
+//   copyrights: Copyright[]
+//   cryptography: Cryptography[];
+//   quality: Quality;
+// }
+//
+// export interface Copyright {
+//   name: string;
+//   source: string;
+// };
+//
+// export interface Cryptography {
+//   algorithm: string;
+//   strength: string;
+// };
+//
+// export interface Quality {
+//   scoreAvg: number;
+//   count: number;
+//   sum: number; //TODO remove
+// };
+//
+// export interface Health {
+//   creation_date: string;
+//   issues: number;
+//   last_push: string;
+//   last_update: string;
+//   watchers: number;
+//   country: string;
+//   stars: number;
+//   forks: number;
+// }
+// /*************  Component interface definition  *************/
+//
+//
+// /*************  Dependency interface definition  *************/
+// export interface DependencyDataLayer {
+//   file: string;
+//   dependencies: Dependency[];
+// }
+//
+// export interface License {
+//   name: string;
+//   spdxid: string;
+// }
+//
+// export interface Dependency {
+//   purl: string;
+//   component: string;
+//   version: string;
+//   licenses: License[]
+// }
+// /*************  Dependency interface definition  *************/
+//
+// /*************  Vulnerability interface definition  *************/
+// export interface Vulnerability {
+//   id: string;
+//   cve: string;
+//   url: string;
+//   summary: string;
+//   severity: string;
+//   published: string;
+//   modified: string;
+//   source: string;
+// }
+//
+// export interface VulnerabilityDataLayer {
+//   purl: string;
+//   vulnerability: Vulnerability[];
+// }
+// /*************  Vulnerability interface definition  *************/
+//
+//
+// /*************  License interface definition  *************/
+// export interface LicenseDataLayer {
+//   label: string;
+//   value: number;  //Number of licenses found
+//   components: Array<LicenseComponent>;
+//   incompatibleWith: Array<string>;
+//   hasIncompatibles: Array<string>;
+//   copyleft:boolean;
+// }
+//
+//
+// export interface LicenseComponent {
+//   purl: string;
+//   name: string;
+//   versions: Array<string>;
+//   url:string;
+//   vendor:string;
+// }
+// /*************  License interface definition  *************/
+//
+// export interface SummaryDataLayer {
+//   projectName: string;
+//   timestamp: Date;
+//   matchedFiles: number;
+//   noMatchFiles: number;
+//   totalFiles: number;
+// }
+//
+// // Each layer is created to group by differents criteria.
+// export interface IDataLayers {
+//   licenses: LicenseDataLayer[];
+//   component: ComponentDataLayer[];
+//   dependencies: DependencyDataLayer[];
+//   vulnerabilities: VulnerabilityDataLayer[];
+//   summary: SummaryDataLayer;
+// }
+//
+//
+// export interface DataProvider {
+//   getData(): IDataLayers;
+//   getLayerName(): string;
+// }
+//