scanoss 0.2.18 → 0.2.21

package/src/commands/dep.ts

@@ -1,18 +1,30 @@
 import fs from "fs";
-import { Dependency } from "..";
+import { DependencyScanner } from "../lib/dependencies/DependencyScanner";
+import { DependencyScannerCfg } from "../lib/dependencies/DependencyScannerCfg";
 import { Tree } from "../lib/tree/Tree";
+import { isFolder } from "./helpers";
 
 export async function depHandler(rootPath: string, options: any): Promise<void> {
+
   rootPath = rootPath.replace(/\/$/, '');  // Remove trailing slash if exists
   rootPath = rootPath.replace(/^\./, process.env.PWD);  // Convert relative path to absolute path.
+  const pathIsFolder = await isFolder(rootPath);
+  const dependencyScannerCfg = new DependencyScannerCfg();
+  if(options.grpcHost) dependencyScannerCfg.DEFAULT_GRPC_HOST = options.grpcHost;
+  if(options.grpcPort) dependencyScannerCfg.DEFAULT_GRPC_PORT = options.grpcPort;
+
+  const dependencyScanner = new DependencyScanner(dependencyScannerCfg);
 
-  const tree = new Tree(rootPath);
-  tree.buildTree();
+  let fileList: Array<string> = [];
+  fileList.push(rootPath);
 
-  const fileList = tree.getRootFolder().getFiles().map((path) => {return rootPath+path});
+  if (pathIsFolder) {
+    const tree = new Tree(rootPath);
+    tree.buildTree();
+    fileList = tree.getRootFolder().getFiles().map((path) => {return rootPath+path});
+  }
 
-  const dependency = new Dependency();
-  const results = await dependency.scan(fileList);
+  const results = await dependencyScanner.scan(fileList);
 
   if(options.output) {
     fs.promises.writeFile(options.output, JSON.stringify(results, null, 2));

package/src/commands/helpers.ts

@@ -0,0 +1,14 @@
+import fs from 'fs';
+
+// Async function that verify if a path is a folder. If the path is not valid the promise will be rejected
+export const isFolder = (path: string): Promise<boolean> => {
+  return new Promise((resolve, reject) => {
+    fs.stat(path, (err, stats) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(stats.isDirectory());
+      }
+    });
+  });
+}

package/src/commands/scan.ts

@@ -8,20 +8,11 @@ import { DispatcherResponse } from '../lib/scanner/Dispatcher/DispatcherResponse
 import { defaultFilter } from '../lib/filters/defaultFilter';
 import { FilterList } from '../lib/filters/filtering';
 
+import { isFolder } from './helpers';
+
 import fs from 'fs';
 
-// Async function that verify if a path is a folder. If the path is not valid the promise will be rejected
-const isFolder = (path: string): Promise<boolean> => {
-  return new Promise((resolve, reject) => {
-    fs.stat(path, (err, stats) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(stats.isDirectory());
-      }
-    });
-  });
-}
+
 
 export async function scanHandler(rootPath: string, options: any): Promise<void> {
 

package/src/index.ts

@@ -2,4 +2,8 @@ export * from './lib/scanner/ScannerTypes';
 export * from './lib/scanner/ScannerCfg'
 export * from './lib/scanner/Scanner'
 export * from './lib/dependencies/DependencyTypes';
-export * from './lib/dependencies/Dependency';
+export * from './lib/dependencies/DependencyScannerCfg';
+export * from './lib/dependencies/DependencyScanner';
+
+
+

package/src/lib/dependencies/DependencyScanner.ts

@@ -0,0 +1,77 @@
+import { ILocalDependencies } from "./LocalDependency/DependencyTypes";
+import { GrpcDependencyService } from "../grpc/GrpcDependencyService";
+import { DependencyRequest, DependencyResponse } from "../grpc/scanoss/api/dependencies/v2/scanoss-dependencies_pb";
+import { LocalDependencies } from "./LocalDependency/LocalDependency";
+import { DependencyScannerCfg } from "./DependencyScannerCfg";
+import { IDependencyResponse } from "./DependencyTypes";
+
+export class DependencyScanner {
+
+  private localDependency: LocalDependencies;
+
+  private grpcDependencyService: GrpcDependencyService;
+
+  constructor(cfg = new DependencyScannerCfg()) {
+    this.grpcDependencyService = new GrpcDependencyService(cfg.DEFAULT_GRPC_HOST, cfg.DEFAULT_GRPC_PORT);
+    this.localDependency = new LocalDependencies();
+  }
+
+
+  public async scan(files: Array<string>): Promise<IDependencyResponse> {
+    const localDependencies = await this.localDependency.search(files);
+    if (localDependencies.files.length === 0) return null;
+    const request = this.buildRequest(localDependencies);
+    const grpcResponse = await this.grpcDependencyService.get(request);
+    const response = grpcResponse.toObject();
+
+    // Extract scope from localDependencies and add it to response
+    this.mergeScopeField(localDependencies, response);
+    return response;
+  }
+
+
+  private buildRequest(localDependencies: ILocalDependencies): DependencyRequest {
+    try {
+      const depRequest = new DependencyRequest();
+      for (const file of localDependencies.files) {
+        const fileMsg = new DependencyRequest.Files();
+        fileMsg.setFile(file.file);
+        for (const purl of file.purls) {
+          const purlMsg = new DependencyRequest.Purls();
+          purlMsg.setPurl(purl.purl);
+          purlMsg.setRequirement(purl?.requirements);
+          fileMsg.addPurls(purlMsg);
+        }
+        depRequest.addFiles(fileMsg);
+      }
+      return depRequest;
+    } catch (e) {
+      console.error(e);
+      return null;
+    }
+  }
+
+  private mergeScopeField(localdependency: ILocalDependencies, serverResponse: DependencyResponse.AsObject
+    ): IDependencyResponse {
+
+    const scopeHashMap = {};
+
+    for (const file of localdependency.files) {
+      const filename = file.file
+      for (const dependency of file.purls) {
+        if (dependency?.scope) scopeHashMap[filename + dependency.purl] = dependency.scope;
+      }
+    }
+
+    for (const file of serverResponse.filesList) {
+      const filename = file.file
+      for (const dependency of file.dependenciesList) {
+        const scope = scopeHashMap[filename + dependency.purl];
+        if (scope) dependency['scope'] = scope;
+      }
+    }
+
+    return serverResponse;
+  }
+
+}

package/src/lib/dependencies/DependencyScannerCfg.ts

@@ -0,0 +1,7 @@
+export class DependencyScannerCfg {
+
+  DEFAULT_GRPC_PORT = '443';
+
+  DEFAULT_GRPC_HOST = 'scanoss.com';
+
+}

package/src/lib/dependencies/DependencyTypes.ts

@@ -1,22 +1,24 @@
-
-interface ILicense {
-  name: string;
+export interface LicensesList {
+    name: string;
+    spdxId: string;
+    isSpdxApproved: boolean;
 }
 
-export interface IDependency {
-  component: string;
-  purl: string;
-  version: string;
-  licenses: Array<ILicense>;
+export interface DependenciesList {
+    component: string;
+    purl: string;
+    version?: string;
+    scope?: string;
+    licensesList: LicensesList[];
 }
 
-export interface IFile {
-  file: string;
-  id: string;
-  status: string;
-  dependencies: Array<IDependency>;
+export interface FilesList {
+    file: string;
+    id: string;
+    status: string;
+    dependenciesList: DependenciesList[];
 }
 
 export interface IDependencyResponse {
-  files: Array<IFile>;
+    filesList: FilesList[];
 }

package/src/lib/dependencies/LocalDependency/DependencyTypes.ts

@@ -0,0 +1,21 @@
+export interface ILocalPurl {
+  purl: string;
+  requirements?: string;
+  scope?: string;
+}
+
+export interface ILocalDependency {
+  file: string;
+  purls: Array<ILocalPurl>;
+}
+
+export interface ILocalDependencies{
+  files: Array<ILocalDependency>;
+}
+
+/* Parser funcion definition */
+export type ParserFuncType = (fileContent: string, filePath: string) => ILocalDependency;
+
+export interface ParserDefinitions {
+  [key: string]: ParserFuncType;
+}

package/src/lib/dependencies/LocalDependency/LocalDependency.ts

@@ -0,0 +1,48 @@
+import path from 'path';
+import fs from 'fs';
+import { ParserFuncType, ILocalDependencies } from "./DependencyTypes";
+import { requirementsParser } from "./parsers/pyParser";
+import { pomParser } from "./parsers/mavenParser";
+import { packagelockParser, packageParser } from "./parsers/npmParser";
+import { gemfilelockParser, gemfileParser } from "./parsers/rubyParser";
+import { goModParser } from './parsers/golangParser';
+
+export class LocalDependencies {
+
+  private parserMap: Record<string, ParserFuncType>;
+
+  constructor() {
+      /*
+      This is a hash map that connect a filename with it's own parser function
+      Any parser function must return a ILocalDependencies object (See DependencyTypes.ts)
+      */
+      this.parserMap = {
+        'requirements.txt': requirementsParser,
+        'pom.xml': pomParser,
+        'package.json': packageParser,
+        'package-lock.json': packagelockParser,
+        'Gemfile': gemfileParser,
+        'Gemfile.lock': gemfilelockParser,
+        'go.mod': goModParser,
+      };
+  }
+
+  public async search(files: Array<string>): Promise<ILocalDependencies> {
+    let results: ILocalDependencies = {files: []};
+    for (const filePath of files) {
+        const fileName = path.basename(filePath);
+        if(this.parserMap[fileName] != null) {
+          try{
+            const fileContent = await fs.promises.readFile(filePath, 'utf8');
+            const dependency = this.parserMap[fileName](fileContent, filePath);
+            if(dependency.purls.length != 0)
+                results.files.push(dependency);
+          } catch(e) {
+            console.error(e);
+            continue;
+          }
+        }
+    }
+    return results;
+  }
+}

package/src/lib/dependencies/LocalDependency/parsers/golangParser.ts

@@ -0,0 +1,78 @@
+import { ILocalDependency } from "../DependencyTypes";
+
+import { PackageURL } from "packageurl-js";
+import path from "path";
+
+function parseModule (str: string) {
+  const res = /(?<type>[^\s]+)(?:\s)+(?<ns_name>[^\s]+)\s?(?<version>(.*))/.exec(str);
+  return {
+    type: res.groups.type,
+    ns_name: res.groups.ns_name,
+    version: res.groups.version
+  };
+}
+
+function parseDepLink (str: string) {
+  const res = /.*?(?<ns_name>[^\s]+)\s+(?<version>(.*))/.exec(str);
+  return {
+    ns_name: res?.groups?.ns_name,
+    version: res?.groups?.version
+  };
+}
+
+// Removes comments and spaces
+function preprocessLine(line: string) {
+    if (line.includes("//"))
+      line = line.substring(0,line.indexOf("//"));
+    return line.trim();
+}
+
+
+
+
+const PURL_TYPE = 'golang';
+
+
+// See reference on: https://go.dev/ref/mod#go-mod-file
+const MANIFEST_FILE = 'go.mod';
+export function goModParser(fileContent: string, filePath: string): ILocalDependency {
+
+  // If the file is not a go.mod manifest file, return an empty results
+  const results: ILocalDependency = {file: filePath, purls: []};
+  if(path.basename(filePath) != MANIFEST_FILE)
+      return results;
+
+  const lines =	fileContent.split('\n');
+
+	const require = [];
+	const exclude = [];
+
+  for (let num = 0 ; num < lines.length ; num+=1) {
+
+    let line = preprocessLine(lines[num]);
+
+
+    if(line.includes('require') && line.includes('(')) {
+      num+=1;
+      line = preprocessLine(lines[num]);
+      while (num < lines.length && line!==')') {
+
+        const {ns_name, version} = parseDepLink(line);
+
+        const index = ns_name.lastIndexOf('/');
+        const namespace = ns_name.substring(0, index);
+        const name = ns_name.substring(index + 1);
+
+        const purlString = new PackageURL(PURL_TYPE, namespace, name, version, undefined, undefined).toString();
+        results.purls.push({purl: purlString});
+
+        require.push(line);
+
+        num+=1;
+        line = preprocessLine(lines[num]);
+      }
+    }
+  }
+
+  return results;
+}

package/src/lib/dependencies/{parsers → LocalDependency/parsers}/mavenParser.ts

@@ -1,19 +1,19 @@
 import path from "path";
 import { PackageURL } from "packageurl-js";
-import { FileDependency } from "./types";
-import { isValidPath, isValidUrl } from './utils';
+import { ILocalDependency } from "../DependencyTypes";
 
 const PURL_TYPE = 'maven';
 
 
+
 // Parse a pom.txt file from maven manifest file
 // See reference on: https://maven.apache.org/guides/introduction/introduction-to-the-pom.html
 // and https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html
 const MANIFEST_FILE = 'pom.xml';
-export function pomParser(fileContent: string, filePath: string): FileDependency {
+export function pomParser(fileContent: string, filePath: string): ILocalDependency {
 
     // If the file is not a python manifest file, return an empty results
-    const results: FileDependency = {file: filePath, purls: []};
+    const results: ILocalDependency = {file: filePath, purls: []};
     if(path.basename(filePath) != MANIFEST_FILE)
         return results;
 
@@ -34,7 +34,7 @@ export function pomParser(fileContent: string, filePath: string): FileDependency
         let version = versionReg ? versionReg[1] : '';
 
         const ver = version.match(/\${(.*?)}/);
-        if(ver && ver.length >= 1)  {
+        if(ver && ver.length >= 1) {
           if(ver[1] === 'project.version') { // TODO: Add support for project.version
             version = undefined;
           } else {
@@ -50,8 +50,12 @@ export function pomParser(fileContent: string, filePath: string): FileDependency
             purlQualifiers['type'] = type[1]
         }
 
+        // Extract scope.
+        const scopeRes = dependency.match(/<scope>([^<]*)<\/scope>/);
+        const scope = scopeRes ? scopeRes[1] : undefined;
+
         const purlString = new PackageURL(PURL_TYPE, namespace, name, version, purlQualifiers, undefined).toString();
-        results.purls.push({purl: purlString});
+        results.purls.push({purl: purlString, scope: scope});
       });
     }
     return results;

package/src/lib/dependencies/{parsers → LocalDependency/parsers}/npmParser.ts

@@ -1,7 +1,6 @@
 import path from "path";
 import { PackageURL } from "packageurl-js";
-import { FileDependency } from "./types";
-import { isValidPath, isValidUrl } from './utils';
+import { ILocalDependency } from "../DependencyTypes";
 
 const PURL_TYPE = 'npm';
 
@@ -9,19 +8,26 @@ const PURL_TYPE = 'npm';
 // Parse a package.json file from node projects
 // See reference on: https://docs.npmjs.com/cli/v8/configuring-npm/package-json
 const MANIFEST_FILE = 'package.json';
-export function packageParser(fileContent: string, filePath: string): FileDependency {
+export function packageParser(fileContent: string, filePath: string): ILocalDependency {
     // If the file is not manifest file, return an empty results
-    const results: FileDependency = {file: filePath, purls: []};
+    const results: ILocalDependency = {file: filePath, purls: []};
     if(path.basename(filePath) != MANIFEST_FILE)
         return results;
     const o = JSON.parse(fileContent);
     let devDeps = Object.keys(o.devDependencies || {});
     let deps = Object.keys(o.dependencies || {});
     let listDeps = [...deps, ...devDeps];
-    for(const name of listDeps){
+
+    for(const name of deps){
         const purlString = new PackageURL(PURL_TYPE, undefined, name, undefined, undefined, undefined).toString();
-        results.purls.push({purl: purlString});
+        results.purls.push({purl: purlString, scope: "dependencies", requirements: o.dependencies[name]});
+    }
+
+    for(const name of devDeps){
+      const purlString = new PackageURL(PURL_TYPE, undefined, name, undefined, undefined, undefined).toString();
+      results.purls.push({purl: purlString, scope: "devDependencies", requirements: o.devDependencies[name]});
     }
+
     return results;
 }
 
@@ -29,9 +35,9 @@ export function packageParser(fileContent: string, filePath: string): FileDepend
 // Parse a package-lock.json file from node projects
 // See reference on: https://docs.npmjs.com/cli/v8/configuring-npm/package-json
 const MANIFEST_FILE_1 = 'package-lock.json';
-export function packagelockParser(fileContent: string, filePath: string): FileDependency {
+export function packagelockParser(fileContent: string, filePath: string): ILocalDependency {
 
-    const results: FileDependency = {file: filePath, purls: []};
+    const results: ILocalDependency = {file: filePath, purls: []};
     if(path.basename(filePath) != MANIFEST_FILE_1)
         return results;
 

package/src/lib/dependencies/LocalDependency/parsers/pyParser.ts

@@ -0,0 +1,55 @@
+import path from "path";
+import { PackageURL } from "packageurl-js";
+import { ILocalDependency } from "../DependencyTypes";
+import { isValidPath, isValidUrl } from './utils';
+
+
+function parseDep (str: string) {
+  const res = /^(?<name>[-\w]+)\s*(?<sym>[>=~!]*)\s*(?<version>[\d\.]*)/.exec(str);
+  return {
+    name: res?.groups?.name,
+    sym: res?.groups?.sym,
+    version: res?.groups?.version,
+  };
+}
+
+const PURL_TYPE = 'pypi';
+
+// Parse a requirements.txt file from python projects
+// See reference on: https://pip.pypa.io/en/stable/reference/requirements-file-format/
+const MANIFEST_FILE = 'requirements.txt';
+export function requirementsParser(fileContent: string, filePath: string): ILocalDependency {
+
+    // If the file is not a python manifest file, return an empty results
+    const results: ILocalDependency = {file: filePath, purls: []};
+    if(path.basename(filePath) != MANIFEST_FILE)
+        return results;
+
+    const lines: Array<string> = fileContent.split('\n');
+
+    for (let line of lines) {
+        line = line.trim();
+        if(line.length == 0) continue;
+        if(!line.startsWith('#') && line.length>0) { // Avoid comments and new lines
+            if(isValidUrl(line)) {
+                // For reference about the regex see https://www.rfc-editor.org/rfc/rfc3986#appendix-B
+                const res = line.match(/^(([^:\/?#]+):)?(\/\/([^\/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?/);
+                continue;
+            }
+            else if(isValidPath(line)) {continue;} // Do not parse local dependencies.
+            else if(line.startsWith('-r')) {continue;} // Recursive dependencies (NOT SUPPORTED YET)
+            else {
+
+                const dep = parseDep(line);
+                if (dep.sym === '==') {
+                  const purlString = new PackageURL(PURL_TYPE, undefined, dep.name, dep.version, undefined, undefined).toString();
+                  results.purls.push({purl: purlString});
+                } else {
+                  const purlString = new PackageURL(PURL_TYPE, undefined, dep.name, undefined, undefined, undefined).toString();
+                  results.purls.push({purl: purlString, requirements: dep.sym+dep.version});
+                }
+            }
+        }
+    }
+    return results;
+}

package/src/lib/dependencies/{parsers → LocalDependency/parsers}/rubyParser.ts

@@ -1,8 +1,7 @@
 import path from "path";
 import { PackageURL } from "packageurl-js";
-import { FileDependency } from "./types";
-import { isValidPath, isValidUrl } from './utils';
-import { stringify } from "querystring";
+import { ILocalDependency } from "../DependencyTypes";
+
 
 const PURL_TYPE = 'gem';
 
@@ -11,10 +10,10 @@ const PURL_TYPE = 'gem';
 // See reference on: https://bundler.io/gemfile.html
 // and https://bundler.io/man/gemfile.5.html
 const MANIFEST_FILE = 'Gemfile';
-export function gemfileParser(fileContent: string, filePath: string): FileDependency {
-    
+export function gemfileParser(fileContent: string, filePath: string): ILocalDependency {
+
     // If the file is not a manifest file, return an empty results
-    const results: FileDependency = {file: filePath, purls: []};
+    const results: ILocalDependency = {file: filePath, purls: []};
     if(path.basename(filePath) != MANIFEST_FILE)
         return results;
 
@@ -32,7 +31,7 @@ export function gemfileParser(fileContent: string, filePath: string): FileDepend
                 compName = compName.replace(/['"]/g, '');
                 const purlString = new PackageURL(PURL_TYPE, undefined, compName, undefined, undefined, undefined).toString();
                 results.purls.push({purl: purlString});
-            }              
+            }
         }
     }
     return results;
@@ -40,10 +39,10 @@ export function gemfileParser(fileContent: string, filePath: string): FileDepend
 
 
 const MANIFEST_FILE_1 = 'Gemfile.lock';
-export function gemfilelockParser(fileContent: string, filePath: string): FileDependency {
-    
+export function gemfilelockParser(fileContent: string, filePath: string): ILocalDependency {
+
     // If the file is not a manifest file, return an empty results
-    const results: FileDependency = {file: filePath, purls: []};
+    const results: ILocalDependency = {file: filePath, purls: []};
     if(path.basename(filePath) != MANIFEST_FILE_1)
         return results;
 
@@ -80,13 +79,13 @@ const firstDepLevelRegex = /^ {4}(?! )/;
 class GemfileLockParser {
 
     private statesMap;
-    
-    private state;  
-    
-    private current_options: Record<string, string>; 
-    
-    private current_gem; 
-    
+
+    private state;
+
+    private current_options: Record<string, string>;
+
+    private current_gem;
+
     private purlList;
 
     constructor () {
@@ -127,7 +126,6 @@ class GemfileLockParser {
             // process the line
             if (this.state) this.state(line);
         }
-        this.refine();
         return this.purlList;
     }
 
@@ -141,21 +139,21 @@ class GemfileLockParser {
         const key = match.length>=1 ? match[1] : null;
         const value = match.length>=2 ? match[2] : null;
         if(key) this.current_options[key] = value;
-    }   
+    }
 
     private parseDependency(line: string) {}
     private parsePlatform(line: string){}
-    
+
 
     private parseSpec(line: string) {
-        
+
         if(this.current_gem == GEM) {
-            if(firstDepLevelRegex.test(line)) { 
+            if(firstDepLevelRegex.test(line)) {
                 line = line.trimStart();
                 const match = line.match(specRegex);
 
                 const purl = new PackageURL( PURL_TYPE,
-                                undefined, 
+                                undefined,
                                 match.groups.name,
                                 match.groups.version,
                                 undefined,
@@ -171,11 +169,5 @@ class GemfileLockParser {
 
         // Purl from local dependencies are not generated
         if(this.current_gem == PATH){}
-
     }
-
-    private refine() {}
-
-
-
 }

package/src/lib/dependencies/{parsers → LocalDependency/parsers}/utils.ts

@@ -1,4 +1,3 @@
-import path from "path";
 import { URL } from "url";
 
 export function isValidUrl(string: string): boolean {