scanoss 0.7.9 → 0.7.11

package/src/sdk/Dependencies/LocalDependency/parsers/mavenParser.ts

@@ -1,209 +0,0 @@
-import path from "path";
-import { PackageURL } from "packageurl-js";
-import { ILocalDependency } from "../DependencyTypes";
-
-const PURL_TYPE = 'maven';
-
-// Parse a pom.txt file from maven manifest file
-// See reference on: https://maven.apache.org/guides/introduction/introduction-to-the-pom.html
-// and https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html
-const MANIFEST_FILE = 'pom.xml';
-export function pomParser(fileContent: string, filePath: string): Promise<ILocalDependency> {
-
-    // If the file is not a python manifest file, return an empty results
-    const results: ILocalDependency = {file: filePath, purls: []};
-    if(path.basename(filePath) != MANIFEST_FILE)
-      return Promise.resolve(results);
-
-    const dependencies = fileContent.match(/<dependency>((?:.|\n)*?)<\/dependency>/gm);
-    if(dependencies) {
-
-      dependencies.forEach(dependency => {
-        // Extract groupId. It's the purl namespace
-        const groupId = dependency.match(/<groupId>([^<]*)<\/groupId>/);
-        const namespace = (groupId && groupId.length>=1) ? groupId[1] : null;
-
-        // Extract artifact id. It's the purl name
-        const artifactId = dependency.match(/<artifactId>([^<]*)<\/artifactId>/);
-        const name = (artifactId && artifactId.length>=1) ? artifactId[1] : null;
-
-        const versionReg = dependency.match(/<version>([^<]*)<\/version>/);
-        let version;
-        if(versionReg && versionReg.length>=1) version = resolve_version(versionReg[1], fileContent);
-
-
-        // Extract scope.
-        const scopeRes = dependency.match(/<scope>([^<]*)<\/scope>/);
-        const scope = (scopeRes && scopeRes.length>=1) ? scopeRes[1] : null;
-
-
-        //Detect and extract purl qualifiers
-        const classifierRes = dependency.match(/<classifier>([^<]*)<\/classifier>/);
-        const classifier = (classifierRes && classifierRes.length>=1) ? classifierRes[1] : null;
-
-        const typeRes = dependency.match(/<type>([^<]*)<\/type>/);
-        const type = (typeRes && typeRes.length>=1) ? typeRes[1] : null;
-
-        let purlQualifiers;
-        if (type || classifier)
-        purlQualifiers = {
-          ...(type && {type}),
-          ...(classifier && {classifier}),
-        };
-
-        const purlString = new PackageURL(PURL_TYPE, namespace, name, undefined, purlQualifiers, undefined).toString();
-        results.purls.push({purl: purlString, requirement: version, scope: scope});
-
-      });
-    }
-
-    //Remove purls duplicated
-    const nonDuplicatedResults = removeDuplicated(results);
-    return Promise.resolve(nonDuplicatedResults);
-}
-
-function removeDuplicated(results: ILocalDependency): ILocalDependency {
-  const map = {};
-  for (let res of results.purls) {
-    map[res.purl] = res
-  }
-  results.purls = Object.values(map);
-  return results
-}
-
-function resolve_version(dependency_version: string, file_content: string): string {
-  // See properties: https://maven.apache.org/pom.html#properties
-  let version = '';
-  if(dependency_version) {
-    if(/\${project.version}/.test(dependency_version)) {
-      version = extract_content_from_tag(file_content, ['project', 'version']);
-    } else if(/\${.*?}/.test(dependency_version)) {
-      const property = dependency_version.match(/\${(.*?)}/)[1];
-      const result = file_content.match(new RegExp(`<${property}>([^<]*)<\/${property}>`));
-      if (result && result.length>0) version = result[1];
-    } else {
-      version = dependency_version.toString();
-    }
-  }
-  return version;
-}
-
-
-function get_start_tag_name(line: string): string {
-  const result = line.match(/\<([\w\-\.]+).*?>/);
-  if (result) return result[1].trim();
-  return '';
-}
-
-function get_end_tag_name(line: string): string {
-  const result = line.match(/\<\/([\w\-\.]+) ?>/);
-  if (result) return result[1].trim();
-  return '';
-}
-
-function get_end_tag(line: string): string {
-  const result = get_end_tag_name(line);
-  if (result !== '') return `</${result}>`
-  return '';
-}
-
-function get_start_tag(line: string): string {
-  const result = get_start_tag_name(line);
-  if (result !== '') return `<${result}>`
-  return '';
-}
-
-function element_match(openTag: string, closeTag:string): boolean {
-  return get_start_tag_name(openTag) === get_end_tag_name(closeTag);
-}
-
-function is_element_complete(line: string) {
-  return get_start_tag_name(line) === get_end_tag_name(line);
-}
-
-
-function get_offset_until_end_of_tag(lines: Array<string>, end_tag_name: string) {
-  let i = 0;
-  for (const line of lines) {
-      if ( get_end_tag_name(line) === end_tag_name) break;
-      i += 1;
-  }
-  return i;
-}
-
-function remove_comments(lines: Array<string>): Array<string> {
-  for (let i=0; i<lines.length; i+=1) {
-      let openCommentFlag = /<!--/.test(lines[i]);
-      let endCommentFlag = /-->/.test(lines[i]);
-
-      if(openCommentFlag && endCommentFlag)
-          lines[i] = lines[i].replace(/<!--.*-->/, '');
-      else if (openCommentFlag){
-          while(!/-->/.test(lines[i]) && i<lines.length) {
-              lines[i] = '';
-              i += 1;
-          }
-          lines[i] = lines[i].replace(/.*-->/, '');
-      }
-
-  }
-  return lines;
-}
-
-
-function extract_content_from_tag(file_content: string, selector: Array<string>): string {
-  let lines = file_content.split('\n');
-  const stack: Array<string> = [];
-
-  let selectorIndex = 0;
-  let startTagName = '';
-  let endTagName = '';
-  let content = '';
-
-  // Sanitize xml: Removes comments
-  lines = remove_comments(lines);
-
-  for (let i=0; i<lines.length; i+=1) {
-    let line = lines[i].trim();
-    if(line === '') continue;
-
-    startTagName = get_start_tag_name(line);
-    endTagName = get_end_tag_name(line);
-
-    // Element complete in the same line and different than my selector
-    if(selector[selectorIndex] !== startTagName && is_element_complete(line)) continue;
-
-    // Element spans multiline and is different than my selector
-    // Loop until find corresponding end tag
-    if (selector[selectorIndex] !== startTagName) {
-       i += 1;
-       while (i<lines.length && !element_match(line, lines[i])) i+=1;
-       continue;
-    }
-
-    // lines[i] points to the opening tag of the current selector[selectorIndex]
-    selectorIndex += 1;
-    stack.push(startTagName);
-
-    // Target reached
-    if(selector.length === stack.length) {
-      // Target has only one line
-      if (is_element_complete(line)) {
-          line = line.replace(get_end_tag(line), '');
-          line = line.replace(get_start_tag(line), '');
-          return line;
-      }
-
-        // Extracts everything beetwen opening and closing tag and return.
-        i += 1;
-        while (i<lines.length && !element_match(line, lines[i]) ) {
-          content += lines[i].trim();
-          i += 1;
-        }
-        return content;
-    }
-    startTagName='';
-    endTagName='';
-  }
-  return '';
-}

package/src/sdk/Dependencies/LocalDependency/parsers/npmParser.ts

@@ -1,227 +0,0 @@
-import path from "path";
-import { PackageURL } from "packageurl-js";
-import { ILocalDependency } from "../DependencyTypes";
-
-const PURL_TYPE = 'npm';
-
-
-// Parse a package.json file from node projects
-// See reference on: https://docs.npmjs.com/cli/v8/configuring-npm/package-json
-const MANIFEST_FILE = 'package.json';
-export function packageParser(fileContent: string, filePath: string): Promise<ILocalDependency> {
-    // If the file is not manifest file, return an empty results
-    const results: ILocalDependency = {file: filePath, purls: []};
-    if(path.basename(filePath) != MANIFEST_FILE)
-      return Promise.resolve(results);
-
-    const o = JSON.parse(fileContent);
-    let devDeps = Object.keys(o.devDependencies || {});
-    let deps = Object.keys(o.dependencies || {});
-
-    for(const name of deps){
-        const purlString = new PackageURL(PURL_TYPE, undefined, name, undefined, undefined, undefined).toString();
-        results.purls.push({purl: purlString, scope: "dependencies", requirement: o.dependencies[name]});
-    }
-
-    for(const name of devDeps){
-      const purlString = new PackageURL(PURL_TYPE, undefined, name, undefined, undefined, undefined).toString();
-      results.purls.push({purl: purlString, scope: "devDependencies", requirement: o.devDependencies[name]});
-    }
-
-    return Promise.resolve(results);
-}
-
-
-// Parse a package-lock.json file from node projects
-// See reference on: https://docs.npmjs.com/cli/v8/configuring-npm/package-json
-export function packagelockParser(fileContent: string, filePath: string): Promise<ILocalDependency> {
-
-    const results: ILocalDependency = {file: filePath, purls: []};
-
-    if(path.basename(filePath) != 'package-lock.json')
-      return Promise.resolve(results);
-
-    const packages = JSON.parse(fileContent)?.packages;
-
-    if(!packages) return Promise.resolve(results);
-
-    for (const [key, value] of Object.entries(packages)) {
-        if(!key) continue;
-
-        const keySplit = key.split("/")
-        const depName = keySplit[keySplit.length-1]
-
-        let purl = new PackageURL(PURL_TYPE, undefined, depName,undefined, undefined, undefined).toString();
-        let req = value['version'];
-        results.purls.push({purl: purl, requirement: req});
-    }
-
-    return Promise.resolve(results);
-}
-
-
-
-export function yarnLockParser(fileContent: string, filePath: string): Promise<ILocalDependency> {
-  const results: ILocalDependency = {file: filePath, purls: []};
-
-  if(path.basename(filePath) != 'yarn.lock')
-    return Promise.resolve(results);
-
-  const yarnVersion = yarnLockRecognizeVersion(fileContent)
-  if (yarnVersion === YarnLockVersionEnum.V1) return yarnLockV1Parser(fileContent, filePath)
-  else if (yarnVersion === YarnLockVersionEnum.V2) return yarnLockV2Parser(fileContent, filePath)
-
-  return Promise.resolve(results);
-}
-
-enum YarnLockVersionEnum {
-  "V1"  ,
-  "V2",
-  UnknownYarnLockFormat
-}
-
-/*
-    The start of v1 file has this:
-        # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
-        # yarn lockfile v1
-
-    The start of v2 file has this:
-        # This file is generated by running "yarn install" inside your project.
-        # Manual changes might be lost - proceed with caution!
-
-        __metadata:
- */
-export function yarnLockRecognizeVersion(fileContent: string): YarnLockVersionEnum {
-
-  const yarn = fileContent.split("\n", 10) //Check only the first 10 lines;
-  for (const line of yarn) {
-    if ( line.includes('__metadata:') ) return YarnLockVersionEnum.V2
-    if ( line.includes('yarn lockfile v1') ) return YarnLockVersionEnum.V1
-  }
-  return YarnLockVersionEnum.UnknownYarnLockFormat
-}
-
-export function yarnLockV1Parser(fileContent: string, filePath: string): Promise <ILocalDependency> {
-
-  const results: ILocalDependency = {file: filePath, purls: []};
-
-  //Yield an array with each element is a dependency
-  /*
-    "@babel/core@^7.1.0", "@babel/core@^7.3.4":
-      version "7.3.4"
-      resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.3.4.tgz#921a5a13746c21e32445bf0798680e9d11a6530b"
-      integrity sha512-jRsuseXBo9pN197KnDwhhaaBzyZr2oIcLHHTt2oDdQrej5Qp57dCCJafWx5ivU8/alEYDpssYqv1MUqcxwQlrA==
-      dependencies:
-        "@babel/code-frame" "^7.0.0"
-        "@babel/generator" "^7.3.4"
-   */
-  const yl_dependencies = fileContent.split("\n\n");
-
-  for (const yl_dependency of yl_dependencies) {
-
-
-
-    const dependencyData: Record<string, string> = {}
-    const topRequirements = [];
-
-    const dep_lines = yl_dependency.split("\n");
-    if (dep_lines.every((line) =>  line.trim().startsWith("#") == true)) continue //All lines are coments
-    if (dep_lines.every((line) =>  line.trim() == "")) continue  //All lines are empty lines
-
-    for (const dep_line of dep_lines) {
-
-      // Clean comments and empty lines
-      const trimmed = dep_line.trim();
-      const comment = trimmed.startsWith('#');
-      if (!trimmed || comment) continue
-
-      // Do nothing with it's own dependencies
-      //    "@babel/code-frame" "^7.0.0"
-      //    "@babel/generator" "^7.3.4"
-      if (dep_line.startsWith(' '.repeat(4))) {}
-
-      //  version "7.3.4"
-      //  resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.3.4.tgz#921a5a13746c21e32445bf0798680e9d11a6530b"
-      //  integrity sha512-jRsuseXBo9pN197KnDwhhaaBzyZr2oIcLHHTt2oDdQrej5Qp57dCCJafWx5ivU8/alEYDpssYqv1MUqcxwQlrA==
-      //  dependencies:
-      else if (dep_line.startsWith(' '.repeat(2))) {
-        const dep = trimmed.split(" ")
-        const key = dep[0].trim();
-        if (key !== "dependencies:" && key!=="optionalDependencies:") {
-          dependencyData[key] = dep[1].replace(/\"|\'/g, "");
-        }
-      }
-
-      // the first line of a dependency has the name and requirements
-      //"@babel/core@^7.1.0", "@babel/core@^7.3.4":
-      else if (!dep_line.startsWith(' ')){
-        const dep = dep_line.replace(/:/g, "").split(",");
-        const requirements = dep.map(line => line.trim().replace(/"|'/g, ""));
-
-        for (const req of requirements) {
-
-          const atIndex = req.lastIndexOf("@")
-
-          let constraint = req.slice(atIndex+1)  // gets ^7.1.0
-          constraint = constraint.replace(/"|'/g, "");
-
-          const ns_name = req.slice(0, atIndex)
-
-          let ns = '';
-          let name = ns_name;
-          if (ns_name.includes("/")) {
-            const slashIndex = req.lastIndexOf("/")
-            ns = ns_name.slice(0,slashIndex);
-            name = ns_name.slice(slashIndex+1)
-          }
-
-          topRequirements.push({constraint: constraint, ns: ns, name: name });
-        }
-
-      }
-
-
-    }
-
-    //Make sure that name and namespace are equal for the same dependency
-    const isNsNameEqual = topRequirements.every((topRequirement) => {
-      return topRequirement.ns === topRequirements[0].ns && topRequirement.name === topRequirements[0].name
-    });
-
-    if (!isNsNameEqual) {
-      console.error("Different names for same dependency is not supported")
-      continue
-    }
-    const topRequirement = topRequirements[0];
-    const namespace = topRequirement.ns;
-    const name = topRequirement.name;
-    const version = dependencyData['version'];
-    const purl = new PackageURL(PURL_TYPE, namespace, name, version, undefined, undefined).toString()
-
-    let requirement = ''
-    for (const topRequirement of topRequirements) {
-      requirement += topRequirement.constraint + ", "
-    }
-    if (requirement.endsWith(", ")) {
-      requirement = requirement.slice(0, requirement.length-2)
-    }
-
-    results.purls.push({purl: purl, requirement: requirement})
-
-  }
-
-
-  return Promise.resolve(results);
-
-}
-
-
-//TODO: Implement yarn lock V2 parser
-export function yarnLockV2Parser(fileContent: string, filePath: string): Promise<ILocalDependency> {
-
-  const results: ILocalDependency = {file: filePath, purls: []};
-
-
-  return Promise.resolve(results);
-
-}

package/src/sdk/Dependencies/LocalDependency/parsers/nugetParser.ts

@@ -1,64 +0,0 @@
-import { ILocalDependency } from '../DependencyTypes';
-import xml, { Element } from 'xml-js';
-import { PackageURL } from 'packageurl-js';
-
-const PURL_TYPE = "nuget";
-
-export function csprojParser(fileContent: string, filePath: string): Promise<ILocalDependency> {
-
-  const results: ILocalDependency = {file: filePath, purls: []};
-
-  try {
-    const fileProject = xml.xml2js(fileContent);
-    const project = fileProject.elements.find(item => item.name == 'Project');
-    const itemGroups = project.elements.filter(item => item.name == 'ItemGroup');
-
-    const packageReference: Array<Element> = [];
-    itemGroups.forEach(itemGroup => {
-      itemGroup.elements.forEach(item => {
-        if (item.name == "PackageReference")
-          packageReference.push(item);
-      });
-    });
-
-    //Generates purls
-    packageReference.forEach(itemPackage => {
-      const packageName = itemPackage.attributes?.Include?.toString();
-      const version = itemPackage.attributes?.Version?.toString();
-
-      results.purls.push({
-        purl: new PackageURL(PURL_TYPE, null, packageName, null, null,null).toString(),
-        requirement: version
-      })
-    });
-
-  } catch (e) {
-    console.error(e);
-    return Promise.resolve({file: filePath, purls: []});
-  }
-
-  return Promise.resolve(results)
-}
-
-
-export function packagesConfigParser(fileContent: string, filePath: string): Promise<ILocalDependency> {
-
-  const results: ILocalDependency = { file: filePath, purls: [] };
-  const packageConfig = xml.xml2js(fileContent);
-
-  const packages = packageConfig.elements.filter(item => item.name == "packages")
-
-  packages.forEach(pkg => {
-    pkg.elements.forEach(dep => {
-      const depName = dep.attributes?.id;
-      const depVersion = dep.attributes?.version;
-
-      results.purls.push({
-        purl: new PackageURL(PURL_TYPE, null, depName, null, null, null).toString(),
-        requirement: depVersion
-      })
-    })
-  });
-
-  return Promise.resolve(results);
-}

package/src/sdk/Dependencies/LocalDependency/parsers/pyParser.ts

@@ -1,58 +0,0 @@
-import path from "path";
-import { PackageURL } from "packageurl-js";
-import { ILocalDependency } from "../DependencyTypes";
-import { isValidPath, isValidUrl } from './utils';
-
-
-function parseDep (str: string) {
-  const res = /^(?<name>[-\w]+)\s*(?<sym>[>=~!]*)\s*(?<version>[\d\.]*)/.exec(str);
-  return {
-    name: res?.groups?.name,
-    sym: res?.groups?.sym,
-    version: res?.groups?.version,
-  };
-}
-
-const PURL_TYPE = 'pypi';
-
-// Parse a requirements.txt file from python projects
-// See reference on: https://pip.pypa.io/en/stable/reference/requirements-file-format/
-const MANIFEST_FILE = 'requirements.txt';
-export function requirementsParser(fileContent: string, filePath: string): Promise<ILocalDependency> {
-
-    // If the file is not a python manifest file, return an empty results
-    const results: ILocalDependency = {file: filePath, purls: []};
-    if(path.basename(filePath) != MANIFEST_FILE)
-        return Promise.resolve(results);
-
-    const lines: Array<string> = fileContent.split('\n');
-
-    for (let line of lines) {
-        line = line.trim();
-        if(line.length == 0) continue;
-        if(!line.startsWith('#') && line.length>0) { // Avoid comments and new lines
-            if(isValidUrl(line)) {
-                // For reference about the regex see https://www.rfc-editor.org/rfc/rfc3986#appendix-B
-                const res = line.match(/^(([^:\/?#]+):)?(\/\/([^\/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?/);
-                continue;
-            }
-            else if(isValidPath(line)) {continue;} // Do not parse local dependencies.
-            else if(line.startsWith('-r')) {continue;} // Recursive dependencies (NOT SUPPORTED YET)
-            else {
-
-                const dep = parseDep(line);
-                if (!dep.sym) {
-                  const purlString = new PackageURL(PURL_TYPE, undefined, dep.name, undefined, undefined, undefined).toString();
-                  results.purls.push({purl: purlString});
-                } else if (dep.sym === '==') {
-                  const purlString = new PackageURL(PURL_TYPE, undefined, dep.name, dep.version, undefined, undefined).toString();
-                  results.purls.push({purl: purlString});
-                } else {
-                  const purlString = new PackageURL(PURL_TYPE, undefined, dep.name, undefined, undefined, undefined).toString();
-                  results.purls.push({purl: purlString, requirement: dep.sym+dep.version});
-                }
-            }
-        }
-    }
-  return Promise.resolve(results);
-}