owasp-depscan 5.5.0__py3-none-any.whl → 6.0.0a3__py3-none-any.whl

depscan/lib/tomlparse.py

@@ -0,0 +1,116 @@
+"""
+Module for parsing command line arguments and TOML configuration files.
+
+This module provides a class, `ArgumentParser`, which extends the functionality
+of `argparse.ArgumentParser` by allowing users to specify default values for
+arguments in a TOML file, in addition to the command line.
+"""
+# Based on https://github.com/florianmahner/tomlparse/blob/main/tomlparse/argparse.py
+# MIT license
+import argparse
+import os
+from typing import Any, Dict, List, MutableMapping, Optional, Tuple
+
+try:
+    import tomllib
+except ImportError:
+    import tomli as tomllib
+
+
+class ArgumentParser(argparse.ArgumentParser):
+    """A wrapper of the argparse.ArgumentParser class that adds the ability to
+    specify the values for arguments using a TOML file.
+
+    This class extends the functionality of the standard argparse.ArgumentParser by allowing
+    users to specify default values for arguments in a TOML file, in addition to the command line.
+    We can use all functionalities from the argument parser as usual:
+
+    Example:
+        >>> from depscan.lib.tomlparse import argparse
+        >>> parser = argparse.ArgumentParser(description='Example argparse-toml app')
+        >>> parser.add_argument('--foo', type=int, help='An example argument')
+        >>> args = parser.parse_args()
+
+    The above code will work as with the standard argparse.ArgumentParser class. We can also
+    specify the default values for the arguments in a TOML file. For this the TOML ArgumentParser
+    has one additional argument: `--config`. The `--config` argument is used
+    to specify the path to the TOML file.
+
+    We have the following hierarchy of arguments:
+        1. Arguments passed through the command line are selected over TOML
+           arguments, even if both are passed
+        2. Arguments from the TOML file are preferred over the default arguments
+    """
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        default_config = os.path.join(os.getcwd(), ".config", "depscan.toml")
+        self.add_argument("--config", help="Path to the configuration file. Default: $PWD/.config/depscan.toml",
+                          default=os.getenv("DEPSCAN_CONFIG", default_config))
+
+    def extract_args(
+        self, args: Optional[List[str]] = None, namespace: Optional[object] = None
+    ) -> Tuple[argparse.Namespace, argparse.Namespace]:
+        """Find the default arguments of the argument parser if any and the
+        ones that are passed through the command line"""
+        default_args = super().parse_args([])
+        cmdl_args = super().parse_args(args, namespace)
+
+        return default_args, cmdl_args
+
+    def find_changed_args(
+        self, default_args: argparse.Namespace, sys_args: argparse.Namespace
+    ) -> List[str]:
+        """Find the arguments that have been changed from the command
+        line to replace the .toml arguments"""
+        default_dict = vars(default_args)
+        sys_dict = vars(sys_args)
+        changed_dict = []
+        for key, value in default_dict.items():
+            sys_value = sys_dict[key]
+            if sys_value != value:
+                changed_dict.append(key)
+        return changed_dict
+
+    def load_toml(self, path: str) -> MutableMapping[str, Any]:
+        try:
+            with open(path, "rb") as f:
+                config = tomllib.load(f)
+        except FileNotFoundError:
+            self.error(f'Configuration file "{path}" doesn\'t exist')
+        return config
+
+    def remove_nested_keys(self, dictionary: Dict[str, Any]) -> Dict[str, Any]:
+        new_dict = {}
+        for key, value in dictionary.items():
+            if not isinstance(value, dict):
+                new_dict[key] = value
+        return new_dict
+
+    def parse_args(
+        self, args: Optional[List[str]] = None, namespace: Optional[object] = None
+    ) -> argparse.Namespace:
+        """Parse the arguments from the command line and the TOML file
+        and return the updated arguments. Same functionality as the
+        `argparse.ArgumentParser.parse_args` method."""
+        default_args, sys_args = self.extract_args(args, namespace)
+        config = sys_args.config
+        # These are the default arguments options updated by the command line
+        if not config or not os.path.exists(config):
+            return sys_args
+
+        # If a config file is passed, update the cmdl args with the config file unless
+        # the argument is already specified in the command line
+        toml_data = self.load_toml(config)
+        changed_args = self.find_changed_args(default_args, sys_args)
+        toml_args = self.remove_nested_keys(toml_data)
+
+        # Replaced unchanged command line arguments with arguments from
+        # the TOML file.
+        for key, value in toml_args.items():
+            if key not in changed_args:
+                setattr(sys_args, key, value)
+                # Support both hyphen and underscore representations
+                setattr(sys_args, key.replace("-", "_"), value)
+
+        return sys_args

depscan/lib/utils.py

@@ -1,19 +1,16 @@
 import ast
-import json
 import os
 import re
 import shutil
-from collections import defaultdict
 from datetime import datetime
-from importlib.metadata import distribution
 
+from custom_json_diff.lib.utils import file_read, file_write, json_load
 from jinja2 import Environment
-from vdb.lib import db as db_lib
-from vdb.lib.utils import version_compare
 
-from depscan.lib import config, normalize
+from depscan.lib.config import ignore_directories
+from depscan.lib.logger import LOG
 
-lic_symbol_regex = re.compile(r"[(),]")
+LIC_SYMBOL_REGEX = re.compile(r"[(),]")
 
 
 def filter_ignored_dirs(dirs):
@@ -26,7 +23,7 @@ def filter_ignored_dirs(dirs):
     [
         dirs.remove(d)
         for d in list(dirs)
-        if d.lower() in config.ignore_directories or d.startswith(".")
+        if d.lower() in ignore_directories or d.startswith(".")
     ]
     return dirs
 
@@ -49,24 +46,22 @@ def find_python_reqfiles(path):
     ]
     for root, dirs, files in os.walk(path):
         filter_ignored_dirs(dirs)
-        for name in req_files:
-            if name in files:
-                result.append(os.path.join(root, name))
+        result.extend(os.path.join(root, name) for name in req_files if name in files)
     return result
 
 
-def find_files(src, src_ext_name, quick=False, filter=True):
+def find_files(src, src_ext_name, quick=False, filter_dirs=True):
     """
     Method to find files with given extension
 
     :param src: source directory to search
     :param src_ext_name: type of source file
     :param quick: only return first match found
-    :param filter: filter out ignored directories
+    :param filter_dirs: filter out ignored directories
     """
     result = []
     for root, dirs, files in os.walk(src):
-        if filter:
+        if filter_dirs:
             filter_ignored_dirs(dirs)
         for file in files:
             if file == src_ext_name or file.endswith(src_ext_name):
@@ -80,9 +75,7 @@ def is_binary_string(content):
     """
     Method to check if the given content is a binary string
     """
-    textchars = bytearray(
-        {7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7F}
-    )
+    textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7F})
     return bool(content.translate(None, textchars))
 
 
@@ -171,7 +164,7 @@ def detect_project_type(src_dir):
         os.path.join(src_dir, ".github", "workflows"),
         ".yml",
         quick=True,
-        filter=False,
+        filter_dirs=False,
     ):
         project_types.append("github")
     # jars
@@ -187,125 +180,6 @@ def detect_project_type(src_dir):
     return project_types
 
 
-def get_pkg_vendor_name(pkg):
-    """
-    Method to extract vendor and name information from package. If vendor
-    information is not available package url is used to extract the package
-    registry provider such as pypi, maven
-
-    :param pkg: a dictionary representing a package
-    :return: vendor and name as a tuple
-    """
-    vendor = pkg.get("vendor")
-    if not vendor:
-        purl = pkg.get("purl")
-        if purl:
-            purl_parts = purl.split("/")
-            if purl_parts:
-                vendor = purl_parts[0].replace("pkg:", "")
-        else:
-            vendor = ""
-    name = pkg.get("name")
-    return vendor, name
-
-
-def search_pkgs(db, project_type, pkg_list):
-    """
-    Method to search packages in our vulnerability database
-
-    :param db: DB instance
-    :param project_type: Project type
-    :param pkg_list: List of packages to search
-    :returns: raw_results, pkg_aliases, purl_aliases
-    """
-    expanded_list = []
-    # The challenge we have is to broaden our search and create several
-    # variations of the package and vendor names to perform a broad search.
-    # We then have to map the results back to the original package names and
-    # package urls.
-    pkg_aliases = defaultdict(list)
-    purl_aliases = {}
-    for pkg in pkg_list:
-        variations = normalize.create_pkg_variations(pkg)
-        if variations:
-            expanded_list += variations
-        vendor, name = get_pkg_vendor_name(pkg)
-        version = pkg.get("version")
-        if pkg.get("purl"):
-            ppurl = pkg.get("purl")
-            purl_aliases[pkg.get("purl")] = pkg.get("purl")
-            purl_aliases[f"{vendor.lower()}:{name.lower()}:{version}"] = ppurl
-            if ppurl.startswith("pkg:npm"):
-                purl_aliases[f"npm:{vendor.lower()}/{name.lower()}:{version}"] = ppurl
-            if not purl_aliases.get(f"{vendor.lower()}:{name.lower()}"):
-                purl_aliases[f"{vendor.lower()}:{name.lower()}"] = ppurl
-        if variations:
-            for vari in variations:
-                vari_full_pkg = f"""{vari.get("vendor")}:{vari.get("name")}"""
-                pkg_aliases[
-                    f"{vendor.lower()}:{name.lower()}:{version}"
-                ].append(vari_full_pkg)
-                if pkg.get("purl"):
-                    purl_aliases[f"{vari_full_pkg.lower()}:{version}"] = pkg.get("purl")
-    quick_res = db_lib.bulk_index_search(expanded_list)
-    raw_results = db_lib.pkg_bulk_search(db, quick_res)
-    raw_results = normalize.dedup(project_type, raw_results)
-    pkg_aliases = normalize.dealias_packages(
-        raw_results,
-        pkg_aliases=pkg_aliases,
-        purl_aliases=purl_aliases,
-    )
-    return raw_results, pkg_aliases, purl_aliases
-
-
-def get_pkgs_by_scope(pkg_list):
-    """
-    Method to return the packages by scope as defined in CycloneDX spec -
-    required, optional and excluded
-
-    :param pkg_list: List of packages
-    :return: Dictionary of packages categorized by scope if available. Empty if
-                no scope information is available
-    """
-    scoped_pkgs = {}
-    for pkg in pkg_list:
-        if pkg.get("scope"):
-            vendor, name = get_pkg_vendor_name(pkg)
-            scope = pkg.get("scope").lower()
-            if pkg.get("purl"):
-                scoped_pkgs.setdefault(scope, []).append(pkg.get("purl"))
-            else:
-                scoped_pkgs.setdefault(scope, []).append(f"{vendor}:{name}")
-    return scoped_pkgs
-
-
-def get_scope_from_imports(project_type, pkg_list, all_imports):
-    """
-    Method to compute the packages scope defined in CycloneDX spec - required,
-    optional and excluded
-
-    :param project_type: Project type
-    :param pkg_list: List of packages
-    :param all_imports: List of imports detected
-    :return: Dictionary of packages categorized by scope if available. Empty if
-                no scope information is available
-    """
-    scoped_pkgs = {}
-    if not pkg_list or not all_imports:
-        return scoped_pkgs
-    for pkg in pkg_list:
-        scope = "optional"
-        vendor, name = get_pkg_vendor_name(pkg)
-        if name in all_imports or name.lower().replace("py", "") in all_imports:
-            scope = "required"
-        if pkg.get("purl"):
-            scoped_pkgs.setdefault(scope, []).append(pkg.get("purl"))
-        else:
-            scoped_pkgs.setdefault(scope, []).append(f"{vendor}:{name}")
-        scoped_pkgs[scope].append(f"{project_type}:{name.lower()}")
-    return scoped_pkgs
-
-
 def cleanup_license_string(license_str):
     """
     Method to clean up license string by removing problematic symbols and
@@ -322,33 +196,10 @@ def cleanup_license_string(license_str):
         .replace(" & ", " OR ")
         .replace("&", " OR ")
     )
-    license_str = lic_symbol_regex.sub("", license_str)
+    license_str = LIC_SYMBOL_REGEX.sub("", license_str)
     return license_str.upper()
 
 
-def max_version(version_list):
-    """
-    Method to return the highest version from the list
-
-    :param version_list: single version string or set of versions
-    :return: max version
-    """
-    if isinstance(version_list, str):
-        return version_list
-    if isinstance(version_list, set):
-        version_list = list(version_list)
-    if len(version_list) == 1:
-        return version_list[0]
-    min_ver = "0"
-    max_ver = version_list[0]
-    for i, vl in enumerate(version_list):
-        if not vl:
-            continue
-        if not version_compare(vl, min_ver, max_ver):
-            max_ver = vl
-    return max_ver
-
-
 def get_all_imports(src_dir):
     """
     Method to collect all package imports from a python file
@@ -359,9 +210,7 @@ def get_all_imports(src_dir):
     if not py_files:
         return import_list
     for afile in py_files:
-        with open(os.path.join(afile), "rb", encoding="utf-8") as f:
-            content = f.read()
-        parsed = ast.parse(content)
+        parsed = ast.parse(file_read(os.path.join(afile), True, log=LOG))
         for node in ast.walk(parsed):
             if isinstance(node, ast.Import):
                 for name in node.names:
@@ -379,18 +228,11 @@ def get_all_imports(src_dir):
     return import_list
 
 
-def get_version():
-    """
-    Returns the version of depscan
-    """
-    return distribution("owasp-depscan").version
-
-
 def export_pdf(
     html_file,
     pdf_file,
     title="DepScan Analysis",
-    footer=f'Report generated by OWASP dep-scan at {datetime.now().strftime("%B %d, %Y %H:%M")}',
+    footer=f"Report generated by OWASP dep-scan at {datetime.now().strftime('%B %d, %Y %H:%M')}",
 ):
     """
     Method to export html as pdf using pdfkit
@@ -428,31 +270,35 @@ def render_template_report(
     summary,
     template_file,
     result_file,
+    depscan_options={},
 ):
     """
     Render the given vdr_file (falling back to bom_file if no vdr was written)
     and summary dict using the template_file with Jinja, rendered output is written
     to named result_file in reports directory.
     """
-    if vdr_file and os.path.isfile(vdr_file):
-        with open(vdr_file, "r", encoding="utf-8") as f:
-            bom = json.load(f)
-    else:
-        with open(bom_file, "r", encoding="utf-8") as f:
-            bom = json.load(f)
-    with open(template_file, "r", encoding="utf-8") as tmpl_file:
-        template = tmpl_file.read()
-    jinja_env = Environment(autoescape=False)
+    bom = {}
+    if vdr_file:
+        bom = json_load(vdr_file, log=LOG)
+    if not bom:
+        bom = json_load(bom_file, log=LOG)
+    template = file_read(template_file, log=LOG)
+    jinja_env = Environment(autoescape=True)
     jinja_tmpl = jinja_env.from_string(template)
     report_result = jinja_tmpl.render(
-        metadata=bom.get("metadata", None),
-        vulnerabilities=bom.get("vulnerabilities", None),
-        components=bom.get("components", None),
-        dependencies=bom.get("dependencies", None),
-        services=bom.get("services", None),
+        metadata=bom.get("metadata"),
+        vulnerabilities=bom.get("vulnerabilities"),
+        components=bom.get("components"),
+        dependencies=bom.get("dependencies"),
+        services=bom.get("services"),
         summary=summary,
         pkg_vulnerabilities=pkg_vulnerabilities,
         pkg_group_rows=pkg_group_rows,
     )
-    with open(result_file, "w", encoding="utf-8") as outfile:
-        outfile.write(report_result)
+    file_write(
+        result_file,
+        report_result,
+        error_msg=f"Failed to export report: {result_file}",
+        success_msg=f"Report written to {result_file}.",
+        log=LOG,
+    )