owasp-depscan 6.0.0a3__py3-none-any.whl → 6.0.0b4__py3-none-any.whl

depscan/cli.py

@@ -2,10 +2,8 @@
 # -*- coding: utf-8 -*-
 
 import contextlib
-import json
 import os
 import sys
-import tempfile
 from typing import List
 
 from analysis_lib import (
@@ -24,7 +22,7 @@ from analysis_lib.utils import (
 )
 from analysis_lib.vdr import VDRAnalyzer
 from analysis_lib.reachability import get_reachability_impl
-from custom_json_diff.lib.utils import file_write, json_load
+from custom_json_diff.lib.utils import json_load
 from rich.panel import Panel
 from rich.terminal_theme import DEFAULT_TERMINAL_THEME, MONOKAI
 from vdb.lib import config
@@ -54,6 +52,8 @@ from depscan.lib.config import (
 from depscan.lib.license import build_license_data, bulk_lookup
 from depscan.lib.logger import DEBUG, LOG, SPINNER, console, IS_CI
 
+from reporting_lib.htmlgen import ReportGenerator
+
 if sys.platform == "win32" and os.environ.get("PYTHONIOENCODING") is None:
     sys.stdin.reconfigure(encoding="utf-8")
     sys.stdout.reconfigure(encoding="utf-8")
@@ -65,14 +65,11 @@ LOGO = """
          |
 """
 
-QUART_AVAILABLE = False
+SERVER_LIB = None
 try:
-    from quart import Quart, request
+    from server_lib import simple, ServerOptions
 
-    app = Quart(__name__, static_folder=None)
-    app.config.from_prefixed_env()
-    app.config["PROVIDE_AUTOMATIC_OPTIONS"] = True
-    QUART_AVAILABLE = True
+    SERVER_LIB = simple
 except ImportError:
     pass
 
@@ -100,6 +97,7 @@ def vdr_analyze_summarize(
     scoped_pkgs,
     bom_file,
     bom_dir,
+    reports_dir,
     pkg_list,
     reachability_analyzer,
     reachability_options,
@@ -115,6 +113,7 @@ def vdr_analyze_summarize(
     :param scoped_pkgs: Dict containing package scopes.
     :param bom_file: Single BOM file.
     :param bom_dir: Directory containining bom files.
+    :param reports_dir: Directory containining report files.
     :param pkg_list: Direct list of packages when the bom file is empty.
     :param reachability_analyzer: Reachability Analyzer specified.
     :param reachability_options: Reachability Analyzer options.
@@ -165,7 +164,11 @@ def vdr_analyze_summarize(
     )
     ds_version = get_version()
     vdr_result = VDRAnalyzer(vdr_options=options).process()
-    vdr_file = bom_file.replace(".cdx.json", ".vdr.json") if bom_file else None
+    # Set vdr_file in report folder
+    vdr_file = (
+        os.path.join(reports_dir, os.path.basename(bom_file)) if bom_file else None
+    )
+    vdr_file = vdr_file.replace(".cdx.json", ".vdr.json") if vdr_file else None
     if not vdr_file and bom_dir:
         vdr_file = os.path.join(bom_dir, DEPSCAN_DEFAULT_VDR_FILE)
     if vdr_result.success:
@@ -231,237 +234,6 @@ def set_project_types(args, src_dir):
     return pkg_list, project_types_list
 
 
-if QUART_AVAILABLE:
-
-    @app.get("/")
-    async def index():
-        """
-
-        :return: An empty dictionary
-        """
-        return {}
-
-    @app.get("/download-vdb")
-    async def download_vdb():
-        """
-
-        :return: a JSON response indicating the status of the caching operation.
-        """
-        if db_lib.needs_update(days=0, hours=VDB_AGE_HOURS, default_status=False):
-            if not ORAS_AVAILABLE:
-                return {
-                    "error": "true",
-                    "message": "The oras package must be installed to automatically download the vulnerability database. Install depscan using `pip install owasp-depscan[all]` or use the official container image.",
-                }
-            if download_image(vdb_database_url, config.DATA_DIR):
-                return {
-                    "error": "false",
-                    "message": "vulnerability database downloaded successfully",
-                }
-            return {
-                "error": "true",
-                "message": "vulnerability database did not get downloaded correctly. Check the server logs.",
-            }
-        return {
-            "error": "false",
-            "message": "vulnerability database already exists",
-        }
-
-    @app.route("/scan", methods=["GET", "POST"])
-    async def run_scan():
-        """
-        :return: A JSON response containing the SBOM file path and a list of
-        vulnerabilities found in the scanned packages
-        """
-        q = request.args
-        params = await request.get_json()
-        uploaded_bom_file = await request.files
-
-        url = None
-        path = None
-        multi_project = None
-        project_type = None
-        results = []
-        profile = "generic"
-        deep = False
-        suggest_mode = True if q.get("suggest") in ("true", "1") else False
-        fuzzy_search = True if q.get("fuzzy_search") in ("true", "1") else False
-        if q.get("url"):
-            url = q.get("url")
-        if q.get("path"):
-            path = q.get("path")
-        if q.get("multiProject"):
-            multi_project = q.get("multiProject", "").lower() in ("true", "1")
-        if q.get("deep"):
-            deep = q.get("deep", "").lower() in ("true", "1")
-        if q.get("type"):
-            project_type = q.get("type")
-        if q.get("profile"):
-            profile = q.get("profile")
-        if params is not None:
-            if not url and params.get("url"):
-                url = params.get("url")
-            if not path and params.get("path"):
-                path = params.get("path")
-            if not multi_project and params.get("multiProject"):
-                multi_project = params.get("multiProject", "").lower() in (
-                    "true",
-                    "1",
-                )
-            if not deep and params.get("deep"):
-                deep = params.get("deep", "").lower() in (
-                    "true",
-                    "1",
-                )
-            if not project_type and params.get("type"):
-                project_type = params.get("type")
-            if not profile and params.get("profile"):
-                profile = params.get("profile")
-
-        if not path and not url and (uploaded_bom_file.get("file", None) is None):
-            return {
-                "error": "true",
-                "message": "path or url or a bom file upload is required",
-            }, 400
-        if not project_type:
-            return {"error": "true", "message": "project type is required"}, 400
-        if db_lib.needs_update(days=0, hours=VDB_AGE_HOURS, default_status=False):
-            return (
-                {
-                    "error": "true",
-                    "message": "Vulnerability database is empty. Prepare the "
-                    "vulnerability database by invoking /download-vdb endpoint "
-                    "before running scans.",
-                },
-                500,
-                {"Content-Type": "application/json"},
-            )
-
-        cdxgen_server = app.config.get("CDXGEN_SERVER_URL")
-        bom_file_path = None
-
-        if uploaded_bom_file.get("file", None) is not None:
-            bom_file = uploaded_bom_file["file"]
-            bom_file_content = bom_file.read().decode("utf-8")
-            try:
-                _ = json.loads(bom_file_content)
-            except Exception as e:
-                LOG.info(e)
-                return (
-                    {
-                        "error": "true",
-                        "message": "The uploaded file must be a valid JSON or XML.",
-                    },
-                    400,
-                    {"Content-Type": "application/json"},
-                )
-
-            LOG.debug("Processing uploaded file")
-            bom_file_suffix = str(bom_file.filename).rsplit(".", maxsplit=1)[-1]
-            tmp_bom_file = tempfile.NamedTemporaryFile(
-                delete=False, suffix=f".bom.{bom_file_suffix}"
-            )
-            path = tmp_bom_file.name
-            file_write(path, bom_file_content)
-
-        # Path points to a project directory
-        # Bug# 233. Path could be a url
-        if url or (path and os.path.isdir(path)):
-            with tempfile.NamedTemporaryFile(delete=False, suffix=".bom.json") as bfp:
-                project_type_list = project_type.split(",")
-                bom_status = create_bom(
-                    bfp.name,
-                    path,
-                    {
-                        "url": url,
-                        "path": path,
-                        "project_type": project_type_list,
-                        "multiProject": multi_project,
-                        "cdxgen_server": cdxgen_server,
-                        "profile": profile,
-                        "deep": deep,
-                    },
-                )
-                if bom_status:
-                    LOG.debug("BOM file was generated successfully at %s", bfp.name)
-                    bom_file_path = bfp.name
-
-        # Path points to a SBOM file
-        else:
-            if os.path.exists(path):
-                bom_file_path = path
-        # Direct purl-based lookups are not supported yet.
-        if bom_file_path is not None:
-            pkg_list, _ = get_pkg_list(bom_file_path)
-            # Here we are assuming there will be only one type
-            if project_type in type_audit_map:
-                audit_results = audit(project_type, pkg_list)
-                if audit_results:
-                    results = results + audit_results
-            if not pkg_list:
-                LOG.debug("Empty package search attempted!")
-            else:
-                LOG.debug("Scanning %d oss dependencies for issues", len(pkg_list))
-            bom_data = json_load(bom_file_path)
-            if not bom_data:
-                return (
-                    {
-                        "error": "true",
-                        "message": "Unable to generate SBOM. Check your input path or url.",
-                    },
-                    400,
-                    {"Content-Type": "application/json"},
-                )
-            options = VdrAnalysisKV(
-                project_type,
-                results,
-                pkg_aliases={},
-                purl_aliases={},
-                suggest_mode=suggest_mode,
-                scoped_pkgs={},
-                no_vuln_table=True,
-                bom_file=bom_file_path,
-                pkg_list=[],
-                direct_purls={},
-                reached_purls={},
-                console=console,
-                logger=LOG,
-                fuzzy_search=fuzzy_search,
-            )
-            vdr_result = VDRAnalyzer(vdr_options=options).process()
-            if vdr_result.success:
-                pkg_vulnerabilities = vdr_result.pkg_vulnerabilities
-                if pkg_vulnerabilities:
-                    bom_data["vulnerabilities"] = pkg_vulnerabilities
-                return json.dumps(bom_data), 200, {"Content-Type": "application/json"}
-        return (
-            {
-                "error": "true",
-                "message": "Unable to generate SBOM. Check your input path or url.",
-            },
-            500,
-            {"Content-Type": "application/json"},
-        )
-
-    def run_server(args):
-        """
-        Run depscan as server
-
-        :param args: Command line arguments passed to the function.
-        """
-        print(LOGO)
-        console.print(
-            f"Depscan server running on {args.server_host}:{args.server_port}"
-        )
-        app.config["CDXGEN_SERVER_URL"] = args.cdxgen_server
-        app.run(
-            host=args.server_host,
-            port=args.server_port,
-            debug=os.getenv("SCAN_DEBUG_MODE") == "debug",
-            use_reloader=False,
-        )
-
-
 def run_depscan(args):
     """
     Detects the project type, performs various scans and audits,
@@ -510,8 +282,20 @@ def run_depscan(args):
         os.environ["CDXGEN_DEBUG_MODE"] = "debug"
         LOG.setLevel(DEBUG)
     if args.server_mode:
-        if QUART_AVAILABLE:
-            return run_server(args)
+        if SERVER_LIB:
+            server_options = ServerOptions(
+                server_host=args.server_host,
+                server_port=args.server_port,
+                cdxgen_server=args.cdxgen_server,
+                allowed_hosts=args.server_allowed_hosts,
+                allowed_paths=args.server_allowed_paths,
+                console=console,
+                logger=LOG,
+                debug=args.enable_debug or os.environ.get("SCAN_DEBUG_MODE") == "debug",
+                create_bom=create_bom,
+                max_content_length=os.getenv("DEPSCAN_SERVER_MAX_CONTENT_LENGTH"),
+            )
+            return simple.run_server(server_options)
         else:
             LOG.info(
                 "The required packages for server mode are unavailable. Reinstall depscan using `pip install owasp-depscan[all]`."
@@ -557,9 +341,7 @@ def run_depscan(args):
     bom_dir_mode = args.bom_dir and os.path.exists(args.bom_dir)
     # Are we running with a config file
     config_file_mode = args.config and os.path.exists(args.config)
-    depscan_options = {**vars(args)}
-    depscan_options["src_dir"] = src_dir
-    depscan_options["reports_dir"] = reports_dir
+    depscan_options = {**vars(args), "src_dir": src_dir, "reports_dir": reports_dir}
     # Is the user looking for semantic analysis?
     # We can default to this when run against a BOM directory
     if (
@@ -617,15 +399,11 @@ def run_depscan(args):
     html_report_file = depscan_options.get(
         "html_report_file", os.path.join(reports_dir, "depscan.html")
     )
-    pdf_report_file = depscan_options.get(
-        "pdf_report_file", os.path.join(reports_dir, "depscan.pdf")
-    )
     txt_report_file = depscan_options.get(
         "txt_report_file", os.path.join(reports_dir, "depscan.txt")
     )
     run_config_file = os.path.join(reports_dir, "depscan.toml.sample")
     depscan_options["html_report_file"] = html_report_file
-    depscan_options["pdf_report_file"] = pdf_report_file
     depscan_options["txt_report_file"] = txt_report_file
     # Create reports directory
     if reports_dir and not os.path.exists(reports_dir):
@@ -934,6 +712,7 @@ def run_depscan(args):
             scoped_pkgs=scoped_pkgs,
             bom_file=bom_files[0] if len(bom_files) == 1 else None,
             bom_dir=args.bom_dir,
+            reports_dir=args.reports_dir,
             pkg_list=pkg_list,
             reachability_analyzer=reachability_analyzer,
             reachability_options=reachability_options,
@@ -975,7 +754,13 @@ def run_depscan(args):
         theme=(MONOKAI if os.getenv("USE_DARK_THEME") else DEFAULT_TERMINAL_THEME),
     )
     console.save_text(txt_report_file, clear=False)
-    utils.export_pdf(html_report_file, pdf_report_file)
+    # Prettify the rich html report
+    html_report_generator = ReportGenerator(
+        input_rich_html_path=html_report_file,
+        report_output_path=html_report_file,
+        raw_content=False,
+    )
+    html_report_generator.parse_and_generate_report()
     # This logic needs refactoring
     # render report into template if wished
     if args.report_template and os.path.isfile(args.report_template):

depscan/cli_options.py

@@ -245,6 +245,19 @@ def build_parser():
         dest="server_port",
         help="depscan server port",
     )
+    parser.add_argument(
+        "--server-allowed-hosts",
+        nargs="*",
+        help="List of allowed hostnames or IPs that can access the server (e.g., 'localhost 192.168.1.10'). If unspecified, no host allowlist is enforced.",
+        default=None,
+    )
+
+    parser.add_argument(
+        "--server-allowed-paths",
+        nargs="*",
+        help="List of allowed filesystem paths that can be scanned by the server. Restricts `path` parameter in /scan requests.",
+        default=None,
+    )
     parser.add_argument(
         "--cdxgen-server",
         default=os.getenv("CDXGEN_SERVER_URL"),

depscan/lib/bom.py

@@ -556,7 +556,11 @@ def annotate_vdr(vdr_file, txt_report_file):
         return
     vdr = json_load(vdr_file)
     metadata = vdr.get("metadata", {})
-    tools = metadata.get("tools", {}).get("components", {})
+    # Some cyclonedx sbom don't containg tools.components
+    if "components" in metadata.get("tools"):
+        tools = metadata.get("tools", {}).get("components", {})
+    else:
+        tools = {}
     with open(txt_report_file, errors="ignore", encoding="utf-8") as txt_fp:
         report = txt_fp.read()
         annotations = vdr.get("annotations", []) or []

depscan/lib/explainer.py

@@ -33,7 +33,7 @@ def explain(project_type, src_dir, bom_dir, vdr_file, vdr_result, explanation_mo
     pattern_methods = {}
     has_any_explanation = False
     has_any_crypto_flows = False
-    slices_files = glob.glob(f"{bom_dir}/**/*reachables.slices.json", recursive=True)
+    slices_files = glob.glob(f"{bom_dir}/**/*reachables.slices*.json", recursive=True)
     openapi_spec_files = None
     # Should we explain the endpoints and Code Hotspots
     if explanation_mode in (
@@ -47,9 +47,14 @@ def explain(project_type, src_dir, bom_dir, vdr_file, vdr_result, explanation_mo
         rsection = Markdown("""## Service Endpoints
 
 The following endpoints and code hotspots were identified by depscan. Verify that proper authentication and authorization mechanisms are in place to secure them.""")
-        console.print(rsection)
+        any_endpoints_shown = False
         for ospec in openapi_spec_files:
-            pattern_methods = print_endpoints(ospec)
+            pattern_methods = print_endpoints(
+                ospec, rsection if not any_endpoints_shown else None
+            )
+            if not any_endpoints_shown and pattern_methods:
+                any_endpoints_shown = True
+
     # Return early for endpoints only explanations
     if explanation_mode in ("Endpoints",):
         return
@@ -65,9 +70,10 @@ The following endpoints and code hotspots were identified by depscan. Verify tha
             if "-" in fn:
                 section_label = f"# Explanations for {fn.split('-')[0].upper()}"
             console.print(Markdown(section_label))
-        if (reachables_data := json_load(sf, log=LOG)) and reachables_data.get(
-            "reachables"
-        ):
+        if reachables_data := json_load(sf, log=LOG):
+            # Backwards compatibility
+            if isinstance(reachables_data, dict) and reachables_data.get("reachables"):
+                reachables_data = reachables_data.get("reachables")
             if explanation_mode in ("NonReachables",):
                 rsection = Markdown(
                     f"""## {section_title}
@@ -109,7 +115,7 @@ def _track_usage_targets(usage_targets, usages_object):
                 usage_targets.add(f"{file}#{l}")
 
 
-def print_endpoints(ospec):
+def print_endpoints(ospec, header_section=None):
     if not ospec:
         return
     paths = json_load(ospec).get("paths") or {}
@@ -151,6 +157,9 @@ def print_endpoints(ospec):
         sorted_areas.sort()
         rtable.add_row(k, ("\n".join(v)).upper(), "\n".join(sorted_areas))
     if pattern_methods:
+        # Print the header section
+        if header_section:
+            console.print(header_section)
         console.print()
         console.print(rtable)
     return pattern_methods
@@ -178,13 +187,17 @@ def explain_reachables(
     reachable_explanations = 0
     checked_flows = 0
     has_crypto_flows = False
+    explained_ids = {}
     purls_reachable_explanations = defaultdict(int)
     source_reachable_explanations = defaultdict(int)
     sink_reachable_explanations = defaultdict(int)
     has_explanation = False
     header_shown = False
     has_cpp_flow = False
-    for areach in reachables.get("reachables", []):
+    # Backwards compatibility
+    if isinstance(reachables, dict) and reachables.get("reachables"):
+        reachables = reachables.get("reachables")
+    for areach in reachables:
         cpp_flow = is_cpp_flow(areach.get("flows"))
         if not has_cpp_flow and cpp_flow:
             has_cpp_flow = True
@@ -194,16 +207,9 @@ def explain_reachables(
             or (not areach.get("purls") and not cpp_flow)
         ):
             continue
-        # Focus only on the prioritized list if available
-        # if project_type in ("java",) and pkg_group_rows:
-        #     is_prioritized = False
-        #     for apurl in areach.get("purls"):
-        #         if pkg_group_rows.get(apurl):
-        #             is_prioritized = True
-        #     if not is_prioritized:
-        #         continue
         (
             flow_tree,
+            added_ids,
             comment,
             source_sink_desc,
             source_code_str,
@@ -218,7 +224,13 @@ def explain_reachables(
             project_type,
             vdr_result,
         )
-        if not source_sink_desc or not flow_tree or len(flow_tree.children) < 5:
+        # The goal is to reduce duplicate explanations by checking if a given flow is similar to one we have explained
+        # before. We do this by checking the node ids, source-sink explanations, purl tags and so on.
+        added_ids_str = "-".join(added_ids)
+        # Have we seen this sequence before?
+        if explained_ids.get(added_ids_str) or len(added_ids) < 4:
+            continue
+        if not source_sink_desc or not flow_tree or len(flow_tree.children) < 4:
             continue
         # In non-reachables mode, we are not interested in reachable flows.
         if (
@@ -269,6 +281,7 @@ def explain_reachables(
             header_shown = True
         console.print()
         console.print(rtable)
+        explained_ids[added_ids_str] = True
         reachable_explanations += 1
         if purls_str:
             purls_reachable_explanations[purls_str] += 1
@@ -428,7 +441,7 @@ def filter_tags(tags):
 
 
 def is_filterable_code(project_type, code):
-    if len(code) < 5:
+    if len(code) < 3:
         return True
     for c in (
         "console.log",
@@ -455,8 +468,16 @@ def flow_to_str(explanation_mode, flow, project_type):
         and flow.get("lineNumber")
         and not flow.get("parentFileName").startswith("unknown")
     ):
-        file_loc = f"{flow.get('parentFileName').replace('src/main/java/', '').replace('src/main/scala/', '')}#{flow.get('lineNumber')}    "
+        # strip common prefixes
+        name = flow.get("parentFileName", "")
+        for p in ("src/main/java/", "src/main/scala/"):
+            name = name.removeprefix(p)
+        file_loc = f"{name}#{flow.get('lineNumber')}    "
     node_desc = flow.get("code").split("\n")[0]
+    if (len(node_desc) < 3 or node_desc.endswith("{")) and len(flow.get("code")) > 3:
+        node_desc = " ".join(flow.get("code", "").split())
+        if "(" in node_desc:
+            node_desc = node_desc.split("(")[0] + "() ..."
     if node_desc.endswith("("):
         node_desc = f":diamond_suit: {node_desc})"
     elif node_desc.startswith("return "):
@@ -510,6 +531,7 @@ def explain_flows(explanation_mode, flows, purls, project_type, vdr_result):
     if purls:
         purls_str = "\n".join(purls)
         comments.append(f"[info]Reachable Packages:[/info]\n{purls_str}")
+    added_ids = []
     added_flows = []
     added_node_desc = []
     has_check_tag = False
@@ -547,6 +569,7 @@ def explain_flows(explanation_mode, flows, purls, project_type, vdr_result):
         if flow_str in added_flows or node_desc in added_node_desc:
             continue
         added_flows.append(flow_str)
+        added_ids.append(str(aflow.get("id", "")))
         added_node_desc.append(node_desc)
         if not tree:
             tree = Tree(flow_str)
@@ -561,6 +584,7 @@ def explain_flows(explanation_mode, flows, purls, project_type, vdr_result):
         )
     return (
         tree,
+        added_ids,
         "\n".join(comments),
         source_sink_desc,
         source_code_str,

depscan/lib/utils.py

@@ -1,8 +1,6 @@
 import ast
 import os
 import re
-import shutil
-from datetime import datetime
 
 from custom_json_diff.lib.utils import file_read, file_write, json_load
 from jinja2 import Environment
@@ -84,11 +82,12 @@ def is_exe(src):
     Detect if the source is a binary file
 
     :param src: Source path
-    :return True if binary file. False otherwise.
+    :return: True if binary file. False otherwise.
     """
     if os.path.isfile(src):
         try:
-            return is_binary_string(open(src, "rb").read(1024))
+            with open(src, "rb") as f:
+                return is_binary_string(f.read(1024))
         except Exception:
             return False
     return False
@@ -228,40 +227,6 @@ def get_all_imports(src_dir):
     return import_list
 
 
-def export_pdf(
-    html_file,
-    pdf_file,
-    title="DepScan Analysis",
-    footer=f"Report generated by OWASP dep-scan at {datetime.now().strftime('%B %d, %Y %H:%M')}",
-):
-    """
-    Method to export html as pdf using pdfkit
-    """
-    pdf_options = {
-        "page-size": "A2",
-        "margin-top": "0.5in",
-        "margin-right": "0.25in",
-        "margin-bottom": "0.5in",
-        "margin-left": "0.25in",
-        "encoding": "UTF-8",
-        "outline": None,
-        "title": title,
-        "footer-right": footer,
-        "minimum-font-size": "12",
-        "disable-smart-shrinking": "",
-    }
-    if shutil.which("wkhtmltopdf"):
-        try:
-            import pdfkit
-
-            if not pdf_file and html_file:
-                pdf_file = html_file.replace(".html", ".pdf")
-            if os.path.exists(html_file):
-                pdfkit.from_file(html_file, pdf_file, options=pdf_options)
-        except Exception:
-            pass
-
-
 def render_template_report(
     vdr_file,
     bom_file,