owasp-depscan 5.4.8__py3-none-any.whl → 6.0.0a2__py3-none-any.whl

depscan/lib/logger.py

@@ -1,7 +1,8 @@
-# This file is part of Scan.
-
 import logging
 import os
+import random
+import re
+import sys
 
 from rich.console import Console
 from rich.highlighter import RegexHighlighter
@@ -10,7 +11,7 @@ from rich.theme import Theme
 
 
 class CustomHighlighter(RegexHighlighter):
-    base_style = "atom."
+    base_style = "depscan."
     highlights = [
         r"(?P<method>([\w-]+\.)+[\w-]+[^<>:(),]?)",
         r"(?P<path>(\w+\/.*\.[\w:]+))",
@@ -21,25 +22,29 @@ class CustomHighlighter(RegexHighlighter):
 
 custom_theme = Theme(
     {
-        "atom.path": "#7c8082",
-        "atom.params": "#5a7c90",
-        "atom.opers": "#7c8082",
-        "atom.method": "#FF753D",
+        "depscan.path": "#7c8082",
+        "depscan.params": "#5a7c90",
+        "depscan.opers": "#7c8082",
+        "depscan.method": "#FF753D",
         "info": "#5A7C90",
         "warning": "#FF753D",
         "danger": "bold red",
     }
 )
 
+IS_CI = os.getenv("CI") or os.getenv("CONTINUOUS_INTEGRATION")
+
 console = Console(
     log_time=False,
     log_path=False,
     theme=custom_theme,
-    color_system="256",
-    force_terminal=True,
-    highlight=True,
+    color_system=os.getenv("CONSOLE_COLOR_SCHEME", "256"),
+    width=140 if IS_CI else None,
+    highlight=not IS_CI,
+    tab_size=2,
     highlighter=CustomHighlighter(),
-    record=True,
+    record=sys.platform == "win32",
+    emoji=os.getenv("DISABLE_CONSOLE_EMOJI", "") not in ("true", "1"),
 )
 
 logging.basicConfig(
@@ -58,14 +63,57 @@ logging.basicConfig(
 LOG = logging.getLogger(__name__)
 
 # Set logging level
-if (
-    os.getenv("SCAN_DEBUG_MODE") == "debug"
-    or os.getenv("AT_DEBUG_MODE") == "debug"
-):
+if os.getenv("SCAN_DEBUG_MODE") == "debug":
     LOG.setLevel(logging.DEBUG)
 
 DEBUG = logging.DEBUG
-
 for log_name, log_obj in logging.Logger.manager.loggerDict.items():
-    if log_name != __name__:
+    if not log_name.startswith("depscan"):
         log_obj.disabled = True
+
+SPINNER = os.getenv(
+    "DEPSCAN_SPINNER",
+    random.choice(
+        [
+            "pong",
+            "arrow3",
+            "bouncingBall",
+            "dots2",
+            "material",
+            "shark",
+            "simpleDotsScrolling",
+            "toggle9",
+        ]
+    ),
+)
+
+# Support for thought logging
+tlogger = None
+
+
+def thought_log(s):
+    if s and tlogger and tlogger.isEnabledFor(DEBUG):
+        s = re.sub(r"([.!?])?$", ".", s)
+        tlogger.debug(s)
+
+
+def thought_begin():
+    thought_log("<think>")
+
+
+def thought_end():
+    thought_log("</think>")
+
+
+if os.getenv("DEPSCAN_THINK_MODE", "") in ("true", "1"):
+    tlogger = logging.getLogger("depscan_thoughts")
+    tlogger.setLevel(DEBUG)
+    file_handler = logging.FileHandler(
+        os.getenv(
+            "DEPSCAN_THOUGHT_LOG", os.path.join(os.getcwd(), "depscan-thoughts.log")
+        )
+    )
+    file_handler.setLevel(DEBUG)
+    formatter = logging.Formatter("%(message)s")
+    file_handler.setFormatter(formatter)
+    tlogger.addHandler(file_handler)

depscan/lib/package_query/cargo_pkg.py

@@ -0,0 +1,124 @@
+from datetime import datetime, timezone
+
+from depscan.lib import config
+from depscan.lib.package_query.pkg_query import compute_time_risks, calculate_risk_score
+from semver import Version
+
+def set_binary_risks(risk_metrics, current_version, latest_version):
+    """
+    If current version has bin_names. then we should set "pkg_includes_binary_risk" as True.
+    and add the number of bin_names to the "pkg_includes_binary_value" key.
+    """
+    version = current_version if current_version else latest_version
+    bin_names = version.get('bin_names', [])
+    risk_metrics["pkg_includes_binary_risk"] = True if len(bin_names)>0 else False
+    risk_metrics["pkg_includes_binary_value"] = len(bin_names)
+
+
+def cargo_pkg_risk(pkg_metadata, is_private_pkg, scope, pkg):
+    """
+    Calculate various package risks based on the metadata from cargo.
+    """
+    risk_metrics = {
+        "pkg_deprecated_risk": False,
+        "pkg_version_deprecated_risk": False,
+        "pkg_version_missing_risk": False,
+        "pkg_includes_binary_risk": False,
+        "pkg_min_versions_risk": False,
+        "created_now_quarantine_seconds_risk": False,
+        "latest_now_max_seconds_risk": False,
+        "mod_create_min_seconds_risk": False,
+        "pkg_min_maintainers_risk": False,
+        "pkg_node_version_risk": False,
+        "pkg_private_on_public_registry_risk": False,
+    }
+    versions_list = pkg_metadata.get("versions", [])
+    versions_dict = {
+        crate_version.get('num'): crate_version
+        for crate_version in versions_list}
+    versions_nums = [
+        crate_version.get('num')
+        for crate_version in versions_list]
+
+    is_deprecated = versions_list[0].get("yanked")
+    is_version_deprecated = False
+    info = pkg_metadata.get("crate", {})
+    if not is_deprecated and pkg and pkg.get("version"):
+        theversion = versions_dict.get(pkg.get("version"), {})
+        if isinstance(theversion, dict) and len(theversion) > 0:
+            theversion = theversion.get('num')
+        elif theversion and theversion.get("yanked"):
+            is_version_deprecated = True
+        # Check if the version exists in the registry
+        if not theversion:
+            risk_metrics["pkg_version_missing_risk"] = True
+            risk_metrics["pkg_version_missing_value"] = 1
+
+    pkg_description = info.get("description", "").lower()
+    if not is_deprecated and (
+        "is deprecated" in pkg_description
+        or "no longer maintained" in pkg_description
+    ):
+        is_deprecated = True
+    latest_deprecated = False
+
+    try:
+        first_version_num = min(
+            versions_nums,
+            key=lambda x: Version.parse(x, optional_minor_and_patch=True),
+        )
+        latest_version_num = max(
+            versions_nums,
+            key=lambda x: Version.parse(x, optional_minor_and_patch=True),
+        )
+    except (ValueError, TypeError):
+        first_version_num = versions_nums[-1]
+        latest_version_num = versions_nums[0]
+    
+    first_version = versions_dict[first_version_num]
+    latest_version = versions_list[latest_version_num]
+
+    # Is the private package available publicly? Dependency confusion.
+    if is_private_pkg and pkg_metadata:
+        risk_metrics["pkg_private_on_public_registry_risk"] = True
+        risk_metrics["pkg_private_on_public_registry_value"] = 1
+
+    # If the package has fewer than minimum number of versions
+    if len(versions_list):
+        if len(versions_list) < config.pkg_min_versions:
+            risk_metrics["pkg_min_versions_risk"] = True
+            risk_metrics["pkg_min_versions_value"] = len(versions_list)
+        # Check if the latest version is deprecated
+        if latest_version and latest_version.get("yanked"):
+            latest_deprecated = True
+
+    # Created and modified time related checks
+    if first_version and latest_version:
+        created = first_version.get("created_at")
+        modified = latest_version.get("updated_at")
+        if created and modified:
+            modified_dt = datetime.fromisoformat(modified)
+            created_dt = datetime.fromisoformat(created)
+            mod_create_diff = modified_dt - created_dt
+            latest_now_diff = datetime.now(timezone.utc) - modified_dt
+            created_now_diff = datetime.now(timezone.utc) - created_dt
+            risk_metrics = compute_time_risks(
+                risk_metrics, created_now_diff, mod_create_diff, latest_now_diff
+            )
+
+    # Is the package deprecated
+    if is_deprecated or latest_deprecated:
+        risk_metrics["pkg_deprecated_risk"] = True
+        risk_metrics["pkg_deprecated_value"] = 1
+    elif is_version_deprecated:
+        risk_metrics["pkg_version_deprecated_risk"] = True
+        risk_metrics["pkg_version_deprecated_value"] = 1
+    # Add package scope related weight
+    if scope:
+        risk_metrics[f"pkg_{scope}_scope_risk"] = True
+        risk_metrics[f"pkg_{scope}_scope_value"] = 1
+
+    set_binary_risks(risk_metrics, versions_dict.get(pkg.get("version"), {}), latest_version)
+
+    risk_metrics["risk_score"] = calculate_risk_score(risk_metrics)
+    return risk_metrics

depscan/lib/package_query/metadata.py

@@ -0,0 +1,170 @@
+from rich.progress import Progress
+
+from depscan.lib import config
+from depscan.lib.logger import console, LOG
+from depscan.lib.package_query.npm_pkg import npm_pkg_risk
+from depscan.lib.package_query.pkg_query import get_lookup_url, httpclient
+from depscan.lib.package_query.pypi_pkg import pypi_pkg_risk
+from depscan.lib.package_query.cargo_pkg import cargo_pkg_risk
+
+
+def metadata_from_registry(
+    registry_type, scoped_pkgs, pkg_list, private_ns=None
+):
+    """
+    Method to query registry for the package metadata
+
+    :param registry_type: The type of registry to query
+    :param scoped_pkgs: Dictionary of lists of packages per scope
+    :param pkg_list: List of package dictionaries
+    :param private_ns: Private namespace
+    :return:  A dict of package metadata, risk metrics, and private package
+    flag for each package
+    """
+    metadata_dict = {}
+    # Circuit breaker flag to break the risk audit in case of many api errors
+    circuit_breaker = False
+    # Track the api failures count
+    failure_count = 0
+    done_count = 0
+    with Progress(
+        console=console,
+        transient=True,
+        redirect_stderr=False,
+        redirect_stdout=False,
+        refresh_per_second=1,
+        disable=len(pkg_list) < 10
+    ) as progress:
+        task = progress.add_task(
+            "[green] Auditing packages", total=len(pkg_list)
+        )
+        for pkg in pkg_list:
+            if circuit_breaker:
+                LOG.info(
+                    "Risk audited has been interrupted due to frequent api "
+                    "errors. Please try again later."
+                )
+                progress.stop()
+                return {}
+            scope = pkg.get("scope", "").lower()
+            key, lookup_url = get_lookup_url(registry_type, pkg)
+            if not key or not lookup_url or key.startswith("https://"):
+                progress.advance(task)
+                continue
+            progress.update(task, description=f"Checking {key}")
+            try:
+                r = httpclient.get(
+                    url=lookup_url,
+                    follow_redirects=True,
+                    timeout=config.request_timeout_sec,
+                )
+                json_data = r.json()
+                # Npm returns this error if the package is not found
+                if (
+                    json_data.get("code") == "MethodNotAllowedError"
+                    or r.status_code > 400
+                ):
+                    continue
+                is_private_pkg = False
+                if private_ns:
+                    namespace_prefixes = private_ns.split(",")
+                    for ns in namespace_prefixes:
+                        if key.lower().startswith(
+                            ns.lower()
+                        ) or key.lower().startswith("@" + ns.lower()):
+                            is_private_pkg = True
+                            break
+                risk_metrics = {}
+                match registry_type:
+                    case "npm":
+                        risk_metrics = npm_pkg_risk(
+                            json_data, is_private_pkg, scope, pkg
+                        )
+                    case "pypi":
+                        project_type_pkg = f"python:{key}".lower()
+                        required_pkgs = scoped_pkgs.get("required", [])
+                        optional_pkgs = scoped_pkgs.get("optional", [])
+                        excluded_pkgs = scoped_pkgs.get("excluded", [])
+                        if (
+                            pkg.get("purl") in required_pkgs
+                            or project_type_pkg in required_pkgs
+                        ):
+                            scope = "required"
+                        elif (
+                            pkg.get("purl") in optional_pkgs
+                            or project_type_pkg in optional_pkgs
+                        ):
+                            scope = "optional"
+                        elif (
+                            pkg.get("purl") in excluded_pkgs
+                            or project_type_pkg in excluded_pkgs
+                        ):
+                            scope = "excluded"
+                        risk_metrics = pypi_pkg_risk(
+                            json_data, is_private_pkg, scope, pkg
+                        )
+                    case "cargo":
+                        risk_metrics = cargo_pkg_risk(
+                            json_data, is_private_pkg, scope, pkg
+                        )
+                    case _:
+                        pass
+                # TODO: remove unnecessary if elif statements
+                # if registry_type == "npm":
+                #     pass
+                # elif registry_type == "pypi":
+                #     pass
+                metadata_dict[key] = {
+                    "scope": scope,
+                    "purl": pkg.get("purl"),
+                    "pkg_metadata": json_data,
+                    "risk_metrics": risk_metrics,
+                    "is_private_pkg": is_private_pkg,
+                }
+            except Exception as e:
+                LOG.debug(e)
+                failure_count += 1
+            progress.advance(task)
+            done_count += 1
+            if failure_count >= config.max_request_failures:
+                circuit_breaker = True
+    LOG.debug(
+        "Retrieved package metadata for %d/%d packages. Failures count %d",
+        done_count,
+        len(pkg_list),
+        failure_count,
+    )
+    return metadata_dict
+
+
+def cargo_metadata(scoped_pkgs, pkg_list, private_ns=None):
+    """
+    Method to query cargo for the package metadata
+    """
+    return metadata_from_registry("cargo", scoped_pkgs, pkg_list, private_ns)
+
+
+def npm_metadata(scoped_pkgs, pkg_list, private_ns=None):
+    """
+    Method to query npm for the package metadata
+
+    :param scoped_pkgs: Dictionary of lists of packages per scope
+    :param pkg_list: List of package dictionaries
+    :param private_ns: Private namespace
+    :return: A dict of package metadata, risk metrics, and private package
+    flag for each package
+    """
+    return metadata_from_registry("npm", scoped_pkgs, pkg_list, private_ns)
+
+
+def pypi_metadata(scoped_pkgs, pkg_list, private_ns=None):
+    """
+    Method to query pypi for the package metadata
+
+    :param scoped_pkgs: Dictionary of lists of packages per scope
+    :param pkg_list: List of package dictionaries
+    :param private_ns: Private namespace
+    :return: A dict of package metadata, risk metrics, and private package
+    flag for each package
+    """
+    return metadata_from_registry("pypi", scoped_pkgs, pkg_list, private_ns)