devsecops-engine-tools 1.6.7__py3-none-any.whl

devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/dependencies_sca_scan.py

@@ -0,0 +1,44 @@
+from devsecops_engine_tools.engine_sca.engine_dependencies.src.domain.model.gateways.tool_gateway import (
+    ToolGateway,
+)
+from devsecops_engine_tools.engine_sca.engine_dependencies.src.domain.model.gateways.deserializator_gateway import (
+    DeserializatorGateway,
+)
+
+
+class DependenciesScan:
+    def __init__(
+        self,
+        tool_run: ToolGateway,
+        tool_deserializator: DeserializatorGateway,
+        remote_config,
+        dir_to_scan_path,
+        bypass_limits_flag,
+        token,
+    ):
+        self.tool_run = tool_run
+        self.tool_deserializator = tool_deserializator
+        self.remote_config = remote_config
+        self.dir_to_scan_path = dir_to_scan_path
+        self.bypass_limits_flag = bypass_limits_flag
+        self.token = token
+
+    def process(self):
+        """
+        Process SCA dependencies scan.
+
+        Return: dict: SCA scanning results.
+        """
+        return self.tool_run.run_tool_dependencies_sca(
+            self.remote_config,
+            self.dir_to_scan_path,
+            self.bypass_limits_flag,
+            self.token,
+        )
+
+    def deserializator(self, dependencies_scanned):
+        """
+        Process the results deserializer.
+        Terun: list: Deserialized list of findings.
+        """
+        return self.tool_deserializator.get_list_findings(dependencies_scanned)

devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/find_artifacts.py

@@ -0,0 +1,95 @@
+import os
+import tarfile
+import subprocess
+import shutil
+import re
+
+from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger
+from devsecops_engine_tools.engine_utilities import settings
+
+logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger()
+
+
+class FindArtifacts:
+    def __init__(
+        self,
+        working_dir,
+        pattern,
+    ):
+        self.working_dir = working_dir
+        self.pattern = pattern
+
+    def find_packages(self, pattern, working_dir):
+        npm_packages = []
+        py_packages = []
+        ext_files = []
+        extension_pattern = re.compile(pattern, re.IGNORECASE)
+        for root, dirs, files in os.walk(working_dir):
+            components = root.split(os.path.sep)
+            if not ("node_modules" in components) and not (
+                "site-packages" in components
+            ):
+                if "site-packages" in dirs:
+                    py_packages.append(os.path.join(root, "site-packages"))
+                if "node_modules" in dirs:
+                    npm_packages.append(os.path.join(root, "node_modules"))
+                for file in files:
+                    if extension_pattern.search(file):
+                        ext_files.append(os.path.join(root, file))
+        return npm_packages, py_packages, ext_files
+
+    def get_recent_package(self, packages):
+        recent_package = None
+        recent_time = 0
+        for path in packages:
+            created_time = os.path.getctime(path)
+            if created_time > recent_time:
+                recent_time = created_time
+                recent_package = path
+        return recent_package
+
+    def compress_and_mv(self, tar_path, package):
+        try:
+            if os.path.exists(tar_path):
+                os.remove(tar_path)
+            with tarfile.open(tar_path, "w") as tar:
+                tar.add(
+                    package,
+                    arcname=os.path.basename(package),
+                    filter=lambda x: None if "/.bin/" in x.name else x,
+                )
+                logger.debug(f"File to scan: {tar_path}")
+
+        except subprocess.CalledProcessError as e:
+            logger.error(f"Error during {package} compression: {e}")
+
+    def move_files(self, dir_to_scan_path, finded_files):
+        for file in finded_files:
+            target = os.path.join(dir_to_scan_path, os.path.basename(file))
+            shutil.copy2(file, target)
+            logger.debug(f"File to scan: {file}")
+
+    def find_artifacts(self):
+        dir_to_scan_path = os.path.join(self.working_dir, "dependencies_to_scan")
+        if os.path.exists(dir_to_scan_path):
+            shutil.rmtree(dir_to_scan_path)
+        os.makedirs(dir_to_scan_path)
+
+        npm_packages, py_packages, ext_files = self.find_packages(
+            self.pattern, self.working_dir
+        )
+
+        if len(npm_packages):
+            npm_recent = self.get_recent_package(npm_packages)
+            tar_path = os.path.join(dir_to_scan_path, "node_modules.tar")
+            self.compress_and_mv(tar_path, npm_recent)
+
+        if len(py_packages):
+            py_recent = self.get_recent_package(py_packages)
+            tar_path = os.path.join(dir_to_scan_path, "site-packages.tar")
+            self.compress_and_mv(tar_path, py_recent)
+
+        if len(ext_files):
+            self.move_files(dir_to_scan_path, ext_files)
+
+        return dir_to_scan_path

devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/find_mono_repos.py

@@ -0,0 +1,29 @@
+import os
+
+
+class FindMonoRepos:
+    def __init__(
+        self,
+        pipeline_name,
+    ):
+        self.pipeline_name = pipeline_name
+
+    def find_mono_repo(self):
+        """
+        Handle find mono repository dir.
+
+        Return: String: Directory to scan.
+        """
+        current_dir = os.getcwd()
+        pattern = "_MR_"
+        if pattern in self.pipeline_name:
+            mr_dir = self.pipeline_name.split(pattern)[1]
+            mr_dir_path = os.path.join(current_dir, mr_dir)
+            if os.path.isdir(mr_dir_path):
+                return mr_dir_path
+
+            for root, dirs, files in os.walk(current_dir):
+                if mr_dir in dirs:
+                    return os.path.join(root, mr_dir)
+
+        return current_dir

devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/handle_remote_config_patterns.py

@@ -0,0 +1,90 @@
+import re
+import os
+
+class HandleRemoteConfigPatterns:
+    def __init__(
+        self,
+        remote_config,
+        exclusions,
+        pipeline_name,
+        agent_directory,
+    ):
+        self.remote_config = remote_config
+        self.exclusions = exclusions
+        self.pipeline_name = pipeline_name
+        self.agent_directory = agent_directory
+
+    def excluded_files(self):
+        """
+        Handle excluded files.
+
+        Return: string: new regex expresion.
+        """
+
+        pattern = self.remote_config["REGEX_EXPRESSION_EXTENSIONS"]
+        if (self.pipeline_name in self.exclusions) and (
+            self.exclusions[self.pipeline_name].get("SKIP_FILES", 0)
+        ):
+            exclusion = self.exclusions[self.pipeline_name]["SKIP_FILES"]
+            if exclusion.get("files", 0):
+                excluded_file_types = exclusion["files"]
+                pattern2 = pattern
+                for ext in excluded_file_types:
+                    pattern2 = (
+                        pattern2.replace("|" + ext, "")
+                        .replace(ext + "|", "")
+                        .replace(ext, "")
+                    )
+                pattern = pattern2
+
+        return pattern
+
+    def ignore_analysis_pattern(self):
+        """
+        Handle analysis pattern.
+
+        Return: bool: False -> not scan, True -> scan.
+        """
+        ignore = self.remote_config["IGNORE_ANALYSIS_PATTERN"]
+        if re.match(ignore, self.pipeline_name, re.IGNORECASE):
+            return False
+        else:
+            return True
+
+    def bypass_archive_limits(self):
+        """
+        Handle bypass archive limits.
+
+        Return: bool: True -> Bypass archive limits, False -> Without bypass archive limits.
+        """
+        bypass_limits = self.remote_config["BYPASS_ARCHIVE_LIMITS"]
+        if re.match(bypass_limits, self.pipeline_name):
+            return True
+        else:
+            return False
+
+    def skip_from_exclusion(self):
+        """
+        Handle skip tool.
+
+        Return: bool: True -> skip tool, False -> not skip tool.
+        """
+        if (self.pipeline_name in self.exclusions) and (
+            self.exclusions[self.pipeline_name].get("SKIP_TOOL", 0)
+        ):
+            return True
+        else:
+            return False
+
+    def different_working_directory(self):
+        """
+        Handle working directory.
+
+        Return: String: Working directory.
+        """
+        work_dir_different_flag = self.remote_config["WORK_DIR_DIFFERENT_FLAG"]
+        if self.agent_directory:
+            for root, dirs, files in os.walk(self.agent_directory):
+                if work_dir_different_flag in dirs:
+                    return self.agent_directory
+        return os.getcwd()

devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/set_input_core.py

@@ -0,0 +1,65 @@
+from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore
+from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold
+from devsecops_engine_tools.engine_core.src.domain.model.exclusions import Exclusions
+
+
+class SetInputCore:
+    def __init__(self, remote_config, exclusions, pipeline_name, tool):
+        self.remote_config = remote_config
+        self.exclusions = exclusions
+        self.pipeline_name = pipeline_name
+        self.tool = tool
+
+    def get_exclusions(self, exclusions_data, pipeline_name, tool):
+        list_exclusions = []
+        for key, value in exclusions_data.items():
+            if (key == "All") or (key == pipeline_name):
+                if value.get(tool, 0):
+                    exclusions = [
+                        Exclusions(
+                            id=item.get("id", ""),
+                            where=item.get("where", ""),
+                            cve_id=item.get("cve_id", ""),
+                            create_date=item.get("create_date", ""),
+                            expired_date=item.get("expired_date", ""),
+                            severity=item.get("severity", ""),
+                            hu=item.get("hu", ""),
+                            reason=item.get("reason", "Risk acceptance")
+                        )
+                        for item in value[tool]
+                    ]
+                    list_exclusions.extend(exclusions)
+        return list_exclusions
+
+    def update_threshold(self, threshold, exclusions_data, pipeline_name):
+        if (pipeline_name in exclusions_data) and (
+            exclusions_data[pipeline_name].get("THRESHOLD", 0)
+        ):
+            threshold["VULNERABILITY"] = exclusions_data[pipeline_name][
+                "THRESHOLD"
+            ].get("VULNERABILITY")
+        return threshold
+
+    def set_input_core(self, dependencies_scanned):
+        """
+        Set the input core.
+
+        Returns:
+            dict: Input core.
+        """
+        return InputCore(
+            self.get_exclusions(
+                self.exclusions,
+                self.pipeline_name,
+                self.tool,
+            ),
+            Threshold(
+                self.update_threshold(
+                    self.remote_config["THRESHOLD"], self.exclusions, self.pipeline_name
+                )
+            ),
+            dependencies_scanned,
+            self.remote_config["MESSAGE_INFO_ENGINE_DEPENDENCIES"],
+            self.pipeline_name,
+            "Build",
+        )

devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/xray_tool/xray_deserialize_output.py

@@ -0,0 +1,57 @@
+from devsecops_engine_tools.engine_sca.engine_dependencies.src.domain.model.gateways.deserializator_gateway import (
+    DeserializatorGateway,
+)
+from devsecops_engine_tools.engine_core.src.domain.model.finding import (
+    Finding,
+    Category,
+)
+from dataclasses import dataclass
+import json
+from datetime import datetime
+
+
+@dataclass
+class XrayDeserializator(DeserializatorGateway):
+    def set_list_finding(self, vul):
+        vulnerabilities = [
+            Finding(
+                id=vul.get("issue_id", ""),
+                cvss=(
+                    vul["cves"][0].get("cvss_v3_score")
+                    if vul.get("cves", 0) and vul["cves"][0].get("cvss_v3_score", 0)
+                    else ""
+                )
+                + (
+                    vul["cves"][0].get("cvss_v2_score")
+                    if vul.get("cves", 0)
+                    and not (vul["cves"][0].get("cvss_v3_score", 0))
+                    and vul["cves"][0].get("cvss_v2_score", 0)
+                    else ""
+                ),
+                where=(component),
+                description=(
+                    vul["cves"][0].get("cve", "") if vul.get("cves", 0) else ""
+                ),
+                severity=vul.get("severity", "").lower(),
+                identification_date=datetime.now().strftime("%d-%m-%Y %H:%M:%S"),
+                published_date_cve=None,
+                module="engine_dependencies",
+                category=Category.VULNERABILITY,
+                requirements=(
+                    "".join(vul["components"][component].get("fixed_versions", [""]))
+                ),
+                tool="XRAY",
+            )
+            for component in vul.get("components", [])
+        ]
+        return vulnerabilities
+
+    def get_list_findings(self, dependencies_scanned_file) -> "list[Finding]":
+        list_open_vulnerabilities = []
+        with open(dependencies_scanned_file, "rb") as file:
+            json_data = json.loads(file.read())
+            if json_data:
+                for data in json_data:
+                    for vul in data.get("vulnerabilities", []):
+                        list_open_vulnerabilities.extend(self.set_list_finding(vul))
+        return list_open_vulnerabilities

devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/driven_adapters/xray_tool/xray_manager_scan.py

@@ -0,0 +1,154 @@
+from devsecops_engine_tools.engine_sca.engine_dependencies.src.domain.model.gateways.tool_gateway import (
+    ToolGateway,
+)
+
+import subprocess
+import platform
+import requests
+import re
+import os
+import json
+
+from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger
+from devsecops_engine_tools.engine_utilities import settings
+
+logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger()
+
+
+class XrayScan(ToolGateway):
+    def install_tool_linux(self, version):
+        installed = subprocess.run(
+            ["which", "./jf"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        if installed.returncode == 1:
+            command = ["chmod", "+x", "./jf"]
+            try:
+                url = f"https://releases.jfrog.io/artifactory/jfrog-cli/v2-jf/{version}/jfrog-cli-linux-amd64/jf"
+                file = "./jf"
+                response = requests.get(url, allow_redirects=True)
+                with open(file, "wb") as archivo:
+                    archivo.write(response.content)
+                subprocess.run(
+                    command, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+                )
+            except subprocess.CalledProcessError as error:
+                logger.error(f"Error during Jfrog Cli installation on Linux: {error}")
+
+    def install_tool_windows(self, version):
+        try:
+            subprocess.run(
+                ["./jf.exe", "--version"],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+        except:
+            try:
+                url = f"https://releases.jfrog.io/artifactory/jfrog-cli/v2-jf/{version}/jfrog-cli-windows-amd64/jf.exe"
+                exe_file = "./jf.exe"
+                response = requests.get(url, allow_redirects=True)
+                with open(exe_file, "wb") as archivo:
+                    archivo.write(response.content)
+            except subprocess.CalledProcessError as error:
+                logger.error(f"Error while Jfrog Cli installation on Windows: {error}")
+
+    def install_tool_darwin(self, version):
+        installed = subprocess.run(
+            ["which", "./jf"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        if installed.returncode == 1:
+            command = ["chmod", "+x", "./jf"]
+            try:
+                url = f"https://releases.jfrog.io/artifactory/jfrog-cli/v2-jf/{version}/jfrog-cli-mac-386/jf"
+                file = "./jf"
+                response = requests.get(url, allow_redirects=True)
+                with open(file, "wb") as archivo:
+                    archivo.write(response.content)
+                subprocess.run(
+                    command, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+                )
+            except subprocess.CalledProcessError as error:
+                logger.error(f"Error during Jfrog Cli installation on Darwin: {error}")
+
+    def config_server(self, prefix, token):
+        try:
+            c_import = [prefix, "c", "im", token]
+            result = subprocess.run(
+                c_import,
+                check=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+            )
+            server_id = re.search(r"'(.*?)'", result.stderr).group(1)
+            c_set_server = [prefix, "c", "use", server_id]
+            subprocess.run(
+                c_set_server,
+                check=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+            )
+        except subprocess.CalledProcessError as error:
+            logger.error(f"Error during Xray Server configuration: {error}")
+
+    def scan_dependencies(self, prefix, target_dir_name, bypass_limits_flag):
+        try:
+            if bypass_limits_flag:
+                command = [
+                    prefix,
+                    "scan",
+                    "--format=json",
+                    "--bypass-archive-limits",
+                    f"{target_dir_name}/",
+                ]
+            else:
+                command = [prefix, "scan", "--format=json", f"{target_dir_name}/"]
+            result = subprocess.run(
+                command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+            )
+            scan_result = json.loads(result.stdout)
+            file_result = os.path.join(target_dir_name, "scan_result.json")
+            with open(file_result, "w") as file:
+                json.dump(scan_result, file, indent=4)
+            return file_result
+        except subprocess.CalledProcessError as error:
+            logger.error(f"Error executing jf scan: {error}")
+
+    def run_tool_dependencies_sca(
+        self,
+        remote_config,
+        dir_to_scan_path,
+        bypass_limits_flag,
+        token,
+    ):
+
+        cli_version = remote_config["XRAY"]["CLI_VERSION"]
+        os_platform = platform.system()
+
+        if os_platform == "Linux":
+            self.install_tool_linux(cli_version)
+            command_prefix = "./jf"
+        elif os_platform == "Windows":
+            self.install_tool_windows(cli_version)
+            command_prefix = "./jf.exe"
+        elif os_platform == "Darwin":
+            command_prefix = "./jf"
+            self.install_tool_darwin(cli_version)
+        else:
+            logger.warning(f"{os_platform} is not supported.")
+
+        self.config_server(command_prefix, token)
+
+        results_file = None
+        if len(os.listdir(dir_to_scan_path)) == 0:
+            logger.warning("No artifacts found")
+        else:
+            results_file = self.scan_dependencies(
+                command_prefix, dir_to_scan_path, bypass_limits_flag
+            )
+
+        return results_file

devsecops_engine_tools/engine_sca/engine_dependencies/src/infrastructure/entry_points/entry_point_tool.py

@@ -0,0 +1,83 @@
+from devsecops_engine_tools.engine_sca.engine_dependencies.src.domain.usecases.dependencies_sca_scan import (
+    DependenciesScan,
+)
+from devsecops_engine_tools.engine_sca.engine_dependencies.src.domain.usecases.set_input_core import (
+    SetInputCore,
+)
+from devsecops_engine_tools.engine_sca.engine_dependencies.src.domain.usecases.handle_remote_config_patterns import (
+    HandleRemoteConfigPatterns,
+)
+from devsecops_engine_tools.engine_sca.engine_dependencies.src.domain.usecases.find_mono_repos import (
+    FindMonoRepos,
+)
+from devsecops_engine_tools.engine_sca.engine_dependencies.src.domain.usecases.find_artifacts import (
+    FindArtifacts,
+)
+
+import os
+import sys
+
+from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger
+from devsecops_engine_tools.engine_utilities import settings
+
+logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger()
+
+
+def init_engine_dependencies(
+    tool_run, tool_remote, tool_deserializator, dict_args, token, tool
+):
+    sys.stdout.reconfigure(encoding="utf-8")
+
+    remote_config = tool_remote.get_remote_config(
+        dict_args["remote_config_repo"], "engine_sca/engine_dependencies/ConfigTool.json"
+    )
+    exclusions = tool_remote.get_remote_config(
+        dict_args["remote_config_repo"], "engine_sca/engine_dependencies/Exclusions.json"
+    )
+    pipeline_name = tool_remote.get_variable("pipeline_name")
+    agent_directory = tool_remote.get_variable("agent_directory")
+
+    handle_remote_config_patterns = HandleRemoteConfigPatterns(
+        remote_config, exclusions, pipeline_name, agent_directory
+    )
+    skip_flag = handle_remote_config_patterns.skip_from_exclusion()
+    scan_flag = handle_remote_config_patterns.ignore_analysis_pattern()
+
+    dependencies_scanned = None
+    deserialized = []
+    input_core = SetInputCore(remote_config, exclusions, pipeline_name, tool)
+
+    if scan_flag and not (skip_flag):
+        find_mono_repo = FindMonoRepos(pipeline_name)
+        mr_path = find_mono_repo.find_mono_repo()
+        agent_path = handle_remote_config_patterns.different_working_directory()
+        current_path = os.getcwd()
+        if agent_path != current_path:
+            current_path = agent_path
+        elif mr_path != current_path:
+            current_path = mr_path
+
+        bypass_limits_flag = handle_remote_config_patterns.bypass_archive_limits()
+        pattern = handle_remote_config_patterns.excluded_files()
+
+        find_artifacts = FindArtifacts(current_path, pattern)
+        dir_to_scan_path = find_artifacts.find_artifacts()
+
+        dependencies_sca_scan = DependenciesScan(
+            tool_run,
+            tool_deserializator,
+            remote_config,
+            dir_to_scan_path,
+            bypass_limits_flag,
+            token,
+        )
+        dependencies_scanned = dependencies_sca_scan.process()
+        if dependencies_scanned:
+            deserialized = dependencies_sca_scan.deserializator(dependencies_scanned)
+    else:
+        print(f"Tool skipped by DevSecOps policy")
+        logger.info(f"Tool skipped by DevSecOps policy")
+
+    core_input = input_core.set_input_core(dependencies_scanned)
+
+    return deserialized, core_input