devsecops-engine-tools 1.6.7__py3-none-any.whl

devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/DeserializeConfigTool.py

@@ -0,0 +1,11 @@
+from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold
+
+class DeserializeConfigTool:
+    def __init__(self, json_data, tool):
+        self.ignore_search_pattern = json_data["IGNORE_SEARCH_PATTERN"]
+        self.message_info_engine_secret = json_data["MESSAGE_INFO_ENGINE_SECRET"]
+        self.level_compliance = Threshold(json_data['THRESHOLD'])
+        self.scope_pipeline = ''
+        self.exclude_path = json_data[tool]["EXCLUDE_PATH"]
+        self.number_threads = json_data[tool]["NUMBER_THREADS"]
+        self.target_branches = json_data["TARGET_BRANCHES"]

devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/gateway/gateway_deserealizator.py

@@ -0,0 +1,11 @@
+from abc import ABCMeta, abstractmethod
+from devsecops_engine_tools.engine_core.src.domain.model.finding import Finding
+
+
+class DeseralizatorGateway(metaclass=ABCMeta):
+    @abstractmethod
+    def get_list_vulnerability(self, results_scan_list: list, path_directory: str, os: str) -> "list[Finding]":
+        "Deseralizator"
+    @abstractmethod
+    def get_where_correctly(self, results_scan_list: any):
+        "Transform Where"

devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/gateway/git_gateway.py

@@ -0,0 +1,15 @@
+from abc import ABCMeta, abstractmethod
+
+
+class GitGateway(metaclass=ABCMeta):
+    @abstractmethod
+    def get_files_pull_request(self, sys_working_dir: str,
+                               target_branch: str,
+                               config_target_branch: dict,
+                               source_branch: str,
+                               access_token: str,
+                               collection_uri: str,
+                               team_project: str,
+                               repository_name: str,
+                               repository_provider: str) -> dict:
+        "get_files_pull_request"

devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/gateway/tool_gateway.py

@@ -0,0 +1,16 @@
+from abc import ABCMeta, abstractmethod
+
+
+class ToolGateway(metaclass=ABCMeta):
+    @abstractmethod
+    def install_tool(self, agent_os: str, agent_temp_dir:str) -> any:
+        "install tool"
+    @abstractmethod
+    def run_tool_secret_scan(self,
+                            files_pullrequest: dict,
+                            exclude_path: dict,
+                            agent_os: str,
+                            agent_work_folder: str,
+                            num_threads: int,
+                            repository_name: str) -> str:
+        "run tool secret scan"

devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py

@@ -0,0 +1,72 @@
+from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore
+from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import (
+    DeserializeConfigTool,
+)
+from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.gateway.tool_gateway import (
+    ToolGateway,
+)
+from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.gateway.gateway_deserealizator import (
+    DeseralizatorGateway,
+)
+from devsecops_engine_tools.engine_core.src.domain.model.gateway.devops_platform_gateway import (
+    DevopsPlatformGateway,
+)
+from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.gateway.git_gateway import (
+    GitGateway
+)
+
+class SecretScan:
+    def __init__(
+        self,
+        tool_gateway: ToolGateway,
+        devops_platform_gateway: DevopsPlatformGateway,
+        tool_deserialize: DeseralizatorGateway,
+        git_gateway: GitGateway
+        ):
+        self.tool_gateway = tool_gateway
+        self.devops_platform_gateway = devops_platform_gateway
+        self.tool_deserialize = tool_deserialize
+        self.git_gateway = git_gateway
+
+    def process(self, dict_args, tool):
+        tool = str(tool).lower()
+        init_config_tool = self.devops_platform_gateway.get_remote_config(
+            dict_args["remote_config_repo"], "engine_sast/engine_secret/ConfigTool.json"
+        )
+        config_tool, skip_tool = self.complete_config_tool(
+            init_config_tool, tool
+        )
+        finding_list = []
+        if skip_tool == "false":
+            self.tool_gateway.install_tool(self.devops_platform_gateway.get_variable("os"), self.devops_platform_gateway.get_variable("temp_directory"))
+            files_pullrequest = self.git_gateway.get_files_pull_request(
+                self.devops_platform_gateway.get_variable("work_folder"),
+                self.devops_platform_gateway.get_variable("target_branch"),
+                config_tool.target_branches,
+                self.devops_platform_gateway.get_variable("source_branch"),
+                self.devops_platform_gateway.get_variable("access_token"),
+                self.devops_platform_gateway.get_variable("organization"),
+                self.devops_platform_gateway.get_variable("project_name"),
+                self.devops_platform_gateway.get_variable("repository"),
+                self.devops_platform_gateway.get_variable("repository_provider"))
+            finding_list = self.tool_deserialize.get_list_vulnerability(
+                self.tool_gateway.run_tool_secret_scan(
+                    files_pullrequest,
+                    config_tool.exclude_path,
+                    self.devops_platform_gateway.get_variable("os"),
+                    self.devops_platform_gateway.get_variable("work_folder"),
+                    config_tool.number_threads,
+                    self.devops_platform_gateway.get_variable("repository")
+                    ),
+                self.devops_platform_gateway.get_variable("os"),
+                self.devops_platform_gateway.get_variable("path_directory")
+                )
+        return finding_list, config_tool
+    
+    def complete_config_tool(self, data_file_tool, tool):
+        config_tool = DeserializeConfigTool(json_data=data_file_tool, tool=tool)
+        config_tool.scope_pipeline = self.devops_platform_gateway.get_variable("pipeline_name")
+        skip_tool = "false"
+        if config_tool.scope_pipeline in config_tool.ignore_search_pattern:
+            skip_tool = "true"
+        return config_tool, skip_tool

devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/set_input_core.py

@@ -0,0 +1,73 @@
+from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore
+from devsecops_engine_tools.engine_core.src.domain.model.gateway.devops_platform_gateway import (
+    DevopsPlatformGateway,
+)
+from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import (
+    DeserializeConfigTool
+    )
+from devsecops_engine_tools.engine_core.src.domain.model.exclusions import Exclusions
+
+
+class SetInputCore:
+    def __init__(self, tool_remote: DevopsPlatformGateway, dict_args, tool, config_tool: DeserializeConfigTool):
+        self.tool_remote = tool_remote
+        self.dict_args = dict_args
+        self.tool = tool
+        self.config_tool = config_tool
+
+    def get_remote_config(self, file_path):
+        """
+        Get remote configuration.
+
+        Returns:
+            dict: Remote configuration.
+        """
+        return self.tool_remote.get_remote_config(self.dict_args["remote_config_repo"], file_path)
+
+    def get_variable(self, variable):
+        """
+        Get variable.
+
+        Returns:
+            dict: Remote variable.
+        """
+        return self.tool_remote.get_variable(variable)
+
+    def get_exclusions(self, exclusions_data, pipeline_name, tool):
+        list_exclusions = []
+        for key, value in exclusions_data.items():
+            if (key == "All") or (key == pipeline_name):
+                exclusions = [
+                    Exclusions(
+                        id=item.get("id", ""),
+                        where=item.get("where", ""),
+                        create_date=item.get("create_date", ""),
+                        expired_date=item.get("expired_date", ""),
+                        severity=item.get("severity", ""),
+                        hu=item.get("hu", ""),
+                        reason=item.get("reason", "Risk acceptance"),
+                    )
+                    for item in value[tool]
+                ]
+                list_exclusions.extend(exclusions)
+        return list_exclusions
+
+    def set_input_core(self, finding_list):
+        """
+        Set the input core.
+
+        Returns:
+            dict: Input core.
+        """
+        return InputCore(
+            totalized_exclusions=self.get_exclusions(
+                self.get_remote_config("engine_sast/engine_secret/Exclusions.json"),
+                self.get_variable("pipeline_name"),
+                self.tool,
+            ),
+            threshold_defined=self.config_tool.level_compliance,
+            path_file_results=finding_list,
+            custom_message_break_build=self.config_tool.message_info_engine_secret,
+            scope_pipeline=self.config_tool.scope_pipeline,
+            stage_pipeline=self.tool_remote.get_variable("stage").capitalize()
+        )

devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/git_cli/git_run.py

@@ -0,0 +1,60 @@
+from dataclasses import dataclass
+import os
+import subprocess
+import git
+from urllib.parse import quote
+from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.gateway.git_gateway import GitGateway
+
+from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger
+from devsecops_engine_tools.engine_utilities import settings
+
+logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger()
+
+@dataclass
+class GitRun(GitGateway):
+
+    def get_files_pull_request(self,
+                               sys_working_dir,
+                               target_branch,
+                               config_target_branch,
+                               source_branch,
+                               access_token,
+                               collection_uri,
+                               team_project,
+                               repository_name,
+                               repository_provider):
+        try:
+            if repository_provider == 'GitHub' or target_branch not in config_target_branch:
+                return []
+            base_compact_url = (
+            f"https://{collection_uri.rstrip('/').split('/')[-1].replace('.visualstudio.com','')}"
+            f".visualstudio.com/{quote(team_project)}/_git/"
+            f"{repository_name}"
+            )
+            
+            url_without_https = base_compact_url.replace("https://", "")
+            url_with_token = f"https://x-access-token:{access_token}@{url_without_https}"
+
+            path_new_folder = sys_working_dir + '/' + repository_name
+            
+            if os.path.exists(path_new_folder):
+                logger.warning(f"Error: folder {repository_name} already exist")
+                return []
+            os.makedirs(path_new_folder)
+            os.chdir(sys_working_dir)
+            subprocess.run(["git", "clone", url_with_token, path_new_folder], capture_output=True, text=True)
+            os.chdir(path_new_folder)
+            
+            repository = git.Repo(path_new_folder)
+
+            source_branch = source_branch.replace("refs/heads/", "")
+            subprocess.run(["git", "checkout", f"origin/{source_branch}"], capture_output=True, text=True)
+            if source_branch != None:
+                diff = repository.git.diff(f"origin/{source_branch}..origin/{target_branch}", name_only=True)
+                if diff:
+                    diff_files = diff.strip().split("\n")
+                print("Pull Requests Associated Files:",len(diff_files))
+                return diff_files
+        except Exception as e:
+            logger.warning(f"Error getting files PullRequest: {e}")
+            return []

devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_deserealizator.py

@@ -0,0 +1,39 @@
+import re
+from datetime import datetime
+from dataclasses import dataclass
+from typing import List
+from devsecops_engine_tools.engine_core.src.domain.model.finding import Finding, Category
+from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.gateway.gateway_deserealizator import DeseralizatorGateway
+
+@dataclass
+class SecretScanDeserealizator(DeseralizatorGateway):
+
+    def get_list_vulnerability(self, results_scan_list: List[dict], os, path_directory) -> List[Finding]:
+        list_open_vulnerabilities = []
+        for result in results_scan_list:
+            where_text, line = self.get_where_correctly(result, os, path_directory)
+            vulnerability_open = Finding(
+                id="SECRET_SCANNING",
+                cvss=None,
+                where=f"{where_text}, Line: {line}",
+                description="Sensitive information in source code",
+                severity="critical",
+                identification_date=datetime.now().strftime("%d%m%Y"),
+                published_date_cve=None,
+                module="engine_secret",
+                category=Category.VULNERABILITY,
+                requirements=result.get("DetectorName"),
+                tool="Trufflehog",
+            )
+            list_open_vulnerabilities.append(vulnerability_open)
+        return list_open_vulnerabilities
+    
+    def get_where_correctly(self, result: dict, os, path_directory):
+        line = str(result.get("SourceMetadata").get("Data").get("Filesystem").get("line") or "Multiline")
+        original_where = str(result.get("SourceMetadata").get("Data").get("Filesystem").get("file"))
+        if re.search(r'Linux', os):
+            original_where = original_where.replace("\\", "/")
+        
+        path_remove = path_directory or ""
+        where_text = original_where.replace(path_remove, "")
+        return where_text, line

devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py

@@ -0,0 +1,104 @@
+import json
+import re
+import subprocess
+import concurrent.futures
+
+from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.gateway.tool_gateway import (
+    ToolGateway,
+)
+
+result = []
+
+
+class TrufflehogRun(ToolGateway):
+    def install_tool(self, agent_os, agent_temp_dir) -> any:
+        reg_exp_os = r"Windows"
+        check_os = re.search(reg_exp_os, agent_os)
+        if check_os:
+            self.run_install_win(agent_temp_dir)
+        else:
+            command = f"trufflehog --version"
+            result = subprocess.run(command, capture_output=True, shell=True)
+            output = result.stderr.strip()
+            reg_exp = r"not found"
+            check_tool = re.search(reg_exp, output.decode("utf-8"))
+            if check_tool:
+                self.run_install()
+
+    def run_install(self):
+        command = f"curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin"
+        subprocess.run(command, capture_output=True, shell=True)
+
+    def run_install_win(self, agent_temp_dir):
+        command_complete = f"powershell -Command [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; [Net.ServicePointManager]::SecurityProtocol; New-Item -Path {agent_temp_dir} -ItemType Directory -Force; Invoke-WebRequest -Uri 'https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh' -OutFile {agent_temp_dir}\install_trufflehog.sh; bash {agent_temp_dir}\install_trufflehog.sh -b C:/Trufflehog/bin; $env:Path += ';C:/Trufflehog/bin'; C:/Trufflehog/bin/trufflehog.exe --version"
+        process = subprocess.Popen(
+            command_complete, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True
+        )
+        process.communicate()
+
+    def run_tool_secret_scan(
+        self,
+        files_commits,
+        exclude_paths,
+        agent_os,
+        agent_work_folder,
+        num_threads,
+        repository_name,
+    ):
+        trufflehog_command = "trufflehog"
+        if "Windows" in agent_os:
+            trufflehog_command = "C:/Trufflehog/bin/trufflehog.exe"
+        with open(f"{agent_work_folder}/excludedPath.txt", "w") as file:
+            file.write("\n".join(exclude_paths))
+        exclude_path = f"{agent_work_folder}/excludedPath.txt"
+        include_paths = self.config_include_path(files_commits, agent_work_folder)
+        with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
+            results = executor.map(
+                self.run_trufflehog,
+                [trufflehog_command] * len(include_paths),
+                [agent_work_folder] * len(include_paths),
+                [exclude_path] * len(include_paths),
+                include_paths,
+                [repository_name] * len(include_paths),
+            )
+        return self.decode_output(results)
+
+    def config_include_path(self, files, agent_work_folder):
+        chunks = []
+        if len(files) != 0:
+            chunk_size = (len(files) + 3) // 4
+            chunks = [
+                files[i : i + chunk_size] for i in range(0, len(files), chunk_size)
+            ]
+        include_paths = []
+        for i, chunk in enumerate(chunks):
+            if not chunk:
+                continue
+            file_path = f"{agent_work_folder}/includePath{i}.txt"
+            include_paths.append(file_path)
+            with open(file_path, "w") as file:
+                for file_pr_path in chunk:
+                    file.write(f"{file_pr_path.strip()}\n")
+        return include_paths
+
+    def run_trufflehog(
+        self,
+        trufflehog_command,
+        agent_work_folder,
+        exclude_path,
+        include_path,
+        repository_name,
+    ):
+        command = f"{trufflehog_command} filesystem {agent_work_folder + '/' + repository_name} --include-paths {include_path} --exclude-paths {exclude_path} --no-verification --json"
+        result = subprocess.run(command, capture_output=True, shell=True, text=True)
+        return result.stdout.strip()
+
+    def decode_output(self, results):
+        for decode_output in results:
+            if decode_output != "":
+                object_json = decode_output.strip().split("\n")
+                json_list = [json.loads(object) for object in object_json]
+                for json_obj in json_list:
+                    if json_obj not in result:
+                        result.append(json_obj)
+        return result

devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/entry_points/entry_point_tool.py

@@ -0,0 +1,11 @@
+import sys
+from devsecops_engine_tools.engine_sast.engine_secret.src.domain.usecases.secret_scan import SecretScan
+from devsecops_engine_tools.engine_sast.engine_secret.src.domain.usecases.set_input_core import (
+    SetInputCore,
+)
+
+def engine_secret_scan(devops_platform_gateway, tool_gateway, dict_args, tool, tool_deserealizator, git_gateway):
+    sys.stdout.reconfigure(encoding='utf-8')
+    finding_list, config_tool = SecretScan(tool_gateway, devops_platform_gateway, tool_deserealizator, git_gateway).process(dict_args, tool)
+    input_core = SetInputCore(devops_platform_gateway, dict_args, tool, config_tool)
+    return finding_list, input_core.set_input_core(finding_list)

devsecops_engine_tools/engine_sca/engine_container/src/applications/runner_container_scan.py

@@ -0,0 +1,45 @@
+from devsecops_engine_tools.engine_sca.engine_container.src.infrastructure.entry_points.entry_point_tool import (
+    init_engine_sca_rm,
+)
+from devsecops_engine_tools.engine_sca.engine_container.src.infrastructure.driven_adapters.prisma_cloud.prisma_cloud_manager_scan import (
+    PrismaCloudManagerScan,
+)
+from devsecops_engine_tools.engine_sca.engine_container.src.infrastructure.driven_adapters.docker.docker_images import (
+    DockerImages,
+)
+from devsecops_engine_tools.engine_sca.engine_container.src.infrastructure.driven_adapters.prisma_cloud.prisma_deserialize_output import (
+    PrismaDeserealizator,
+)
+from devsecops_engine_tools.engine_sca.engine_container.src.infrastructure.driven_adapters.trivy_tool.trivy_manager_scan import (
+    TrivyScan,
+)
+from devsecops_engine_tools.engine_sca.engine_container.src.infrastructure.driven_adapters.trivy_tool.trivy_deserialize_output import (
+    TrivyDeserializator,
+)
+
+
+def runner_engine_container(dict_args, config_tool, token, tool_remote):
+    try:
+        if config_tool["ENGINE_CONTAINER"]["TOOL"].lower() == "trivy":
+            tool_run = TrivyScan()
+            tool_deseralizator = TrivyDeserializator()
+        elif config_tool["ENGINE_CONTAINER"]["TOOL"].lower() == "prisma":
+            tool_run = PrismaCloudManagerScan()
+            tool_deseralizator = PrismaDeserealizator()
+        tool_images = DockerImages()
+        return init_engine_sca_rm(
+            tool_run,
+            tool_remote,
+            tool_images,
+            tool_deseralizator,
+            dict_args,
+            token,
+            config_tool,
+        )
+
+    except Exception as e:
+        raise Exception(f"Error SCAN engine container : {str(e)}")
+
+
+if __name__ == "__main__":
+    runner_engine_container()

devsecops_engine_tools/engine_sca/engine_container/src/domain/model/gateways/deserealizator_gateway.py

@@ -0,0 +1,8 @@
+from abc import ABCMeta, abstractmethod
+from devsecops_engine_tools.engine_core.src.domain.model.finding import Finding
+
+
+class DeseralizatorGateway(metaclass=ABCMeta):
+    @abstractmethod
+    def get_list_findings(self, results_scan_list: list) -> "list[Finding]":
+        "Deseralizator"

devsecops_engine_tools/engine_sca/engine_container/src/domain/model/gateways/images_gateway.py

@@ -0,0 +1,7 @@
+from abc import ABCMeta, abstractmethod
+
+
+class ImagesGateway(metaclass=ABCMeta):
+    @abstractmethod
+    def list_images(self) -> str:
+        "list images"

devsecops_engine_tools/engine_sca/engine_container/src/domain/model/gateways/tool_gateway.py

@@ -0,0 +1,7 @@
+from abc import ABCMeta, abstractmethod
+
+
+class ToolGateway(metaclass=ABCMeta):
+    @abstractmethod
+    def run_tool_container_sca(self, dict_args, token, scan_image, release) -> str:
+        "run tool container sca"

devsecops_engine_tools/engine_sca/engine_container/src/domain/usecases/container_sca_scan.py

@@ -0,0 +1,82 @@
+from devsecops_engine_tools.engine_core.src.domain.model.gateway.devops_platform_gateway import (
+    DevopsPlatformGateway,
+)
+from devsecops_engine_tools.engine_sca.engine_container.src.domain.model.gateways.tool_gateway import (
+    ToolGateway,
+)
+from devsecops_engine_tools.engine_sca.engine_container.src.domain.model.gateways.images_gateway import (
+    ImagesGateway,
+)
+from devsecops_engine_tools.engine_sca.engine_container.src.domain.model.gateways.deserealizator_gateway import (
+    DeseralizatorGateway,
+)
+
+class ContainerScaScan:
+    def __init__(
+        self,
+        tool_run: ToolGateway,
+        tool_remote: DevopsPlatformGateway,
+        tool_images: ImagesGateway,
+        tool_deseralizator: DeseralizatorGateway,
+        dict_args,
+        token,
+        skip_flag
+    ):
+        self.tool_run = tool_run
+        self.tool_remote = tool_remote
+        self.tool_images = tool_images
+        self.tool_deseralizator = tool_deseralizator
+        self.dict_args = dict_args
+        self.token = token
+        self.skip_flag = skip_flag
+
+    def get_remote_config(self, file_path):
+        """
+        Get remote configuration.
+
+        Returns:
+            dict: Remote configuration.
+        """
+        return self.tool_remote.get_remote_config(self.dict_args["remote_config_repo"], file_path)
+
+    def scan_image(self):
+        """
+        Process the list of images.
+
+        Returns:
+            list: List of processed images.
+        """
+        return self.tool_images.list_images()
+
+    def get_variable(self, variable):
+        """
+        Get variable.
+
+        Returns:
+            dict: Remote variable.
+        """
+        return self.tool_remote.get_variable(variable)
+
+    def process(self):
+        """
+        Process SCA scanning.
+
+        Returns:
+            dict: SCA scanning results.
+        """
+        return self.tool_run.run_tool_container_sca(
+            self.get_remote_config("engine_sca/engine_container/ConfigTool.json"),
+            self.token,
+            self.scan_image(),
+            self.get_variable("build_id"),
+            self.skip_flag
+        )
+
+    def deseralizator(self, image_scanned):
+        """
+        Process the results deserializer.
+
+        Returns:
+            list: Deserialized list of findings.
+        """
+        return self.tool_deseralizator.get_list_findings(image_scanned)