devsecops-engine-tools 1.130.1__tar.gz → 1.131.0__tar.gz

{devsecops_engine_tools-1.130.1 → devsecops_engine_tools-1.131.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devsecops-engine-tools
-Version: 1.130.1
+Version: 1.131.0
 Summary: Tool for DevSecOps strategy
 Home-page: https://github.com/bancolombia/devsecops-engine-tools
 Author: Bancolombia DevSecOps Team

{devsecops_engine_tools-1.130.1 → devsecops_engine_tools-1.131.0}/devsecops_engine_tools/engine_core/src/applications/runner_engine_core.py

@@ -111,6 +111,7 @@ def get_inputs_from_cli(args):
             "xray",
             "dependency_check",
             "kiuwan",
+            "all_tools",
         ],
         type=str,
         required=False,
@@ -243,7 +244,7 @@ def get_inputs_from_cli(args):
 
     TOOLS = {
         "engine_iac": ["checkov", "kics", "kubescape"],
-        "engine_secret": ["trufflehog", "gitleaks"],
+        "engine_secret": ["trufflehog", "gitleaks", "all_tools"],
         "engine_container": ["prisma", "trivy"],
         "engine_dependencies": ["xray", "dependency_check", "trivy"],
         "engine_code": ["bearer", "kiuwan"],

{devsecops_engine_tools-1.130.1 → devsecops_engine_tools-1.131.0}/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/defect_dojo/defect_dojo.py

@@ -71,6 +71,15 @@ class DefectDojoPlatform(VulnerabilityManagementGateway):
         "NUCLEI": "Nuclei Scan",
         "KIUWAN": "Kiuwan Scan"
     }
+    
+    multiple_scan_types = {
+        "engine_secret": {
+            "ALL_TOOLS": {
+                "scanners": ["GITLEAKS", "TRUFFLEHOG"],
+                "file_separator": "#"
+            }
+        }
+    }
 
     def send_vulnerability_management(
         self, vulnerability_management: VulnerabilityManagement
@@ -127,44 +136,83 @@ class DefectDojoPlatform(VulnerabilityManagementGateway):
                     tags = [
                         f"{vulnerability_management.dict_args['module']}_{tag_suffix}"
                     ]
-
-                request = self._build_request_importscan(
-                    vulnerability_management,
-                    token_cmdb,
-                    token_dd,
-                    tags,
-                    use_cmdb,
-                )
-
-                def request_func():
-                    return DefectDojo.send_import_scan(request)
-
-                response = Utils().retries_requests(
-                    request_func,
-                    vulnerability_management.config_tool["VULNERABILITY_MANAGER"][
-                        "DEFECT_DOJO"
-                    ]["MAX_RETRIES_QUERY"],
-                    retry_delay=5,
-                )
-
-                if hasattr(response, "url"):
-                    if vulnerability_management.config_tool.get("VULNERABILITY_MANAGER").get("DEFECT_DOJO").get("PRINT_DOMAIN"):
-                        response.url = response.url.replace(
-                            vulnerability_management.config_tool["VULNERABILITY_MANAGER"]["DEFECT_DOJO"]["HOST_DEFECT_DOJO"],
-                            vulnerability_management.config_tool["VULNERABILITY_MANAGER"]["DEFECT_DOJO"]["PRINT_DOMAIN"]
-                        )
-                    url_parts = response.url.split("//")
-                    test_string = "//".join([url_parts[0] + "/", url_parts[1]])
-                    print(
-                        "Report sent to vulnerability management: ",
-                        f"{test_string}?tags={vulnerability_management.dict_args['module']}",
+                
+                if vulnerability_management.dict_args["module"] in self.multiple_scan_types and \
+                    vulnerability_management.scan_type in self.multiple_scan_types[vulnerability_management.dict_args["module"]]:
+                    files = vulnerability_management.input_core.path_file_results.split(
+                        self.multiple_scan_types[vulnerability_management.dict_args["module"]][vulnerability_management.scan_type]["file_separator"]
                     )
+                    all_tools = self.multiple_scan_types[vulnerability_management.dict_args["module"]][vulnerability_management.scan_type]["scanners"]
+                    print_url = True
+                    for index, file in enumerate(files):
+                        vulnerability_management.input_core.path_file_results = file
+                        vulnerability_management.scan_type = all_tools[index]
+                        self._send_report_to_vulnerability_management(
+                            vulnerability_management,
+                            token_cmdb,
+                            token_dd,
+                            tags,
+                            use_cmdb,
+                            print_url
+                        )
+                        print_url = False
                 else:
-                    raise ExceptionVulnerabilityManagement(response)
+                    self._send_report_to_vulnerability_management(
+                        vulnerability_management,
+                        token_cmdb,
+                        token_dd,
+                        tags,
+                        use_cmdb,
+                    )
+                           
         except Exception as ex:
             raise ExceptionVulnerabilityManagement(
                 f"Error sending report to vulnerability management with the following error: {str(ex)}"
             )
+    
+    def _send_report_to_vulnerability_management(
+        self, 
+        vulnerability_management, 
+        token_cmdb, 
+        token_dd, 
+        tags, 
+        use_cmdb,
+        print_url=True
+    ):
+        request = self._build_request_importscan(
+            vulnerability_management,
+            token_cmdb,
+            token_dd,
+            tags,
+            use_cmdb,
+        )
+
+        def request_func():
+            return DefectDojo.send_import_scan(request)
+
+        response = Utils().retries_requests(
+            request_func,
+            vulnerability_management.config_tool["VULNERABILITY_MANAGER"][
+                "DEFECT_DOJO"
+            ]["MAX_RETRIES_QUERY"],
+            retry_delay=5,
+        )
+
+        if hasattr(response, "url"):
+            if vulnerability_management.config_tool.get("VULNERABILITY_MANAGER").get("DEFECT_DOJO").get("PRINT_DOMAIN"):
+                response.url = response.url.replace(
+                    vulnerability_management.config_tool["VULNERABILITY_MANAGER"]["DEFECT_DOJO"]["HOST_DEFECT_DOJO"],
+                    vulnerability_management.config_tool["VULNERABILITY_MANAGER"]["DEFECT_DOJO"]["PRINT_DOMAIN"]
+                )
+            if print_url:
+                url_parts = response.url.split("//")
+                test_string = "//".join([url_parts[0] + "/", url_parts[1]])
+                print(
+                    "Report sent to vulnerability management: ",
+                    f"{test_string}?tags={vulnerability_management.dict_args['module']}",
+                )
+        else:
+            raise ExceptionVulnerabilityManagement(response)
 
     def get_product_type_pipeline(self, service, dict_args, secret_tool, config_tool):
         try:

{devsecops_engine_tools-1.130.1 → devsecops_engine_tools-1.131.0}/devsecops_engine_tools/engine_sast/engine_secret/src/applications/runner_secret_scan.py

@@ -13,6 +13,12 @@ from devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_
 from devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.gitleaks.gitleaks_deserealizator import (
     GitleaksDeserealizator
     )
+from devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.all_tools.all_tools import (
+    AllToolsSecretScan
+)
+from devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.all_tools.all_tools_deserealizator import (
+    AllToolsSecretScanDeserealizator
+)   
 from devsecops_engine_tools.engine_utilities.git_cli.infrastructure.git_run import (
     GitRun
     )
@@ -28,6 +34,9 @@ def runner_secret_scan(dict_args, tool, devops_platform_gateway, remote_config_s
         elif (tool == "GITLEAKS"):
             tool_gateway = GitleaksTool()
             tool_deserealizator = GitleaksDeserealizator()
+        elif (tool == "ALL_TOOLS"):
+            tool_gateway = AllToolsSecretScan()
+            tool_deserealizator = AllToolsSecretScanDeserealizator()
 
         return engine_secret_scan(
             devops_platform_gateway = devops_platform_gateway,

{devsecops_engine_tools-1.130.1 → devsecops_engine_tools-1.131.0}/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py

@@ -35,7 +35,13 @@ class SecretScan:
         secret_external_checks=dict_args["token_external_checks"]
         files_to_scan = None if dict_args["folder_path"] is None else [dict_args["folder_path"]]
         if skip_tool == False:
-            self.tool_gateway.install_tool(self.devops_platform_gateway.get_variable("os"), self.devops_platform_gateway.get_variable("temp_directory"), config_tool[tool]["VERSION"])
+            if tool == "all_tools":
+                self.tool_gateway.install_tool(self.devops_platform_gateway.get_variable("os"), self.devops_platform_gateway.get_variable("temp_directory"), {
+                    "GITLEAKS": config_tool["gitleaks"]["VERSION"],
+                    "TRUFFLEHOG": config_tool["trufflehog"]["VERSION"]
+                })
+            else:
+                self.tool_gateway.install_tool(self.devops_platform_gateway.get_variable("os"), self.devops_platform_gateway.get_variable("temp_directory"), config_tool[tool]["VERSION"])
             if files_to_scan is None:
                 files_to_scan = self.git_gateway.get_files_pull_request(
                     self.devops_platform_gateway.get_variable("path_directory"),

devsecops_engine_tools-1.131.0/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/all_tools/all_tools.py

@@ -0,0 +1,242 @@
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.gateway.tool_gateway import (
+    ToolGateway,
+)
+from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger
+from devsecops_engine_tools.engine_utilities import settings
+from devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.gitleaks.gitleaks_tool import (
+    GitleaksTool,
+)
+from devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.trufflehog.trufflehog_run import (
+    TrufflehogRun,
+)
+import json
+import os
+
+logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger()
+
+class AllToolsSecretScan(ToolGateway):
+    gitleaks_tool: GitleaksTool = GitleaksTool()
+    trufflehog_tool: TrufflehogRun = TrufflehogRun()
+    
+    def install_tool(self, agent_os, agent_temp_dir, tool_version) -> any:
+        """Install both tools in parallel"""
+        gitleaks_version = tool_version.get("GITLEAKS")
+        trufflehog_version = tool_version.get("TRUFFLEHOG")
+        
+        with ThreadPoolExecutor(max_workers=2) as executor:
+            futures = [
+                executor.submit(self.gitleaks_tool.install_tool, agent_os, agent_temp_dir, gitleaks_version),
+                executor.submit(self.trufflehog_tool.install_tool, agent_os, agent_temp_dir, trufflehog_version)
+            ]
+            
+            for future in as_completed(futures):
+                try:
+                    future.result()
+                except Exception as e:
+                    logger.error(f"Error installing tool: {str(e)}")
+
+    def run_tool_secret_scan(
+        self, 
+        files, 
+        agent_os, 
+        agent_work_folder, 
+        repository_name, 
+        config_tool, 
+        secret_tool, 
+        secret_external_checks, 
+        agent_temp_dir, 
+        tool, 
+        folder_path
+        ):
+        """Run both secret scanning tools in parallel and collect their results."""
+        with ThreadPoolExecutor(max_workers=2) as executor:
+            # Start both scans
+            future_gitleaks = executor.submit(
+                self.gitleaks_tool.run_tool_secret_scan,
+                files, agent_os, agent_work_folder, repository_name, 
+                config_tool, secret_tool, secret_external_checks, 
+                agent_temp_dir, "gitleaks", folder_path
+            )
+            future_trufflehog = executor.submit(
+                self.trufflehog_tool.run_tool_secret_scan,
+                files, agent_os, agent_work_folder, repository_name, 
+                config_tool, secret_tool, secret_external_checks, 
+                agent_temp_dir, "trufflehog", folder_path
+            )
+            
+            # Get results
+            findings_gitleaks, finding_path_gitleaks = future_gitleaks.result()
+            findings_trufflehog, finding_path_trufflehog = future_trufflehog.result()
+        
+        # Deduplicate: Remove TruffleHog findings that match Gitleaks findings
+        try:
+            # Build normalized set from Gitleaks
+            gitleaks_normalized = set()
+            for g in findings_gitleaks or []:
+                norm = self._normalize_for_cross_tool_dedup(g, is_gitleaks=True)
+                if norm:
+                    gitleaks_normalized.add(norm)
+
+            # Deduplicate within TruffleHog
+            trufflehog_after_internal_dedup = self._deduplicate_trufflehog_internal(findings_trufflehog or [])
+            
+            # Filter TruffleHog against Gitleaks
+            filtered_trufflehog = []
+            for idx, t in enumerate(trufflehog_after_internal_dedup):
+                norm = self._normalize_for_cross_tool_dedup(t, is_gitleaks=False)
+
+                if norm and norm in gitleaks_normalized:
+                    continue
+                else:
+                    filtered_trufflehog.append(t)
+
+            findings = [{"gitleaks": findings_gitleaks, "trufflehog": filtered_trufflehog}]
+            
+            # Rewrite the TruffleHog findings file with only deduplicated results
+            self._rewrite_trufflehog_file(filtered_trufflehog, finding_path_trufflehog)
+            
+        except Exception as e:
+            logger.error(f"Error during deduplication: {e}")
+            findings = [{"gitleaks": findings_gitleaks, "trufflehog": findings_trufflehog}]
+
+        finding_path = finding_path_gitleaks + "#" + finding_path_trufflehog
+        return findings, finding_path
+
+    def _deduplicate_trufflehog_internal(self, findings: list) -> list:
+        """Deduplicate within TruffleHog: remove findings with same filename+line+secret but different detectors."""
+        if not findings:
+            return findings
+        
+        # Use filename+line+secret as key
+        seen_by_file_line_secret = {}
+        deduplicated = []
+        
+        for finding in findings:
+            path = ""
+            try:
+                path = (
+                    finding.get("SourceMetadata", {})
+                    .get("Data", {})
+                    .get("Filesystem", {})
+                    .get("file", "")
+                )
+            except (AttributeError, TypeError):
+                path = finding.get("file", "") or ""
+            
+            # Extract only filename
+            filename = path.split('/')[-1] if '/' in path else path
+            filename = filename.split('\\')[-1] if '\\' in filename else filename
+            
+            # Extract line number
+            try:
+                line = (
+                    finding.get("SourceMetadata", {})
+                    .get("Data", {})
+                    .get("Filesystem", {})
+                    .get("line", "")
+                )
+            except (AttributeError, TypeError):
+                line = finding.get("line") or ""
+            
+            # Get the secret
+            secret = finding.get("Raw") or finding.get("RawV2") or finding.get("Match") or finding.get("Redacted") or ""
+            masked_secret = self._mask_secret(secret)
+            
+            # Key: filename + line + secret
+            key = f"{filename}|{line}|{masked_secret}"
+            
+            if key not in seen_by_file_line_secret:
+                seen_by_file_line_secret[key] = True
+                deduplicated.append(finding)
+        
+        return deduplicated
+
+    def _normalize_for_cross_tool_dedup(self, item: dict, is_gitleaks: bool = True, include_line: bool = True) -> str:
+        """
+        Normalize item for cross-tool deduplication (Gitleaks vs TruffleHog).
+        """
+        if not item:
+            return ""
+        
+        # Extract filename
+        if is_gitleaks:
+            path = item.get("File", "") or ""
+        else:
+            # TruffleHog nested path extraction
+            try:
+                path = (
+                    item.get("SourceMetadata", {})
+                    .get("Data", {})
+                    .get("Filesystem", {})
+                    .get("file", "")
+                )
+            except (AttributeError, TypeError):
+                path = item.get("file", "") or ""
+        
+        # Extract only filename from path
+        filename = path.split('/')[-1] if '/' in path else path
+        filename = filename.split('\\')[-1] if '\\' in filename else filename
+        
+        # Extract line number if requested
+        line = ""
+        if include_line:
+            if is_gitleaks:
+                line = item.get("StartLine") or item.get("line") or ""
+            else:
+                try:
+                    line = (
+                        item.get("SourceMetadata", {})
+                        .get("Data", {})
+                        .get("Filesystem", {})
+                        .get("line", "")
+                    )
+                except (AttributeError, TypeError):
+                    line = item.get("line") or ""
+        
+        # Get the secret from various possible fields
+        if is_gitleaks:
+            secret = item.get("Secret") or item.get("Match") or ""
+        else:
+            secret = item.get("Raw") or item.get("RawV2") or item.get("Match") or item.get("Redacted") or ""
+        
+        masked = self._mask_secret(secret)
+        
+        # Build normalized key
+        if not (filename or masked):
+            return ""
+        
+        if include_line:
+            # Strict: filename|line|secret
+            normalized = f"{filename}|{line}|{masked}"
+        else:
+            # Fallback: filename|secret (no line)
+            normalized = f"{filename}|{masked}"
+        
+        return normalized.strip()
+
+    def _rewrite_trufflehog_file(self, filtered_findings: list, file_path: str) -> None:
+        """Rewrite the TruffleHog findings file with deduplicated results."""
+        try:
+            
+            if not file_path or not os.path.exists(file_path):
+                return
+            
+            with open(file_path, "w") as file:
+                file.writelines(json.dumps(finding) + '\n' for finding in filtered_findings)
+                
+        except Exception as e:
+            logger.error(f"Error rewriting TruffleHog file: {e}")
+            import traceback
+            logger.error(traceback.format_exc())
+
+    def _mask_secret(self, secret: str) -> str:
+        """Mask secret value for comparison: keep first 3 and last 3 chars"""
+        if not secret:
+            return ""
+        secret = str(secret)
+        if "*" in secret:
+            return secret
+        if len(secret) > 6:
+            return secret[:3] + "*" * 9 + secret[-3:]
+        return secret

devsecops_engine_tools-1.131.0/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/all_tools/all_tools_deserealizator.py

@@ -0,0 +1,92 @@
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass, field
+import os
+import re
+from typing import List
+from devsecops_engine_tools.engine_core.src.domain.model.finding import Finding
+from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.gateway.gateway_deserealizator import (
+    DeseralizatorGateway
+)
+from devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.gitleaks.gitleaks_deserealizator import (
+    GitleaksDeserealizator
+)
+from devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.trufflehog.trufflehog_deserealizator import (
+    SecretScanDeserealizator
+)
+
+@dataclass
+class AllToolsSecretScanDeserealizator(DeseralizatorGateway):
+    gitleaks_deserealizator: GitleaksDeserealizator = field(default_factory=GitleaksDeserealizator)
+    trufflehog_deserealizator: SecretScanDeserealizator = field(default_factory=SecretScanDeserealizator)
+
+    def get_list_vulnerability(
+        self, 
+        results_scan_list: List[dict], 
+        path_directory: str, 
+        os: str
+        ) -> List[Finding]:
+        list_open_vulnerabilities = []
+        all_results = results_scan_list[0]
+        findings_gitleaks = all_results.get("gitleaks", [])
+        findings_trufflehog = all_results.get("trufflehog", [])
+        with ThreadPoolExecutor(max_workers=2) as executor:
+            future_gitleaks = executor.submit(
+                self.gitleaks_deserealizator.get_list_vulnerability,
+                findings_gitleaks, path_directory, os
+            )
+            future_trufflehog = executor.submit(
+                self.trufflehog_deserealizator.get_list_vulnerability,
+                findings_trufflehog, os, path_directory
+            )
+            list_open_gitleaks = future_gitleaks.result()
+            list_open_trufflehog = future_trufflehog.result()
+        
+        # Add all vulnerabilities (already deduplicated in all_tools.py)
+        list_open_vulnerabilities.extend(list_open_gitleaks)
+        list_open_vulnerabilities.extend(list_open_trufflehog)
+
+        return list_open_vulnerabilities
+    
+    def get_where_correctly(self, result: dict, path_directory=""):
+        # Build a more detailed where string including line and detector/extra name
+        full_path = result.get("File", "") or ""
+        # Try trufflehog style nested path
+        if not full_path:
+            try:
+                full_path = (
+                    result.get("SourceMetadata", {})
+                    .get("Data", {})
+                    .get("Filesystem", {})
+                    .get("file", "")
+                )
+            except Exception:
+                pass
+
+        # Remove path_directory prefix
+        try:
+            relative = full_path.replace(path_directory, "") if path_directory else full_path
+        except Exception:
+            relative = full_path
+        path = relative.lstrip('/')
+
+        # extract line if available
+        line = result.get("StartLine") or result.get("line") or (
+            result.get("SourceMetadata", {}).get("Data", {}).get("Filesystem", {}).get("line")
+        )
+        
+        detector = (result.get("ExtraData") or {}).get("name") or result.get("DetectorName") or result.get("RuleID") or ""
+
+        secret = str(result.get("Secret", "") or result.get("Match", "") or result.get("Raw", "") or result.get("RawV2", "") or result.get("Redacted", ""))
+        if '*' in secret:
+            hidden_secret = secret
+        else:
+            hidden_secret = (secret[:3] + '*' * 9 + secret[-3:]) if len(secret) > 6 else secret
+
+        where = f"{path}"
+        if line:
+            where += f":{line}"
+        if detector:
+            where += f", Detector: {detector}"
+        where += f", Secret: {hidden_secret}"
+
+        return where

devsecops_engine_tools-1.131.0/devsecops_engine_tools/version.py

@@ -0,0 +1 @@
+version = '1.131.0'

{devsecops_engine_tools-1.130.1 → devsecops_engine_tools-1.131.0}/devsecops_engine_tools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devsecops-engine-tools
-Version: 1.130.1
+Version: 1.131.0
 Summary: Tool for DevSecOps strategy
 Home-page: https://github.com/bancolombia/devsecops-engine-tools
 Author: Bancolombia DevSecOps Team

{devsecops_engine_tools-1.130.1 → devsecops_engine_tools-1.131.0}/devsecops_engine_tools.egg-info/SOURCES.txt

@@ -207,6 +207,9 @@ devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan
 devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/set_input_core.py
 devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/__init__.py
 devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/__init__.py
+devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/all_tools/__init__.py
+devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/all_tools/all_tools.py
+devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/all_tools/all_tools_deserealizator.py
 devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/gitleaks/__init__.py
 devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/gitleaks/gitleaks_deserealizator.py
 devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/gitleaks/gitleaks_tool.py

devsecops_engine_tools-1.130.1/devsecops_engine_tools/version.py

@@ -1 +0,0 @@
-version = '1.130.1'