devsecops-engine-tools 1.7.32__tar.gz → 1.7.34__tar.gz

{devsecops_engine_tools-1.7.32 → devsecops_engine_tools-1.7.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: devsecops-engine-tools
-Version: 1.7.32
+Version: 1.7.34
 Summary: Tool for DevSecOps strategy
 Home-page: https://github.com/bancolombia/devsecops-engine-tools
 Author: Bancolombia DevSecOps Team
@@ -64,7 +64,7 @@ pip3 install devsecops-engine-tools
 ### Scan running - flags (CLI)
 
 ```bash
-devsecops-engine-tools --platform_devops ["local","azure","github"] --remote_config_repo ["remote_config_repo"] --tool ["engine_iac", "engine_dast", "engine_secret", "engine_dependencies", "engine_container"] --folder_path ["Folder path scan engine_iac"] --platform ["k8s","cloudformation","docker", "openapi"] --use_secrets_manager ["false", "true"] --use_vulnerability_management ["false", "true"] --send_metrics ["false", "true"] --token_cmdb ["token_cmdb"] --token_vulnerability_management ["token_vulnerability_management"] --token_engine_container ["token_engine_container"] --token_engine_dependencies ["token_engine_dependencies"] 
+devsecops-engine-tools --platform_devops ["local","azure","github"] --remote_config_repo ["remote_config_repo"] --tool ["engine_iac", "engine_dast", "engine_secret", "engine_dependencies", "engine_container"] --folder_path ["Folder path scan engine_iac"] --platform ["k8s","cloudformation","docker", "openapi"] --use_secrets_manager ["false", "true"] --use_vulnerability_management ["false", "true"] --send_metrics ["false", "true"] --token_cmdb ["token_cmdb"] --token_vulnerability_management ["token_vulnerability_management"] --token_engine_container ["token_engine_container"] --token_engine_dependencies ["token_engine_dependencies"] --xray_mode ["scan", "audit"]
 ```
 
 ### Structure Remote Config
@@ -150,9 +150,42 @@ $ set +a
 devsecops-engine-tools --platform_devops local --remote_config_repo DevSecOps_Remote_Config --tool engine_iac
 
 ```
-### Scan result sample (CLI)
 
-![Dashboard Grafana](docs/demo_session.svg)
+![Demo CLI Local](docs/demo_session.svg)
+
+
+### Scan running sample - Azure Pipelines
+
+The remote config should be in a Azure Devops repository.
+
+Note: By default the tool gets the token from the SYSTEM_ACCESSTOKEN variable to get the remote configuration repository. You must ensure that this token has permission to access this resource.
+
+```yaml
+name: $(Build.SourceBranchName).$(date:yyyyMMdd)$(rev:.r)
+
+trigger:
+  branches:
+    include:
+      - trunk
+      - feature/*
+
+stages:
+  - stage: engine_tools
+    displayName: Example Engine Tools
+    jobs:
+      - job: engine_tools
+        pool:
+          name: Azure Pipelines
+        steps:
+          - script: |
+              # Install devsecops-engine-tools
+              pip3 install -q devsecops-engine-tools
+              devsecops-engine-tools --platform_devops azure --remote_config_repo remote_config --tool engine_iac
+            displayName: "Engine Tools"
+        env:
+          SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+
+```
 
 ### Scan running sample - Github Actions
 
@@ -167,7 +200,7 @@ For more information, refer to [Automatic token authentication](https://docs.git
 
 1. Create a personal access token with the necessary permissions to access the repository.
 2. Add the token as a secret in the GitHub repository.
-    ![Dashboard Grafana](docs/secret_token.png)
+    ![Demo Github](docs/secret_token.png)
 
 3. Configure the yml file containing the workflow using the created secret.
 

{devsecops_engine_tools-1.7.32 → devsecops_engine_tools-1.7.34}/devsecops_engine_tools/engine_core/src/applications/runner_engine_core.py

@@ -31,26 +31,48 @@ from devsecops_engine_tools.version import version
 
 logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger()
 
+
 def parse_separated_list(value, choices):
-    values = value.split(',')
+    values = value.split(",")
     # Validar cada elemento de la lista
     for val in values:
         if val not in choices:
-            raise argparse.ArgumentTypeError(f"Invalid value: {val}. Valid values are: {', '.join(choices)}")
-    
+            raise argparse.ArgumentTypeError(
+                f"Invalid value: {val}. Valid values are: {', '.join(choices)}"
+            )
+
     return values
 
+
 def parse_choices(choices):
     def parse_with_choices(value):
         return parse_separated_list(value, choices)
+
     return parse_with_choices
 
+
 def get_inputs_from_cli(args):
     parser = argparse.ArgumentParser()
-    parser.add_argument("-v", "--version", action='version', version='{version}'.format(version=version))
-    parser.add_argument("-pd", "--platform_devops", choices=["azure", "github", "local"], type=str, required=True, help="Platform where is executed")
-    parser.add_argument("-rcf" ,"--remote_config_repo", type=str, required=True, help="Name or Folder Path of Config Repo")
-    parser.add_argument("-t",
+    parser.add_argument(
+        "-v", "--version", action="version", version="{version}".format(version=version)
+    )
+    parser.add_argument(
+        "-pd",
+        "--platform_devops",
+        choices=["azure", "github", "local"],
+        type=str,
+        required=True,
+        help="Platform where is executed",
+    )
+    parser.add_argument(
+        "-rcf",
+        "--remote_config_repo",
+        type=str,
+        required=True,
+        help="Name or Folder Path of Config Repo",
+    )
+    parser.add_argument(
+        "-t",
         "--tool",
         choices=[
             "engine_iac",
@@ -64,9 +86,20 @@ def get_inputs_from_cli(args):
         required=True,
         help="Tool to execute",
     )
-    parser.add_argument("-fp", "--folder_path", type=str, required=False, help="Folder Path to scan, only apply engine_iac tool")
-    parser.add_argument("-p",
-        "--platform", type=parse_choices({"all", "docker", "k8s", "cloudformation", "openapi"}), required=False, default="all" ,help="Platform to scan, only apply engine_iac tool"
+    parser.add_argument(
+        "-fp",
+        "--folder_path",
+        type=str,
+        required=False,
+        help="Folder Path to scan, only apply engine_iac and engine_dependencies tools",
+    )
+    parser.add_argument(
+        "-p",
+        "--platform",
+        type=parse_choices({"all", "docker", "k8s", "cloudformation", "openapi"}),
+        required=False,
+        default="all",
+        help="Platform to scan, only apply engine_iac tool",
     )
     parser.add_argument(
         "--use_secrets_manager",
@@ -89,10 +122,31 @@ def get_inputs_from_cli(args):
         required=False,
         help="Enable or Disable the send metrics to the driven adapter metrics",
     )
-    parser.add_argument("--token_cmdb", required=False, help="Token to connect to the CMDB")
-    parser.add_argument("--token_vulnerability_management", required=False, help="Token to connect to the Vulnerability Management")
-    parser.add_argument("--token_engine_container", required=False, help="Token to execute engine_container if is necessary")
-    parser.add_argument("--token_engine_dependencies", required=False, help="Token to execute engine_dependencies if is necessary. If using xray as engine_dependencies tool, the token is the base64 of artifactory server config.")
+    parser.add_argument(
+        "--token_cmdb", required=False, help="Token to connect to the CMDB"
+    )
+    parser.add_argument(
+        "--token_vulnerability_management",
+        required=False,
+        help="Token to connect to the Vulnerability Management",
+    )
+    parser.add_argument(
+        "--token_engine_container",
+        required=False,
+        help="Token to execute engine_container if is necessary",
+    )
+    parser.add_argument(
+        "--token_engine_dependencies",
+        required=False,
+        help="Token to execute engine_dependencies if is necessary. If using xray as engine_dependencies tool, the token is the base64 of artifactory server config that can be obtain from jfrog cli with 'jf config export <ServerID>' command.",
+    )
+    parser.add_argument(
+        "--xray_mode",
+        choices=["scan", "audit"],
+        required=False,
+        default="scan",
+        help="Mode to execute xray, only apply engine_dependencies xray tool",
+    )
     args = parser.parse_args()
     return {
         "platform_devops": args.platform_devops,
@@ -107,8 +161,10 @@ def get_inputs_from_cli(args):
         "token_vulnerability_management": args.token_vulnerability_management,
         "token_engine_container": args.token_engine_container,
         "token_engine_dependencies": args.token_engine_dependencies,
+        "xray_mode": args.xray_mode,
     }
 
+
 def application_core():
     try:
         # Get inputs from CLI
@@ -120,18 +176,18 @@ def application_core():
         devops_platform_gateway = {
             "azure": AzureDevops(),
             "github": GithubActions(),
-            "local": RuntimeLocal()
+            "local": RuntimeLocal(),
         }.get(args["platform_devops"])
         printer_table_gateway = PrinterPrettyTable()
         metrics_manager_gateway = S3Manager()
-        
+
         init_engine_core(
             vulnerability_management_gateway,
             secrets_manager_gateway,
             devops_platform_gateway,
             printer_table_gateway,
             metrics_manager_gateway,
-            args
+            args,
         )
     except Exception as e:
         logger.error("Error engine_core: {0} ".format(str(e)))
@@ -142,5 +198,6 @@ def application_core():
         )
         print(devops_platform_gateway.result_pipeline("failed"))
 
+
 if __name__ == "__main__":
     application_core()

{devsecops_engine_tools-1.7.32 → devsecops_engine_tools-1.7.34}/devsecops_engine_tools/engine_core/src/infrastructure/driven_adapters/defect_dojo/defect_dojo.py

@@ -24,6 +24,12 @@ from devsecops_engine_tools.engine_core.src.infrastructure.helpers.util import (
 )
 from functools import partial
 
+from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger
+from devsecops_engine_tools.engine_utilities import settings
+import time
+
+logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger()
+
 
 @dataclass
 class DefectDojoPlatform(VulnerabilityManagementGateway):
@@ -56,7 +62,7 @@ class DefectDojoPlatform(VulnerabilityManagementGateway):
                 "TRUFFLEHOG": "Trufflehog Scan",
                 "TRIVY": "Trivy Scan",
                 "KUBESCAPE": "Kubescape Scanner",
-                "KICS": "KICS Scanner"
+                "KICS": "KICS Scanner",
             }
 
             if any(
@@ -64,7 +70,7 @@ class DefectDojoPlatform(VulnerabilityManagementGateway):
                 for branch in vulnerability_management.config_tool[
                     "VULNERABILITY_MANAGER"
                 ]["BRANCH_FILTER"].split(",")
-            ) or (vulnerability_management.dict_args["tool"] == 'engine_secret'):
+            ) or (vulnerability_management.dict_args["tool"] == "engine_secret"):
                 request: ImportScanRequest = Connect.cmdb(
                     cmdb_mapping={
                         "product_type_name": "nombreevc",
@@ -105,7 +111,17 @@ class DefectDojoPlatform(VulnerabilityManagementGateway):
                     tags=vulnerability_management.dict_args["tool"],
                 )
 
-                response = DefectDojo.send_import_scan(request)
+                def request_func():
+                    return DefectDojo.send_import_scan(request)
+
+                response = self._retries_requests(
+                    request_func,
+                    vulnerability_management.config_tool["VULNERABILITY_MANAGER"][
+                        "DEFECT_DOJO"
+                    ]["MAX_RETRIES_QUERY"],
+                    retry_delay=5,
+                )
+
                 if hasattr(response, "url"):
                     url_parts = response.url.split("//")
                     test_string = "//".join([url_parts[0] + "/", url_parts[1]])
@@ -124,11 +140,17 @@ class DefectDojoPlatform(VulnerabilityManagementGateway):
 
     def get_findings_excepted(self, service, dict_args, secret_tool, config_tool):
         try:
-            session_manager = self._get_session_manager(dict_args, secret_tool, config_tool)
+            session_manager = self._get_session_manager(
+                dict_args, secret_tool, config_tool
+            )
 
             dd_limits_query = config_tool["VULNERABILITY_MANAGER"]["DEFECT_DOJO"][
                 "LIMITS_QUERY"
             ]
+            dd_max_retries = config_tool["VULNERABILITY_MANAGER"]["DEFECT_DOJO"][
+                "MAX_RETRIES_QUERY"
+            ]
+
             tool = dict_args["tool"]
 
             risk_accepted_query_params = {
@@ -145,6 +167,7 @@ class DefectDojoPlatform(VulnerabilityManagementGateway):
             exclusions_risk_accepted = self._get_findings_with_exclusions(
                 session_manager,
                 service,
+                dd_max_retries,
                 risk_accepted_query_params,
                 tool,
                 self._format_date_to_dd_format,
@@ -154,6 +177,7 @@ class DefectDojoPlatform(VulnerabilityManagementGateway):
             exclusions_false_positive = self._get_findings_with_exclusions(
                 session_manager,
                 service,
+                dd_max_retries,
                 false_positive_query_params,
                 tool,
                 self._format_date_to_dd_format,
@@ -168,23 +192,29 @@ class DefectDojoPlatform(VulnerabilityManagementGateway):
                 )
             )
 
-    def get_all_findings(
-        self, service, dict_args, secret_tool, config_tool
-    ):
+    def get_all_findings(self, service, dict_args, secret_tool, config_tool):
         try:
             all_findings_query_params = {
-                "limit": config_tool["VULNERABILITY_MANAGER"]["DEFECT_DOJO"]["LIMITS_QUERY"]
+                "limit": config_tool["VULNERABILITY_MANAGER"]["DEFECT_DOJO"][
+                    "LIMITS_QUERY"
+                ]
             }
+            max_retries = config_tool["VULNERABILITY_MANAGER"]["DEFECT_DOJO"][
+                "MAX_RETRIES_QUERY"
+            ]
 
             findings = self._get_findings(
                 self._get_session_manager(dict_args, secret_tool, config_tool),
-                service, 
-                all_findings_query_params
+                service,
+                max_retries,
+                all_findings_query_params,
             )
 
             maped_list = list(
                 map(
-                    partial(self._create_report, date_fn=self._format_date_to_dd_format),
+                    partial(
+                        self._create_report, date_fn=self._format_date_to_dd_format
+                    ),
                     findings,
                 )
             )
@@ -193,33 +223,49 @@ class DefectDojoPlatform(VulnerabilityManagementGateway):
 
         except Exception as ex:
             raise ExceptionGettingFindings(
-                "Error getting all findings with the following error: {0} ".format(
-                    ex
-                )
+                "Error getting all findings with the following error: {0} ".format(ex)
             )
 
     def _get_session_manager(self, dict_args, secret_tool, config_tool):
-        token_dd = dict_args.get(
-                "token_vulnerability_management"
-            ) or secret_tool.get("token_defect_dojo")
+        token_dd = dict_args.get("token_vulnerability_management") or secret_tool.get(
+            "token_defect_dojo"
+        )
         return SessionManager(
             token_dd,
             config_tool["VULNERABILITY_MANAGER"]["DEFECT_DOJO"]["HOST_DEFECT_DOJO"],
         )
 
     def _get_findings_with_exclusions(
-        self, session_manager, service, query_params, tool, date_fn, reason
+        self, session_manager, service, max_retries, query_params, tool, date_fn, reason
     ):
-        findings = self._get_findings(session_manager, service, query_params)
+        findings = self._get_findings(
+            session_manager, service, max_retries, query_params
+        )
         return map(
             partial(self._create_exclusion, date_fn=date_fn, tool=tool, reason=reason),
             findings,
         )
 
-    def _get_findings(self, session_manager, service, query_params):
-        return Finding.get_finding(
-            session=session_manager, service=service, **query_params
-        ).results
+    def _get_findings(self, session_manager, service, max_retries, query_params):
+        def request_func():
+            return Finding.get_finding(
+                session=session_manager, service=service, **query_params
+            ).results
+
+        return self._retries_requests(request_func, max_retries, retry_delay=5)
+
+    def _retries_requests(self, request_func, max_retries, retry_delay):
+        for attempt in range(max_retries):
+            try:
+                return request_func()
+            except Exception as e:
+                logger.error(f"Error making the request: {e}")
+                if attempt < max_retries - 1:
+                    logger.warning(f"Retry in {retry_delay} seconds...")
+                    time.sleep(retry_delay)
+                else:
+                    logger.error("Maximum number of retries reached, aborting.")
+                    raise e
 
     def _create_exclusion(self, finding, date_fn, tool, reason):
         return Exclusions(
@@ -241,9 +287,7 @@ class DefectDojoPlatform(VulnerabilityManagementGateway):
     def _create_report(self, finding, date_fn):
         return Report(
             id=finding.vuln_id_from_tool,
-            date=date_fn(
-                finding.date
-            ),
+            date=date_fn(finding.date),
             status=finding.display_status,
             where=self._get_where_report(finding),
             tags=finding.tags,
@@ -257,7 +301,7 @@ class DefectDojoPlatform(VulnerabilityManagementGateway):
             if date_string
             else None
         )
-    
+
     def _get_where_report(self, finding):
         for tag in finding.tags:
             return self._get_where(finding, tag)

{devsecops_engine_tools-1.7.32 → devsecops_engine_tools-1.7.34}/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/model/gateways/tool_gateway.py

@@ -4,6 +4,6 @@ from abc import ABCMeta, abstractmethod
 class ToolGateway(metaclass=ABCMeta):
     @abstractmethod
     def run_tool_dependencies_sca(
-        self, remote_config, file_to_scan, bypass_limits_flag, token
+        self, remote_config, dict_args, to_scan, token
     ) -> str:
         "run tool dependencies sca"

{devsecops_engine_tools-1.7.32 → devsecops_engine_tools-1.7.34}/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/dependencies_sca_scan.py

@@ -12,15 +12,19 @@ class DependenciesScan:
         tool_run: ToolGateway,
         tool_deserializator: DeserializatorGateway,
         remote_config,
-        file_to_scan,
-        bypass_limits_flag,
+        dict_args,
+        exclusions,
+        pipeline_name,
+        to_scan,
         token,
     ):
         self.tool_run = tool_run
         self.tool_deserializator = tool_deserializator
         self.remote_config = remote_config
-        self.file_to_scan = file_to_scan
-        self.bypass_limits_flag = bypass_limits_flag
+        self.exclusions = exclusions
+        self.pipeline_name = pipeline_name
+        self.dict_args = dict_args
+        self.to_scan = to_scan
         self.token = token
 
     def process(self):
@@ -31,8 +35,10 @@ class DependenciesScan:
         """
         return self.tool_run.run_tool_dependencies_sca(
             self.remote_config,
-            self.file_to_scan,
-            self.bypass_limits_flag,
+            self.dict_args,
+            self.exclusions,
+            self.pipeline_name,
+            self.to_scan,
             self.token,
         )
 

devsecops_engine_tools-1.7.34/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/handle_remote_config_patterns.py

@@ -0,0 +1,38 @@
+import re
+
+
+class HandleRemoteConfigPatterns:
+    def __init__(
+        self,
+        remote_config,
+        exclusions,
+        pipeline_name,
+    ):
+        self.remote_config = remote_config
+        self.exclusions = exclusions
+        self.pipeline_name = pipeline_name
+
+    def ignore_analysis_pattern(self):
+        """
+        Handle analysis pattern.
+
+        Return: bool: False -> not scan, True -> scan.
+        """
+        ignore = self.remote_config["IGNORE_ANALYSIS_PATTERN"]
+        if re.match(ignore, self.pipeline_name, re.IGNORECASE):
+            return False
+        else:
+            return True
+
+    def skip_from_exclusion(self):
+        """
+        Handle skip tool.
+
+        Return: bool: True -> skip tool, False -> not skip tool.
+        """
+        if (self.pipeline_name in self.exclusions) and (
+            self.exclusions[self.pipeline_name].get("SKIP_TOOL", 0)
+        ):
+            return True
+        else:
+            return False