PyPI - devsecops-engine-tools - Versions diffs - 1.153.0__tar.gz → 1.154.1__tar.gz - Mend

Files changed (449) hide show

devsecops_engine_tools-1.154.1/devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/driven_adapters/conftest/conftest_tool.py ADDED Viewed

@@ -0,0 +1,219 @@
+import subprocess
+import json
+import platform
+import requests
+import os
+import shutil
+from typing import List
+from devsecops_engine_tools.engine_sast.engine_iac.src.domain.model.context_iac import (
+    ContextIac,
+)
+from devsecops_engine_tools.engine_sast.engine_iac.src.domain.model.gateways.tool_gateway import (
+    ToolGateway,
+)
+from devsecops_engine_tools.engine_sast.engine_iac.src.infrastructure.driven_adapters.conftest.conftest_deserealizator import (
+    ConftestDeserealizator,
+)
+from devsecops_engine_tools.engine_utilities.utils.utils import Utils
+from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger
+from devsecops_engine_tools.engine_utilities import settings
+logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger()
+PLATFORM_BINARY_MAP = {
+    "Linux": ("conftest_{version}_Linux_x86_64.tar.gz", "conftest"),
+    "Darwin": ("conftest_{version}_Darwin_x86_64.tar.gz", "conftest"),
+    "Windows": ("conftest_{version}_Windows_x86_64.zip", "conftest.exe"),
+}
+class ConftestTool(ToolGateway):
+    TOOL_CONFTEST = "CONFTEST"
+    RESULTS_FILE = "results_conftest.json"
+    def run_tool(self, config_tool, folders_to_scan, **kwargs):
+        conftest_config = config_tool.get(self.TOOL_CONFTEST, {})
+        version = conftest_config.get("VERSION")
+        default_severity = conftest_config.get("DEFAULT_SEVERITY", "low")
+        default_category = conftest_config.get("DEFAULT_CATEGORY", "vulnerability")
+        secret_tool = kwargs.get("secret_tool")
+        secret_external_checks = kwargs.get("secret_external_checks")
+        work_folder = kwargs.get("work_folder")
+        platform_to_scan = kwargs.get("platform_to_scan") or None
+        policy_path = self._resolve_policy_path(
+            conftest_config, secret_tool, secret_external_checks, work_folder
+        )
+        os_platform = platform.system()
+        command_prefix = self._install_binary(version, os_platform)
+        if command_prefix is None:
+            return [], None
+        results = self._execute_conftest(folders_to_scan, command_prefix, policy_path, namespaces=platform_to_scan)
+        if results is None:
+            return [], None
+        results_path = os.path.abspath(self.RESULTS_FILE)
+        with open(results_path, "w") as f:
+            json.dump(results, f, indent=4)
+        deserealizator = ConftestDeserealizator()
+        findings_list = deserealizator.get_list_finding(
+            results, default_severity, default_category,
+            rules_config=conftest_config.get("RULES", {})
+        )
+        return findings_list, results_path
+    def _resolve_policy_path(
+        self, conftest_config, secret_tool, secret_external_checks, work_folder
+    ):
+        if conftest_config.get("USE_EXTERNAL_CHECKS_DIR", False):
+            util = Utils()
+            util.configurate_external_checks(
+                self.TOOL_CONFTEST,
+                {self.TOOL_CONFTEST: conftest_config},
+                secret_tool,
+                secret_external_checks,
+                agent_work_folder=work_folder,
+            )
+            base_policies_path = os.path.join(work_folder, "rules", "conftest")
+            policies_subfolder = conftest_config.get(
+                "EXTERNAL_DIR_POLICIES_PATH", ""
+            ).strip("/\\")
+            if policies_subfolder:
+                candidate_path = os.path.join(base_policies_path, policies_subfolder)
+                if os.path.isdir(candidate_path):
+                    return candidate_path
+            return base_policies_path
+        return conftest_config.get("POLICY_PATH", "policy")
+    def get_iac_context_from_results(self, path_file_results: str) -> List[ContextIac]:
+        with open(path_file_results, "r") as f:
+            results = json.load(f)
+        context_list = []
+        for file_result in results:
+            filename = file_result.get("filename", "unknown")
+            for failure in file_result.get("failures", []) or []:
+                msg = failure.get("msg", "unknown")
+                metadata = failure.get("metadata", {}) or {}
+                finding_id = metadata.get("id") or metadata.get("query", "unknown")
+                severity = metadata.get("severity", "medium").lower()
+                context_iac = ContextIac(
+                    id=finding_id,
+                    check_name=msg,
+                    check_class="conftest",
+                    severity=severity,
+                    where=filename,
+                    resource=filename,
+                    description=msg,
+                    module="engine_iac",
+                    tool="Conftest",
+                )
+                context_list.append(context_iac)
+        return context_list
+    def _install_binary(self, version: str, os_platform: str) -> str:
+        if os_platform not in PLATFORM_BINARY_MAP:
+            logger.warning(f"Unsupported platform for conftest: {os_platform}")
+            return None
+        filename_template, binary_name = PLATFORM_BINARY_MAP[os_platform]
+        filename = filename_template.format(version=version)
+        if os.path.isfile(binary_name):
+            logger.info(f"Conftest binary already present: {binary_name}")
+            return f"./{binary_name}"
+        binary_in_path = shutil.which(binary_name) or shutil.which("conftest")
+        if binary_in_path:
+            logger.info(f"Conftest binary found in PATH: {binary_in_path}")
+            return binary_in_path
+        base_url = (
+            f"https://github.com/open-policy-agent/conftest/releases/download/v{version}/"
+        )
+        url = base_url + filename
+        try:
+            logger.info(f"Downloading conftest from {url}")
+            response = requests.get(url, allow_redirects=True, timeout=60)
+            response.raise_for_status()
+            with open(filename, "wb") as f:
+                f.write(response.content)
+            if filename.endswith(".tar.gz"):
+                Utils().extract_targz_file(filename, ".")
+            elif filename.endswith(".zip"):
+                Utils().unzip_file(filename, ".")
+            if os_platform != "Windows":
+                subprocess.run(["chmod", "+x", binary_name], check=True)
+            return f"./{binary_name}"
+        except Exception as e:
+            logger.error(f"Error installing conftest: {e}")
+            return None
+    def _execute_conftest(
+        self, folders_to_scan: list, command_prefix: str, policy_path: str, namespaces=None
+    ):
+        files_to_test = self._collect_files(folders_to_scan)
+        if not files_to_test:
+            logger.warning("No files found to scan with conftest")
+            return []
+        if namespaces:
+            namespace_args = []
+            for ns in namespaces:
+                namespace_args += ["--namespace", ns]
+        else:
+            namespace_args = ["--all-namespaces"]
+        command = (
+            [command_prefix, "test"]
+            + files_to_test
+            + ["--policy", policy_path]
+            + namespace_args
+            + ["--output", "json", "--no-fail"]
+        )
+        try:
+            result = subprocess.run(
+                command,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+            output = result.stdout.decode("utf-8").strip()
+            if not output:
+                logger.warning("Conftest returned empty output")
+                return None
+            return json.loads(output)
+        except Exception as e:
+            logger.error(f"Error executing conftest: {e}")
+            return None
+    def _collect_files(self, folders_to_scan: list) -> list:
+        files = []
+        for path in folders_to_scan:
+            if os.path.isfile(path):
+                files.append(path)
+            elif os.path.isdir(path):
+                for root, _, filenames in os.walk(path):
+                    for fname in filenames:
+                        files.append(os.path.join(root, fname))
+        return files

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devsecops-engine-tools
-Version: 1.153.0
+Version: 1.154.1
 Summary: Tool for DevSecOps strategy
 Home-page: https://github.com/bancolombia/devsecops-engine-tools
 Author: Bancolombia DevSecOps Team

@@ -109,6 +109,7 @@ def get_inputs_from_cli(args):
             "checkov",
             "kics",
             "kubescape",
+            "conftest",
             "trufflehog",
             "gitleaks",
             "prisma",
@@ -156,7 +157,7 @@ def get_inputs_from_cli(args):
     parser.add_argument(
         "-p",
         "--platform",
-        type=parse_choices({"all", "docker", "k8s", "cloudformation", "openapi", "terraform", "serverless", "bicep"}),
+        type=parse_choices({"all", "docker", "k8s", "cloudformation", "openapi", "terraform", "serverless", "bicep", "n8n"}),
         required=False,
         default="all",
         help="Platform to scan, applies only to the engine_iac tool and it is possible to select several {all, docker, k8s, cloudformation, openapi, terraform}",
@@ -262,7 +263,7 @@ def get_inputs_from_cli(args):
     )
     TOOLS = {
-        "engine_iac": ["checkov", "kics", "kubescape"],
+        "engine_iac": ["checkov", "kics", "kubescape", "conftest"],
         "engine_secret": ["trufflehog", "gitleaks", "all_tools"],
         "engine_container": ["prisma", "trivy"],
         "engine_dependencies": ["xray", "dependency_check", "trivy"],

@@ -58,6 +58,7 @@ class DefectDojoPlatform(VulnerabilityManagementGateway):
     scan_type_mapping = {
         "CHECKOV": "Checkov Scan",
+        "CONFTEST": "Conftest Scan",
         "PRISMA": "Twistlock Image Scan",
         "XRAY": "JFrog Xray On Demand Binary Scan",
         "TRUFFLEHOG": "Trufflehog Scan",

@@ -10,6 +10,9 @@ from devsecops_engine_tools.engine_sast.engine_iac.src.infrastructure.driven_ada
 from devsecops_engine_tools.engine_sast.engine_iac.src.infrastructure.driven_adapters.kics.kics_tool import (
     KicsTool
 )
+from devsecops_engine_tools.engine_sast.engine_iac.src.infrastructure.driven_adapters.conftest.conftest_tool import (
+    ConftestTool
+)
 def runner_engine_iac(dict_args, tool, secret_tool, devops_platform_gateway, remote_config_source_gateway, env):
@@ -20,7 +23,8 @@ def runner_engine_iac(dict_args, tool, secret_tool, devops_platform_gateway, rem
         tools = {
             "CHECKOV": CheckovTool(),
             "KUBESCAPE": KubescapeTool(),
-            "KICS": KicsTool()
+            "KICS": KicsTool(),
+            "CONFTEST": ConftestTool(),
         }
         if tool in tools:

@@ -0,0 +1,54 @@
+from devsecops_engine_tools.engine_core.src.domain.model.finding import (
+    Category,
+    Finding,
+)
+from datetime import datetime
+from dataclasses import dataclass
+@dataclass
+class ConftestDeserealizator:
+    @classmethod
+    def get_list_finding(
+        cls, results_scan_list: list, default_severity: str, default_category: str, rules_config: dict = None
+    ) -> "list[Finding]":
+        rules_lookup = {}
+        for rule_group in (rules_config or {}).values():
+            for rule_id, rule_data in rule_group.items():
+                rules_lookup[rule_id] = rule_data
+        list_open_findings = []
+        for file_result in results_scan_list:
+            filename = file_result.get("filename", "unknown")
+            for failure in file_result.get("failures", []) or []:
+                msg = failure.get("msg", "unknown")
+                metadata = failure.get("metadata", {}) or {}
+                query = metadata.get("query", "unknown")
+                node_type = metadata.get("node_type", "unknown")
+                node_id = metadata.get("node_id", "unknown")
+                where = f"{filename}: {node_type}.{node_id}"
+                rule_id = metadata.get("id", "")
+                rule_meta = rules_lookup.get(rule_id, {})
+                severity = rule_meta.get("severity", default_severity).lower()
+                category_str = rule_meta.get("category", default_category).lower()
+                finding_open = Finding(
+                    id=rule_id or query,
+                    cvss=None,
+                    where=where,
+                    description=msg,
+                    severity=severity,
+                    identification_date=datetime.now().strftime("%d%m%Y"),
+                    published_date_cve=None,
+                    module="engine_iac",
+                    category=Category(category_str),
+                    requirements=rule_meta.get("url"),
+                    tool="Conftest",
+                )
+                list_open_findings.append(finding_open)
+        return list_open_findings

@@ -36,6 +36,7 @@ list_scan_type = [
     "Checkmarx Scan",
     "Checkmarx Scan detailed",
     "Checkov Scan",
+    "Conftest Scan",
     "Clair Klar Scan",
     "Clair Scan",
     "Cloudsploit Scan",

@@ -86,7 +86,7 @@ class Utils:
                 logger.warning("The secret is not configured for external controls")
-            elif config_tool[tool]["USE_EXTERNAL_CHECKS_GIT"] and platform.system() in (
+            elif config_tool[tool].get("USE_EXTERNAL_CHECKS_GIT", False) and platform.system() in (
                 "Linux", "Darwin",
             ):
                 config_knowns_hosts(
@@ -99,7 +99,7 @@ class Utils:
                 ssh_key_password = decode_base64(secret["repository_ssh_password"])
                 agent_env = add_ssh_private_key(ssh_key_file_path, ssh_key_password)
-            elif config_tool[tool]["USE_EXTERNAL_CHECKS_DIR"]:
+            elif config_tool[tool].get("USE_EXTERNAL_CHECKS_DIR", False):
                 if not github_token:
                     github_token = github_api.get_installation_access_token(
                         secret.get("github_apps"),

@@ -185,6 +185,9 @@ devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/driven_adapters
 devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/driven_adapters/checkov/checkov_config.py
 devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/driven_adapters/checkov/checkov_deserealizator.py
 devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/driven_adapters/checkov/checkov_tool.py
+devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/driven_adapters/conftest/__init__.py
+devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/driven_adapters/conftest/conftest_deserealizator.py
+devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/driven_adapters/conftest/conftest_tool.py
 devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/driven_adapters/kics/__init__.py
 devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/driven_adapters/kics/kics_deserealizator.py
 devsecops_engine_tools/engine_sast/engine_iac/src/infrastructure/driven_adapters/kics/kics_tool.py

@@ -30,7 +30,7 @@ setup(
     package_dir={
         "devsecops_engine_tools": "devsecops_engine_tools",
     },
-    packages=find_packages(exclude=["**test**"]),
+    packages=find_packages(exclude=["*.test", "*.test.*"]),
     entry_points={
         'console_scripts': [
             'devsecops-engine-tools=devsecops_engine_tools.engine_core.src.applications.runner_engine_core:application_core',

	@@ -1 +0,0 @@
1	- version = '1.153.0'

devsecops-engine-tools 1.153.0__tar.gz → 1.154.1__tar.gz

devsecops-engine-tools 1.153.0tar.gz → 1.154.1tar.gz