regscale-cli 6.27.3.0__py3-none-any.whl → 6.28.1.0__py3-none-any.whl

regscale/integrations/compliance_integration.py

@@ -8,6 +8,7 @@ that follow common patterns across different compliance tools (Wiz, Tenable, Sic
 """
 import logging
 import re
+import time
 from abc import ABC, abstractmethod
 from collections import defaultdict
 from typing import Dict, List, Optional, Any, Iterator
@@ -170,6 +171,10 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         # Initialize control matcher for robust control ID matching
         self._control_matcher = ControlMatcher()
 
+        # Performance optimization: cache for control lookups
+        # Key: control ID variation (e.g., 'ac-2(1)') -> (ControlImplementation, SecurityControl)
+        self._control_lookup_cache: Dict[str, tuple[ControlImplementation, SecurityControl]] = {}
+
     def is_poam(self, finding: IntegrationFinding) -> bool:  # type: ignore[override]
         """
         Determines if an issue should be considered a POAM for compliance integrations.
@@ -384,6 +389,50 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         cache_key = f"{implementation_id}_{day_key}"
         return self._existing_assessments_cache.get(cache_key)
 
+    def check_for_existing_evidence(self, file_name_pattern: str) -> bool:
+        """
+        Check if an evidence file matching the pattern already exists in RegScale.
+
+        This method fetches existing files for the plan and checks if any match
+        the provided pattern, helping prevent duplicate evidence uploads.
+
+        :param str file_name_pattern: Pattern to match against existing file names
+        :return: True if a matching file exists, False otherwise
+        :rtype: bool
+        """
+        try:
+            # Import here to avoid circular dependency
+            from regscale.models.regscale_models import File
+
+            # Get all existing files for the plan
+            existing_files = File.get_files_for_parent_from_regscale(
+                parent_id=self.plan_id, parent_module=self.parent_module
+            )
+
+            # Check if any file matches the pattern
+            for file_obj in existing_files:
+                if hasattr(file_obj, "trustedDisplayName") and file_obj.trustedDisplayName:
+                    # Check if the pattern is in the file name
+                    if file_name_pattern in file_obj.trustedDisplayName:
+                        logger.debug(
+                            "Found existing evidence file matching pattern '%s': %s",
+                            file_name_pattern,
+                            file_obj.trustedDisplayName,
+                        )
+                        return True
+
+            logger.debug("No existing evidence files found matching pattern '%s'", file_name_pattern)
+            return False
+
+        except Exception as e:
+            logger.warning(
+                "Unable to check for existing evidence files (pattern: '%s'): %s. Proceeding with upload.",
+                file_name_pattern,
+                e,
+            )
+            # Return False to allow upload to proceed if check fails
+            return False
+
     @abstractmethod
     def fetch_compliance_data(self) -> List[Any]:
         """
@@ -604,6 +653,8 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         pass_count = 0
 
         for result, count in result_counts.items():
+            if result is None:  # Skip None results (controls without evidence)
+                continue
             result_lower = result.lower()
             if result_lower in fail_statuses_lower:
                 fail_count += count
@@ -1131,6 +1182,9 @@ class ComplianceIntegration(ScannerIntegration, ABC):
             logger.warning("No control implementations found for assessment processing")
             return
 
+        # Build control lookup cache for fast O(1) matching
+        self._build_control_lookup_cache(implementations)
+
         all_control_ids = set(self.passing_controls.keys()) | set(self.failing_controls.keys())
         logger.info(f"Processing assessments for {len(all_control_ids)} controls with compliance data")
         logger.info(f"Control IDs with data: {sorted(list(all_control_ids))}")
@@ -1171,6 +1225,56 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         logger.info(f"Found {len(implementations)} control implementations")
         return implementations
 
+    def _build_control_lookup_cache(self, implementations: List[ControlImplementation]) -> None:
+        """
+        Build a lookup cache mapping control ID variations to implementations and security controls.
+
+        This dramatically improves performance by:
+        1. Fetching all SecurityControl objects once (instead of once per match attempt)
+        2. Pre-computing all control ID variations
+        3. Creating a dictionary for O(1) lookup instead of O(n) iteration
+
+        For 1011 implementations x 71 controls = 71,781 iterations in the old code.
+        New code: 1011 fetches + 71 dictionary lookups = ~1082 operations (67x faster!)
+
+        :param List[ControlImplementation] implementations: List of control implementations to cache
+        :return: None
+        :rtype: None
+        """
+        if self._control_lookup_cache:
+            # Cache already built
+            return
+
+        logger.debug(f"Building control lookup cache for {len(implementations)} implementations...")
+        start_time = time.time()
+
+        for implementation in implementations:
+            try:
+                security_control = SecurityControl.get_object(object_id=implementation.controlID)
+                if not security_control or not security_control.controlId:
+                    continue
+
+                # Generate all variations of this control ID for flexible matching
+                control_variations = self._control_matcher._get_control_id_variations(security_control.controlId)
+
+                # Map each variation to this implementation + security control pair
+                for variation in control_variations:
+                    # Store the first implementation found for each variation
+                    # (if multiple implementations have the same control, use the first one)
+                    if variation not in self._control_lookup_cache:
+                        self._control_lookup_cache[variation] = (implementation, security_control)
+
+            except Exception as e:  # noqa: BLE001
+                logger.error(
+                    f"Error caching implementation {implementation.id} with controlID {implementation.controlID}: {e}"
+                )
+                continue
+
+        elapsed = time.time() - start_time
+        logger.info(
+            f"Built control lookup cache with {len(self._control_lookup_cache)} control ID variations in {elapsed:.2f}s"
+        )
+
     def _log_sample_controls(self, implementations: List[ControlImplementation]) -> None:
         """
         Log sample control IDs for debugging purposes.
@@ -1249,9 +1353,10 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         Find matching implementation and security control for a control ID.
 
         Uses ControlMatcher for robust control ID matching with leading zero normalization.
+        Performance optimized with pre-built lookup cache for O(1) matching.
 
         :param str control_id: Control identifier to match
-        :param List[ControlImplementation] implementations: Available implementations
+        :param List[ControlImplementation] implementations: Available implementations (used for fallback only)
         :return: Tuple of matching implementation and security control, or (None, None)
         :rtype: tuple[Optional[ControlImplementation], Optional[SecurityControl]]
         """
@@ -1261,43 +1366,17 @@ class ComplianceIntegration(ScannerIntegration, ABC):
             logger.debug(f"Could not generate control ID variations for: {control_id}")
             return None, None
 
-        matching_implementation = None
-        matching_security_control = None
-
-        for implementation in implementations:
-            try:
-                security_control = SecurityControl.get_object(object_id=implementation.controlID)
-                if not security_control:
-                    logger.debug(
-                        f"No security control found for implementation {implementation.id} with controlID: {implementation.controlID}"
-                    )
-                    continue
-                security_control_id = security_control.controlId
-                if not security_control_id:
-                    logger.debug(f"Security control {security_control.id} has no controlId")
-                    continue
-
-                # Get variations of the security control ID
-                control_variations = self._control_matcher._get_control_id_variations(security_control_id)
-
-                logger.debug(
-                    f"Comparing control '{control_id}' variations {list(search_variations)[:3]} with RegScale control '{security_control_id}' variations {list(control_variations)[:3]} (impl: {implementation.id})"
+        # Try to find a match using the pre-built lookup cache (O(1) lookup)
+        for variation in search_variations:
+            if variation in self._control_lookup_cache:
+                implementation, security_control = self._control_lookup_cache[variation]
+                logger.info(
+                    f"✅ MATCH FOUND: '{security_control.controlId}' == '{control_id}' (implementation: {implementation.id})"
                 )
+                return implementation, security_control
 
-                # Check if any variation matches (set intersection)
-                if search_variations & control_variations:
-                    matching_implementation = implementation
-                    matching_security_control = security_control
-                    logger.info(
-                        f"✅ MATCH FOUND: '{security_control_id}' == '{control_id}' (implementation: {implementation.id})"
-                    )
-                    break
-            except Exception as e:  # noqa: BLE001
-                logger.error(
-                    f"Error processing implementation {implementation.id} with controlID {implementation.controlID}: {e}"
-                )
-                continue
-        return matching_implementation, matching_security_control
+        # No match found in cache
+        return None, None
 
     def _log_no_match(self, control_id: str, implementations: List[ControlImplementation]) -> None:
         """
@@ -1527,8 +1606,8 @@ class ComplianceIntegration(ScannerIntegration, ABC):
                     pass
             else:
                 # Create new assessment
+                # leadAssessorId will be set automatically from the token via the Assessment model's default_factory
                 assessment = Assessment(
-                    leadAssessorId=implementation.createdById,
                     title=f"{self.title} compliance assessment for {control_id.upper()}",
                     assessmentType="Control Testing",
                     plannedStart=get_current_datetime(),
@@ -1631,13 +1710,204 @@ class ComplianceIntegration(ScannerIntegration, ABC):
                 <p><strong>Unique Resources Assessed:</strong> {len(unique_resources)}</p>
                 <p><strong>Passing Assessments:</strong> <span style="color: #2e7d32;">{pass_count}</span></p>
                 <p><strong>Failing Assessments:</strong> <span style="color: #d32f2f;">{fail_count}</span></p>
-                <p><strong>Overall Control Result:</strong> <span style="color: {result_color}; font-weight: bold;">{result}</span></p>
+                <p><strong>Overall Control Result:</strong>
+                   <span style="color: {result_color}; font-weight: bold;">{result}</span></p>
             </div>
              """
             )
 
+            # Add detailed failure information for AWS Audit Manager
+            if result == "Fail" and fail_count > 0:
+                failed_items = [item for item in compliance_items if item.compliance_result not in self.PASS_STATUSES]
+                html_parts.append(self._create_failure_details_section(failed_items))
+
         return "\n".join(html_parts)
 
+    def _create_failure_details_section(self, failed_items: List[ComplianceItem]) -> str:
+        """
+        Create detailed failure information section for failed compliance items.
+
+        :param List[ComplianceItem] failed_items: List of failed compliance items
+        :return: HTML section with detailed failure information
+        :rtype: str
+        """
+        html_parts = [self._get_failure_section_header()]
+
+        for idx, item in enumerate(failed_items, 1):
+            html_parts.append(self._process_failed_item(idx, item))
+
+        html_parts.append("</div>")
+        return "\n".join(html_parts)
+
+    def _get_failure_section_header(self) -> str:
+        """Get the HTML header for failure details section."""
+        return """
+            <div style="margin-top: 20px; padding: 15px; background-color: #fff3e0;
+                        border-left: 4px solid #ff9800; border-radius: 5px;">
+                <h4 style="color: #e65100; margin-top: 0;">Failed Evidence Details</h4>
+            """
+
+    def _process_failed_item(self, idx: int, item: ComplianceItem) -> str:
+        """
+        Process a single failed item and return HTML.
+
+        :param int idx: The index of the failed item
+        :param ComplianceItem item: The failed compliance item
+        :return: HTML for the failed item
+        :rtype: str
+        """
+        if self._has_aws_evidence(item):
+            return self._process_aws_item_with_evidence(idx, item)
+        return self._process_non_aws_item(idx, item)
+
+    def _has_aws_evidence(self, item: ComplianceItem) -> bool:
+        """Check if item has AWS Audit Manager evidence."""
+        return hasattr(item, "evidence_items") and item.evidence_items
+
+    def _process_aws_item_with_evidence(self, idx: int, item: ComplianceItem) -> str:
+        """Process AWS item with evidence details."""
+        evidence_categories = self._categorize_evidence(item)
+
+        if not evidence_categories["failed"]:
+            return ""
+
+        html_parts = []
+        html_parts.append(self._create_failed_check_header(idx, item, evidence_categories))
+        html_parts.append(self._create_failed_evidence_details(evidence_categories["failed"]))
+        html_parts.append(self._add_remediation_guidance(item))
+        html_parts.append("</div>")
+
+        return "\n".join(html_parts)
+
+    def _categorize_evidence(self, item: ComplianceItem) -> Dict[str, List[Any]]:
+        """
+        Categorize evidence items by compliance status.
+
+        :param ComplianceItem item: The compliance item with evidence
+        :return: Dictionary with categorized evidence
+        :rtype: Dict[str, List[Any]]
+        """
+        categories = {"failed": [], "compliant": [], "inconclusive": []}
+
+        for evidence in item.evidence_items:
+            compliance_check = self._get_evidence_compliance_check(item, evidence)
+
+            if compliance_check == "FAILED":
+                categories["failed"].append(evidence)
+            elif compliance_check == "COMPLIANT":
+                categories["compliant"].append(evidence)
+            else:
+                categories["inconclusive"].append(evidence)
+
+        return categories
+
+    def _get_evidence_compliance_check(self, item: ComplianceItem, evidence: Any) -> Optional[str]:
+        """Get compliance check result for evidence."""
+        if hasattr(item, "_get_evidence_compliance"):
+            return item._get_evidence_compliance(evidence)
+        return None
+
+    def _create_failed_check_header(
+        self, idx: int, item: ComplianceItem, evidence_categories: Dict[str, List[Any]]
+    ) -> str:
+        """Create HTML header for failed check."""
+        return f"""
+            <div style="margin-top: 15px; padding: 10px; background-color: #ffebee; border-radius: 3px;">
+                <h5 style="color: #c62828; margin-top: 0;">
+                    Failed Check #{idx}: {item.control_id}
+                </h5>
+                <p><strong>Resource:</strong> {getattr(item, 'resource_name', item.resource_id)}</p>
+                <p><strong>Evidence Summary:</strong>
+                   {len(evidence_categories["failed"])} failed, {len(evidence_categories["compliant"])} compliant,
+                   {len(evidence_categories["inconclusive"])} inconclusive</p>
+            """
+
+    def _create_failed_evidence_details(self, failed_evidence: List[Any]) -> str:
+        """Create HTML for failed evidence details."""
+        html_parts = ['<div style="margin-top: 10px;"><strong>Failed Evidence:</strong><ul>']
+
+        # Limit to 10 failed evidence items
+        for evidence in failed_evidence[:10]:
+            html_parts.append(self._format_single_evidence(evidence))
+
+        html_parts.append("</ul></div>")
+        return "\n".join(html_parts)
+
+    def _format_single_evidence(self, evidence: Dict[str, Any]) -> str:
+        """Format a single evidence item as HTML."""
+        evidence_html = []
+        evidence_source = evidence.get("dataSource", "Unknown source")
+        evidence_id = evidence.get("id", "")[:50]
+
+        evidence_html.append(f"<li><strong>Source:</strong> {evidence_source}")
+
+        if evidence_id:
+            evidence_html.append(f"<br><strong>Evidence ID:</strong> {evidence_id}")
+
+        resources_info = self._get_resources_info(evidence)
+        if resources_info:
+            evidence_html.append(f'<br><strong>Resources:</strong><ul><li>{"</li><li>".join(resources_info)}</li></ul>')
+
+        evidence_html.append("</li>")
+        return "\n".join(evidence_html)
+
+    def _get_resources_info(self, evidence: Dict[str, Any]) -> List[str]:
+        """Extract resource information from evidence."""
+        resources_info = []
+        resources_included = evidence.get("resourcesIncluded", [])
+
+        # Limit to 5 resources per evidence
+        for resource in resources_included[:5]:
+            resource_str = self._format_resource(resource)
+            if resource_str:
+                resources_info.append(resource_str)
+
+        return resources_info
+
+    def _format_resource(self, resource: Dict[str, Any]) -> Optional[str]:
+        """Format a single resource as a string."""
+        resource_type = resource.get("type", "Unknown")
+        resource_value = resource.get("value", "")[:100]
+        resource_check = resource.get("complianceCheck", "N/A")
+
+        if resource_value:
+            return f"{resource_type}: {resource_value} (Status: {resource_check})"
+        return None
+
+    def _add_remediation_guidance(self, item: ComplianceItem) -> str:
+        """Add remediation guidance if available."""
+        if not (hasattr(item, "action_plan_instructions") and item.action_plan_instructions):
+            return ""
+
+        instructions = item.action_plan_instructions[:500]
+        truncated = "..." if len(item.action_plan_instructions) > 500 else ""
+
+        return f"""
+            <div style="margin-top: 10px; padding: 8px; background-color: #e3f2fd;
+                        border-left: 3px solid #1976d2; border-radius: 3px;">
+                <strong>Remediation Guidance:</strong><br>
+                {instructions}{truncated}
+            </div>
+            """
+
+    def _process_non_aws_item(self, idx: int, item: ComplianceItem) -> str:
+        """Process non-AWS items or items without evidence."""
+        description = self._get_item_description(item)
+
+        return f"""
+            <div style="margin-top: 15px; padding: 10px; background-color: #ffebee; border-radius: 3px;">
+                <h5 style="color: #c62828; margin-top: 0;">Failed Check #{idx}: {item.control_id}</h5>
+                <p><strong>Resource:</strong> {getattr(item, 'resource_name', item.resource_id)}</p>
+                <p><strong>Description:</strong> {description}</p>
+            </div>
+            """
+
+    def _get_item_description(self, item: ComplianceItem) -> str:
+        """Get truncated description from item."""
+        if hasattr(item, "description"):
+            return item.description[:200]
+        return "N/A"
+
     def _get_security_plan(self) -> Optional[regscale_models.SecurityPlan]:
         """
         Get the security plan for this integration.

regscale/integrations/due_date_handler.py

@@ -60,6 +60,8 @@ class DueDateHandler:
             "moderate": 120,
             regscale_models.IssueSeverity.Low: 364,
             "low": 364,
+            regscale_models.IssueSeverity.NotAssigned: 364,  # Default to Low severity timeline
+            "notassigned": 364,
         }
 
         # Load integration-specific timelines from config
@@ -90,6 +92,7 @@ class DueDateHandler:
                 "moderate": regscale_models.IssueSeverity.Moderate,
                 "medium": regscale_models.IssueSeverity.Moderate,  # Some integrations use 'medium'
                 "low": regscale_models.IssueSeverity.Low,
+                "notassigned": regscale_models.IssueSeverity.NotAssigned,  # Handle unassigned severities
             }
 
             for config_key, severity in severity_mapping.items():