regscale-cli 6.21.2.0__py3-none-any.whl → 6.28.2.1__py3-none-any.whl

regscale/integrations/compliance_integration.py

@@ -8,11 +8,13 @@ that follow common patterns across different compliance tools (Wiz, Tenable, Sic
 """
 import logging
 import re
+import time
 from abc import ABC, abstractmethod
 from collections import defaultdict
 from typing import Dict, List, Optional, Any, Iterator
 
 from regscale.core.app.utils.app_utils import get_current_datetime, regscale_string_to_datetime
+from regscale.integrations.control_matcher import ControlMatcher
 from regscale.integrations.scanner_integration import (
     ScannerIntegration,
     IntegrationAsset,
@@ -25,6 +27,8 @@ from regscale.models.regscale_models import (
     ControlImplementation,
     Assessment,
     ImplementationObjective,
+    SecurityPlan,
+    ComplianceSettings,
 )
 
 logger = logging.getLogger("regscale")
@@ -102,9 +106,37 @@ class ComplianceIntegration(ScannerIntegration, ABC):
     - Creating assessments and updating control status
     """
 
+    # String literal constants
+    NOT_APPLICABLE_LABEL = "Not Applicable"
+    NOT_APPLICABLE_LOWER = "not applicable"
+    NOT_APPLICABLE_UNDERSCORE = "not_applicable"
+
     # Status mapping constants
-    PASS_STATUSES = ["PASS", "PASSED", "pass", "passed"]
-    FAIL_STATUSES = ["FAIL", "FAILED", "fail", "failed"]
+    PASS_STATUSES = ["PASS", "PASSED", "Pass", "Passed", "pass", "passed", "COMPLIANT", "Compliant", "compliant"]
+    FAIL_STATUSES = [
+        "FAIL",
+        "FAILED",
+        "Fail",
+        "Failed",
+        "fail",
+        "failed",
+        "NONCOMPLIANT",
+        "NonCompliant",
+        "noncompliant",
+    ]
+
+    NOT_APPLICABLE_STATUSES = ["NOT_APPLICABLE", NOT_APPLICABLE_LABEL, "not_applicable", "NA", "N/A"]
+    INCONCLUSIVE_STATUSES = [
+        "INCONCLUSIVE",
+        "Inconclusive",
+        "inconclusive",
+        "UNKNOWN",
+        "Unknown",
+        "unknown",
+        "MANUAL",
+        "Manual",
+        "manual",
+    ]
 
     def __init__(
         self,
@@ -140,6 +172,7 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         self.failed_compliance_items: List[ComplianceItem] = []
         self.passing_controls: Dict[str, ComplianceItem] = {}
         self.failing_controls: Dict[str, ComplianceItem] = {}
+        self.not_applicable_controls: Dict[str, ComplianceItem] = {}
 
         # Asset mapping for compliance to asset correlation
         self.asset_compliance_map: Dict[str, List[ComplianceItem]] = defaultdict(list)
@@ -160,6 +193,20 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         # Set scan date
         self.scan_date = get_current_datetime()
 
+        # Cache for compliance settings
+        self._compliance_settings = None
+        self._security_plan = None
+        self._security_plan_loaded = False  # Track if we've attempted to load
+        self._compliance_settings_loaded = False  # Track if we've attempted to load
+        self._status_mapping_cache = {}  # Cache for status mappings to avoid repeated calculations
+
+        # Initialize control matcher for robust control ID matching
+        self._control_matcher = ControlMatcher()
+
+        # Performance optimization: cache for control lookups
+        # Key: control ID variation (e.g., 'ac-2(1)') -> (ControlImplementation, SecurityControl)
+        self._control_lookup_cache: Dict[str, tuple[ControlImplementation, SecurityControl]] = {}
+
     def is_poam(self, finding: IntegrationFinding) -> bool:  # type: ignore[override]
         """
         Determines if an issue should be considered a POAM for compliance integrations.
@@ -201,7 +248,7 @@ class ComplianceIntegration(ScannerIntegration, ABC):
             self._load_existing_assessments()
 
             self._cache_loaded = True
-            logger.info("🗄️  Loaded existing records cache to prevent duplicates:")
+            logger.info("Loaded existing records cache to prevent duplicates:")
             logger.info(f"   - Assets: {len(self._existing_assets_cache)}")
             logger.info(f"   - Issues: {len(self._existing_issues_cache)}")
             logger.info(f"   - Assessments: {len(self._existing_assessments_cache)}")
@@ -225,9 +272,7 @@ class ComplianceIntegration(ScannerIntegration, ABC):
             )
 
             for asset in existing_assets:
-                # Cache by external_id, identifier, and other_tracking_number for flexible lookup
-                if hasattr(asset, "externalId") and asset.externalId:
-                    self._existing_assets_cache[asset.externalId] = asset
+                # Cache by identifier and other_tracking_number for flexible lookup
                 if hasattr(asset, "identifier") and asset.identifier:
                     self._existing_assets_cache[asset.identifier] = asset
                 if hasattr(asset, "otherTrackingNumber") and asset.otherTrackingNumber:
@@ -253,7 +298,7 @@ class ComplianceIntegration(ScannerIntegration, ABC):
                 parent_id=self.plan_id, parent_module=self.parent_module
             )
             all_issues.update(plan_issues)
-            logger.debug(f"🔍 Found {len(plan_issues)} issues directly under plan {self.plan_id}")
+            logger.debug(f"Found {len(plan_issues)} issues directly under plan {self.plan_id}")
 
             # Method 2: Get issues associated with control implementations (matches scanner integration logic)
             try:
@@ -274,24 +319,22 @@ class ComplianceIntegration(ScannerIntegration, ABC):
                             except Exception as e:
                                 logger.debug(f"Could not load issue {issue_id}: {e}")
 
-                logger.debug(f"🔍 Found {impl_issues_count} additional issues via control implementations")
+                logger.debug(f"Found {impl_issues_count} additional issues via control implementations")
             except Exception as e:
                 logger.debug(f"Could not load issues by control implementation: {e}")
 
-            logger.debug(f"🔍 Total unique issues found: {len(all_issues)} for plan {self.plan_id}")
+            logger.debug(f"Total unique issues found: {len(all_issues)} for plan {self.plan_id}")
 
             wiz_issues = 0
             for issue in all_issues:
                 # Cache by external_id and other_identifier for flexible lookup
-                if hasattr(issue, "externalId") and issue.externalId:
-                    self._existing_issues_cache[issue.externalId] = issue
-                    if "wiz-policy" in issue.externalId.lower():
-                        wiz_issues += 1
-                        logger.debug(f"📋 Cached Wiz issue: {issue.id} -> external_id: {issue.externalId}")
                 if hasattr(issue, "otherIdentifier") and issue.otherIdentifier:
                     self._existing_issues_cache[issue.otherIdentifier] = issue
+                    if "wiz-policy" in issue.otherIdentifier.lower():
+                        wiz_issues += 1
+                        logger.debug(f"Cached Wiz issue: {issue.id} -> other_identifier: {issue.otherIdentifier}")
 
-            logger.debug(f"🔍 Cached {wiz_issues} Wiz policy issues out of {len(all_issues)} total issues")
+            logger.debug(f"Cached {wiz_issues} Wiz policy issues out of {len(all_issues)} total issues")
 
         except Exception as e:
             logger.debug(f"Error loading existing issues: {e}")
@@ -378,6 +421,50 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         cache_key = f"{implementation_id}_{day_key}"
         return self._existing_assessments_cache.get(cache_key)
 
+    def check_for_existing_evidence(self, file_name_pattern: str) -> bool:
+        """
+        Check if an evidence file matching the pattern already exists in RegScale.
+
+        This method fetches existing files for the plan and checks if any match
+        the provided pattern, helping prevent duplicate evidence uploads.
+
+        :param str file_name_pattern: Pattern to match against existing file names
+        :return: True if a matching file exists, False otherwise
+        :rtype: bool
+        """
+        try:
+            # Import here to avoid circular dependency
+            from regscale.models.regscale_models import File
+
+            # Get all existing files for the plan
+            existing_files = File.get_files_for_parent_from_regscale(
+                parent_id=self.plan_id, parent_module=self.parent_module
+            )
+
+            # Check if any file matches the pattern
+            for file_obj in existing_files:
+                if hasattr(file_obj, "trustedDisplayName") and file_obj.trustedDisplayName:
+                    # Check if the pattern is in the file name
+                    if file_name_pattern in file_obj.trustedDisplayName:
+                        logger.debug(
+                            "Found existing evidence file matching pattern '%s': %s",
+                            file_name_pattern,
+                            file_obj.trustedDisplayName,
+                        )
+                        return True
+
+            logger.debug("No existing evidence files found matching pattern '%s'", file_name_pattern)
+            return False
+
+        except Exception as e:
+            logger.warning(
+                "Unable to check for existing evidence files (pattern: '%s'): %s. Proceeding with upload.",
+                file_name_pattern,
+                e,
+            )
+            # Return False to allow upload to proceed if check fails
+            return False
+
     @abstractmethod
     def fetch_compliance_data(self) -> List[Any]:
         """
@@ -411,17 +498,33 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         """
         logger.info("Processing compliance data...")
 
-        # Reset state to avoid double counting on repeated calls
+        self._reset_compliance_state()
+        allowed_controls = self._build_allowed_controls_set()
+        raw_compliance_data = self.fetch_compliance_data()
+
+        processing_stats = self._process_raw_compliance_items(raw_compliance_data, allowed_controls)
+        self._log_processing_summary(raw_compliance_data, processing_stats)
+
+        # Perform control-level categorization based on aggregated results
+        self._categorize_controls_by_aggregation()
+        self._log_final_results()
+
+    def _reset_compliance_state(self) -> None:
+        """Reset state to avoid double counting on repeated calls."""
         self.all_compliance_items = []
         self.failed_compliance_items = []
         self.passing_controls = {}
         self.failing_controls = {}
+        self.not_applicable_controls = {}
         self.asset_compliance_map.clear()
 
-        # Build allowed control IDs from plan/catalog controls to restrict scope
+    def _build_allowed_controls_set(self) -> set[str]:
+        """Build allowed control IDs from plan/catalog controls to restrict scope."""
         allowed_controls_normalized: set[str] = set()
         try:
             controls = self._get_controls()
+            logger.debug(f"Loaded {len(controls)} controls from plan/catalog")
+
             for ctl in controls:
                 cid = (ctl.get("controlId") or "").strip()
                 if not cid:
@@ -429,56 +532,242 @@ class ComplianceIntegration(ScannerIntegration, ABC):
                 base, sub = self._normalize_control_id(cid)
                 normalized = f"{base}({sub})" if sub else base
                 allowed_controls_normalized.add(normalized)
-        except Exception:
-            # If controls cannot be loaded, proceed without additional filtering
+
+            logger.debug(f"Built allowed_controls_normalized set with {len(allowed_controls_normalized)} entries")
+            if allowed_controls_normalized:
+                sample = sorted(allowed_controls_normalized)[:5]
+                logger.debug(f"Sample allowed controls: {sample}")
+        except Exception as e:
+            logger.warning(f"Could not load controls from plan/catalog: {e}")
             allowed_controls_normalized = set()
 
-        # Fetch raw compliance data
-        raw_compliance_data = self.fetch_compliance_data()
+        return allowed_controls_normalized
+
+    def _process_raw_compliance_items(self, raw_compliance_data: list, allowed_controls: set) -> dict:
+        """Process raw compliance items and return processing statistics.
+        :param list raw_compliance_data: Raw compliance data from external system
+        :param set allowed_controls: Allowed control IDs
+        :return: Processed compliance items
+        :rtype: dict
+        """
+        stats = {"skipped_no_control": 0, "skipped_no_resource": 0, "skipped_not_in_plan": 0, "processed_count": 0}
 
-        # Convert to ComplianceItem objects
         for raw_item in raw_compliance_data:
             try:
                 compliance_item = self.create_compliance_item(raw_item)
-                # Skip items that do not resolve to a control or resource
-                if not getattr(compliance_item, "control_id", "") or not getattr(compliance_item, "resource_id", ""):
+                if not self._process_single_compliance_item(compliance_item, allowed_controls, stats):
                     continue
-
-                # If we have an allowed set, restrict to only controls in current plan/catalog
-                if allowed_controls_normalized:
-                    base, sub = self._normalize_control_id(getattr(compliance_item, "control_id", ""))
-                    norm_item = f"{base}({sub})" if sub else base
-                    if norm_item not in allowed_controls_normalized:
-                        continue
-                self.all_compliance_items.append(compliance_item)
-
-                # Build asset mapping
-                self.asset_compliance_map[compliance_item.resource_id].append(compliance_item)
-
-                # Categorize by result
-                if compliance_item.compliance_result in self.FAIL_STATUSES:
-                    self.failed_compliance_items.append(compliance_item)
-                    # Track failing controls (control can fail if ANY asset fails)
-                    control_key = compliance_item.control_id.lower()
-                    self.failing_controls[control_key] = compliance_item
-                    # Remove from passing if it was there
-                    self.passing_controls.pop(control_key, None)
-
-                elif compliance_item.compliance_result in self.PASS_STATUSES:
-                    control_key = compliance_item.control_id.lower()
-                    # Only mark as passing if not already failing
-                    if control_key not in self.failing_controls:
-                        self.passing_controls[control_key] = compliance_item
-
             except Exception as e:
                 logger.error(f"Error processing compliance item: {e}")
                 continue
 
+        return stats
+
+    def _process_single_compliance_item(self, compliance_item: Any, allowed_controls: set, stats: dict) -> bool:
+        """Process a single compliance item and update statistics. Returns True if processed successfully."""
+        control_id = getattr(compliance_item, "control_id", "")
+        resource_id = getattr(compliance_item, "resource_id", "")
+
+        if not control_id:
+            stats["skipped_no_control"] += 1
+            return False
+        if not resource_id:
+            stats["skipped_no_resource"] += 1
+            return False
+
+        if not self._should_process_item(compliance_item, control_id, allowed_controls, stats):
+            return False
+
+        self._add_processed_item(compliance_item, stats)
+        return True
+
+    def _should_process_item(self, compliance_item: Any, control_id: str, allowed_controls: set, stats: dict) -> bool:
+        """Determine if an item should be processed based on control filtering."""
+        if not allowed_controls:
+            return True
+
+        base, sub = self._normalize_control_id(control_id)
+        norm_item = f"{base}({sub})" if sub else base
+
+        if norm_item in allowed_controls:
+            return True
+
+        # Allow PASS controls through even if they don't have existing implementations
+        if compliance_item.compliance_result in self.PASS_STATUSES:
+            return True
+
+        stats["skipped_not_in_plan"] += 1
+        if stats["skipped_not_in_plan"] <= 3:
+            logger.debug(f"Skipping control {norm_item} - not in plan (result: {compliance_item.compliance_result})")
+        return False
+
+    def _add_processed_item(self, compliance_item: Any, stats: dict) -> None:
+        """Add a processed item to collections and update statistics."""
+        self.all_compliance_items.append(compliance_item)
+        stats["processed_count"] += 1
+
+        # Build asset mapping
+        self.asset_compliance_map[compliance_item.resource_id].append(compliance_item)
+
+        # Categorize by result
+        if compliance_item.compliance_result in self.FAIL_STATUSES:
+            self.failed_compliance_items.append(compliance_item)
+            logger.debug(
+                f"Added failing compliance item: control={compliance_item.control_id}, "
+                f"result={compliance_item.compliance_result}, resource={compliance_item.resource_id}"
+            )
+
+    def _log_processing_summary(self, raw_compliance_data: list, stats: dict) -> None:
+        """Log summary of compliance data processing."""
+        logger.debug("Compliance item processing summary:")
+        logger.debug(f"  - Total raw items: {len(raw_compliance_data)}")
+        logger.debug(f"  - Skipped (no control_id): {stats['skipped_no_control']}")
+        logger.debug(f"  - Skipped (no resource_id): {stats['skipped_no_resource']}")
+        logger.debug(f"  - Skipped (not in plan): {stats['skipped_not_in_plan']}")
+        logger.debug(f"  - Processed successfully: {stats['processed_count']}")
+
+    def _log_final_results(self) -> None:
+        """Log final processing results."""
         logger.debug(
             f"Processed {len(self.all_compliance_items)} compliance items: "
             f"{len(self.all_compliance_items) - len(self.failed_compliance_items)} passing, "
             f"{len(self.failed_compliance_items)} failing"
         )
+        logger.debug(
+            f"Control categorization: {len(self.passing_controls)} passing controls, "
+            f"{len(self.failing_controls)} failing controls"
+        )
+
+    def _categorize_controls_by_aggregation(self) -> None:
+        """
+        Categorize controls as passing or failing based on aggregated results across all compliance items.
+
+        This method uses project-scoped aggregation logic instead of the previous "any fail = control fails"
+        approach. For project-scoped integrations (like Wiz), this provides more accurate control status.
+        """
+
+        # Group all compliance items by control ID
+        control_items = self._group_items_by_control()
+
+        # Analyze each control's results
+        for control_key, items in control_items.items():
+            self._categorize_single_control(control_key, items)
+
+    def _group_items_by_control(self) -> dict:
+        """Group compliance items by control ID."""
+        from collections import defaultdict
+
+        control_items = defaultdict(list)
+        for item in self.all_compliance_items:
+            control_key = item.control_id.lower()
+            control_items[control_key].append(item)
+
+        return control_items
+
+    def _categorize_single_control(self, control_key: str, items: list) -> None:
+        """Categorize a single control based on its compliance items."""
+        from collections import Counter
+
+        results = [item.compliance_result for item in items]
+        result_counts = Counter(results)
+        total_items = len(results)
+
+        fail_count, pass_count, not_applicable_count = self._count_results(result_counts)
+
+        if fail_count == 0 and pass_count > 0:
+            self._mark_control_as_passing(control_key, items, pass_count, fail_count)
+        elif fail_count > 0:
+            self._handle_control_with_failures(control_key, items, fail_count, pass_count, total_items)
+        elif not_applicable_count > 0 and pass_count == 0 and fail_count == 0:
+            self._mark_control_as_not_applicable(control_key, items, not_applicable_count)
+        else:
+            logger.debug(f"Control {control_key} has unclear results: {dict(result_counts)}")
+
+    def _count_results(self, result_counts: dict) -> tuple[int, int, int]:
+        """Count pass, fail, and not applicable results from result counts."""
+        fail_statuses_lower = [status.lower() for status in self.FAIL_STATUSES]
+        pass_statuses_lower = [status.lower() for status in self.PASS_STATUSES]
+        not_applicable_statuses_lower = [status.lower() for status in self.NOT_APPLICABLE_STATUSES]
+
+        fail_count = 0
+        pass_count = 0
+        not_applicable_count = 0
+
+        for result, count in result_counts.items():
+            if result is None:  # Skip None results (controls without evidence)
+                continue
+            result_lower = result.lower()
+            if result_lower in fail_statuses_lower:
+                fail_count += count
+            elif result_lower in pass_statuses_lower:
+                pass_count += count
+            elif result_lower in not_applicable_statuses_lower:
+                not_applicable_count += count
+
+        return fail_count, pass_count, not_applicable_count
+
+    def _count_pass_fail_results(self, result_counts: dict) -> tuple[int, int]:
+        """Count pass and fail results from result counts (legacy method)."""
+        fail_count, pass_count, _ = self._count_results(result_counts)
+        return fail_count, pass_count
+
+    def _mark_control_as_passing(self, control_key: str, items: list, pass_count: int, fail_count: int) -> None:
+        """Mark a control as passing."""
+        self.passing_controls[control_key] = items[0]  # Use first item as representative
+        logger.debug(f"Control {control_key} marked as PASSING: {pass_count}P/{fail_count}F")
+
+    def _mark_control_as_not_applicable(self, control_key: str, items: list, not_applicable_count: int) -> None:
+        """Mark a control as not applicable."""
+        self.not_applicable_controls[control_key] = items[0]  # Use first item as representative
+        logger.debug(f"Control {control_key} marked as NOT_APPLICABLE: {not_applicable_count} items")
+
+    def _handle_control_with_failures(
+        self, control_key: str, items: list, fail_count: int, pass_count: int, total_items: int
+    ) -> None:
+        """Handle a control that has some failures."""
+        fail_ratio = fail_count / total_items
+        failure_threshold = getattr(self, "control_failure_threshold", 0.2)
+
+        if fail_ratio > failure_threshold:
+            self._mark_control_as_failing(control_key, items, pass_count, fail_count, fail_ratio, failure_threshold)
+        else:
+            self._mark_control_as_passing_with_warnings(
+                control_key, items, pass_count, fail_count, fail_ratio, failure_threshold
+            )
+
+    def _mark_control_as_failing(
+        self,
+        control_key: str,
+        items: list,
+        pass_count: int,
+        fail_count: int,
+        fail_ratio: float,
+        failure_threshold: float,
+    ) -> None:
+        """Mark a control as failing due to significant failures."""
+        fail_statuses_lower = [status.lower() for status in self.FAIL_STATUSES]
+        failing_item = next(item for item in items if item.compliance_result.lower() in fail_statuses_lower)
+        self.failing_controls[control_key] = failing_item
+        logger.debug(
+            f"Control {control_key} marked as FAILING: {pass_count}P/{fail_count}F "
+            f"({fail_ratio:.1%} fail rate > {failure_threshold:.1%} threshold)"
+        )
+
+    def _mark_control_as_passing_with_warnings(
+        self,
+        control_key: str,
+        items: list,
+        pass_count: int,
+        fail_count: int,
+        fail_ratio: float,
+        failure_threshold: float,
+    ) -> None:
+        """Mark a control as passing despite low failure rate."""
+        self.passing_controls[control_key] = items[0]
+        logger.debug(
+            f"Control {control_key} marked as PASSING (low fail rate): {pass_count}P/{fail_count}F "
+            f"({fail_ratio:.1%} fail rate < {failure_threshold:.1%} threshold)"
+        )
 
     def create_asset_from_compliance_item(self, compliance_item: ComplianceItem) -> Optional[IntegrationAsset]:
         """
@@ -528,6 +817,14 @@ class ComplianceIntegration(ScannerIntegration, ABC):
             control_labels = [compliance_item.control_id] if compliance_item.control_id else []
             severity = self._map_severity(compliance_item.severity)
 
+            # Extract ARNs if available (for AWS Audit Manager and other integrations)
+            arns = None
+            if hasattr(compliance_item, "resource_arns"):
+                arns = compliance_item.resource_arns
+                # Only use ARNs if they're non-empty
+                if not arns:
+                    arns = None
+
             finding = IntegrationFinding(
                 control_labels=control_labels,
                 title=f"Compliance Violation: {compliance_item.control_id}",
@@ -542,10 +839,11 @@ class ComplianceIntegration(ScannerIntegration, ABC):
                 last_seen=self.scan_date,
                 scan_date=self.scan_date,
                 asset_identifier=compliance_item.resource_id,
+                issue_asset_identifier_value=arns,
                 vulnerability_type="Compliance Violation",
                 rule_id=compliance_item.control_id,
                 baseline=compliance_item.framework,
-                affected_controls=",".join(compliance_item.control_id),
+                affected_controls=compliance_item.control_id,
             )
 
             # Ensure affected controls are set to the normalized control label (e.g., RA-5, AC-2(1))
@@ -590,31 +888,60 @@ class ComplianceIntegration(ScannerIntegration, ABC):
 
     def fetch_findings(self, *args, **kwargs) -> Iterator[IntegrationFinding]:
         """
-        Fetch findings from failed compliance items.
+        Fetch findings from failed compliance items to create RegScale issues.
 
         :param args: Variable positional arguments
         :param kwargs: Variable keyword arguments
-        :return: Iterator of integration findings
+        :return: Iterator of integration findings (will be converted to RegScale Issue objects)
         :rtype: Iterator[IntegrationFinding]
         """
-        logger.info("Fetching findings from failed compliance items...")
+        logger.info(f"Preparing to create issues from {len(self.failed_compliance_items)} failed compliance items...")
+
+        # Debug: Show sample of failed items
+        if self.failed_compliance_items:
+            sample_size = min(5, len(self.failed_compliance_items))
+            sample = self.failed_compliance_items[:sample_size]
+            logger.debug(f"Sample failed items (first {sample_size}):")
+            for item in sample:
+                logger.debug(
+                    f"  - Control: {item.control_id}, Result: {item.compliance_result}, Resource: {item.resource_id}"
+                )
 
         total = len(self.failed_compliance_items)
-        task_id = self.finding_progress.add_task(
-            f"[#f68d1f]Creating findings from {total} failed compliance item(s)...",
-            total=total or None,
-        )
+        # Backwards compatibility: check if finding_progress exists and has add_task method
+        task_id = None
+        if self.finding_progress is not None and hasattr(self.finding_progress, "add_task"):
+            task_id = self.finding_progress.add_task(
+                f"[#f68d1f]Creating issues from {total} failed compliance item(s)...",
+                total=total or None,
+            )
 
+        findings_created = 0
         for compliance_item in self.failed_compliance_items:
             finding = self.create_finding_from_compliance_item(compliance_item)
             if finding:
-                self.finding_progress.advance(task_id, 1)
+                findings_created += 1
+                # Backwards compatibility: check if finding_progress exists and has advance method
+                if (
+                    task_id is not None
+                    and self.finding_progress is not None
+                    and hasattr(self.finding_progress, "advance")
+                ):
+                    self.finding_progress.advance(task_id, 1)
                 yield finding
 
         # Ensure task completes if total is known
-        if total:
+        # Backwards compatibility: check if finding_progress exists and has update method
+        if (
+            total
+            and task_id is not None
+            and self.finding_progress is not None
+            and hasattr(self.finding_progress, "update")
+        ):
             self.finding_progress.update(task_id, completed=total)
 
+        logger.info(f"Prepared {findings_created} issue records from {total} failed compliance items")
+
     def sync_compliance(self) -> None:
         """
         Main method to sync compliance data.
@@ -660,6 +987,11 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         assets_processed = self.update_regscale_assets(iter(assets))
         self._log_asset_results(assets_processed)
 
+        # Refresh the asset map after creating/updating assets to ensure
+        # the map contains all assets for issue creation
+        logger.debug("Refreshing asset map after asset sync...")
+        self.asset_map_by_identifier.update(self.get_asset_map())
+
     def _log_asset_results(self, assets_processed: int) -> None:
         """
         Log asset processing results.
@@ -707,8 +1039,32 @@ class ComplianceIntegration(ScannerIntegration, ABC):
             logger.debug("No findings to process into issues")
             return
 
-        issues_created, issues_skipped = self._process_findings_to_issues(findings)
-        self._log_issue_results(issues_created, issues_skipped)
+        # Ensure asset map is populated before processing issues
+        # This handles cases where assets were created in previous runs
+        if not self.asset_map_by_identifier:
+            logger.debug("Loading asset map before issue processing...")
+            self.asset_map_by_identifier.update(self.get_asset_map())
+
+        findings_processed, findings_skipped = self._process_findings_to_issues(findings)
+
+        # CRITICAL FIX: Flush bulk issue operations to database
+        # This ensures all issues created/updated in bulk mode are persisted
+        logger.debug(f"Calling bulk_save for {findings_processed} processed findings ({findings_skipped} skipped)...")
+        issue_results = regscale_models.Issue.bulk_save()
+        logger.debug(
+            f"Bulk save completed - created: {issue_results.get('created_count', 0)}, updated: {issue_results.get('updated_count', 0)}"
+        )
+
+        # Update result counts with actual database operations
+        if hasattr(self, "_results"):
+            if "issues" not in self._results:
+                self._results["issues"] = {}
+            self._results["issues"].update(issue_results)
+
+        # Use actual database results for logging
+        issues_created = issue_results.get("created_count", 0)
+        issues_updated = issue_results.get("updated_count", 0)
+        self._log_issue_results_accurate(issues_created, issues_updated, findings_processed, findings_skipped)
 
     def _process_findings_to_issues(self, findings: List[IntegrationFinding]) -> tuple[int, int]:
         """
@@ -720,14 +1076,20 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         issues_created = 0
         issues_skipped = 0
 
-        for finding in findings:
+        logger.debug(f"Processing {len(findings)} findings into issues...")
+        for i, finding in enumerate(findings):
             try:
+                logger.debug(
+                    f"Processing finding {i + 1}/{len(findings)}: external_id='{finding.external_id}', asset_identifier='{finding.asset_identifier}"
+                )
                 if self._process_single_finding(finding):
                     issues_created += 1
+                    logger.debug(f"  -> Finding {i + 1} processed successfully")
                 else:
                     issues_skipped += 1
+                    logger.debug(f"  -> Finding {i + 1} skipped")
             except Exception as e:
-                logger.error(f"Error processing finding: {e}")
+                logger.error(f"Error processing finding {i + 1}: {e}")
                 issues_skipped += 1
 
         return issues_created, issues_skipped
@@ -739,14 +1101,25 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         :param finding: Finding to process
         :return: True if issue was created/updated, False if skipped
         """
+        logger.debug(
+            f"  -> Processing finding: external_id='{finding.external_id}', asset_identifier='{finding.asset_identifier}'"
+        )
+
         asset = self._get_or_create_asset_for_finding(finding)
         if not asset:
+            logger.debug(f"  -> Asset not found/created for identifier '{finding.asset_identifier}', skipping finding")
             self._log_asset_not_found_error(finding)
             return False
 
+        logger.debug(f"  -> Found/created asset {asset.id} for identifier '{finding.asset_identifier}'")
         issue_title = self.get_issue_title(finding)
         issue = self.create_or_update_issue_from_finding(title=issue_title, finding=finding)
-        return issue is not None
+        success = issue is not None
+        if success and issue:
+            logger.debug(f"  -> Successfully processed finding -> issue {issue.id}")
+        else:
+            logger.debug("  -> Failed to create/update issue for finding")
+        return success
 
     def _get_or_create_asset_for_finding(self, finding: IntegrationFinding) -> Optional[regscale_models.Asset]:
         """
@@ -778,6 +1151,7 @@ class ComplianceIntegration(ScannerIntegration, ABC):
     def _log_issue_results(self, issues_created: int, issues_skipped: int) -> None:
         """
         Log issue processing results.
+        DEPRECATED: Use _log_issue_results_accurate for accurate reporting.
 
         :param int issues_created: Number of issues created/updated
         :param int issues_skipped: Number of issues skipped
@@ -791,6 +1165,36 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         else:
             logger.debug("No issues processed")
 
+    def _log_issue_results_accurate(
+        self, issues_created: int, issues_updated: int, findings_processed: int, findings_skipped: int
+    ) -> None:
+        """
+        Log accurate issue processing results based on actual database operations.
+
+        :param int issues_created: Number of new issues created in database
+        :param int issues_updated: Number of existing issues updated in database
+        :param int findings_processed: Number of findings that were processed
+        :param int findings_skipped: Number of findings that were skipped
+        :return: None
+        :rtype: None
+        """
+        total_db_operations = issues_created + issues_updated
+
+        if total_db_operations > 0:
+            logger.info(
+                f"Processed {findings_processed} findings into issues: {issues_created} new issues created, {issues_updated} existing issues updated"
+            )
+            if findings_skipped > 0:
+                logger.info(f"Skipped {findings_skipped} findings (assets not found)")
+        elif findings_skipped > 0:
+            logger.warning(
+                f"Issues processed: 0 created/updated, {findings_skipped} findings skipped (assets not found)"
+            )
+        else:
+            logger.debug(
+                f"Processed {findings_processed} findings but no database changes were needed (all issues up-to-date)"
+            )
+
     def _finalize_scan_history(self, scan_history: regscale_models.ScanHistory) -> None:
         """
         Finalize scan history with error handling.
@@ -869,6 +1273,9 @@ class ComplianceIntegration(ScannerIntegration, ABC):
             logger.warning("No control implementations found for assessment processing")
             return
 
+        # Build control lookup cache for fast O(1) matching
+        self._build_control_lookup_cache(implementations)
+
         all_control_ids = set(self.passing_controls.keys()) | set(self.failing_controls.keys())
         logger.info(f"Processing assessments for {len(all_control_ids)} controls with compliance data")
         logger.info(f"Control IDs with data: {sorted(list(all_control_ids))}")
@@ -909,6 +1316,56 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         logger.info(f"Found {len(implementations)} control implementations")
         return implementations
 
+    def _build_control_lookup_cache(self, implementations: List[ControlImplementation]) -> None:
+        """
+        Build a lookup cache mapping control ID variations to implementations and security controls.
+
+        This dramatically improves performance by:
+        1. Fetching all SecurityControl objects once (instead of once per match attempt)
+        2. Pre-computing all control ID variations
+        3. Creating a dictionary for O(1) lookup instead of O(n) iteration
+
+        For 1011 implementations x 71 controls = 71,781 iterations in the old code.
+        New code: 1011 fetches + 71 dictionary lookups = ~1082 operations (67x faster!)
+
+        :param List[ControlImplementation] implementations: List of control implementations to cache
+        :return: None
+        :rtype: None
+        """
+        if self._control_lookup_cache:
+            # Cache already built
+            return
+
+        logger.debug(f"Building control lookup cache for {len(implementations)} implementations...")
+        start_time = time.time()
+
+        for implementation in implementations:
+            try:
+                security_control = SecurityControl.get_object(object_id=implementation.controlID)
+                if not security_control or not security_control.controlId:
+                    continue
+
+                # Generate all variations of this control ID for flexible matching
+                control_variations = self._control_matcher._get_control_id_variations(security_control.controlId)
+
+                # Map each variation to this implementation + security control pair
+                for variation in control_variations:
+                    # Store the first implementation found for each variation
+                    # (if multiple implementations have the same control, use the first one)
+                    if variation not in self._control_lookup_cache:
+                        self._control_lookup_cache[variation] = (implementation, security_control)
+
+            except Exception as e:  # noqa: BLE001
+                logger.error(
+                    f"Error caching implementation {implementation.id} with controlID {implementation.controlID}: {e}"
+                )
+                continue
+
+        elapsed = time.time() - start_time
+        logger.info(
+            f"Built control lookup cache with {len(self._control_lookup_cache)} control ID variations in {elapsed:.2f}s"
+        )
+
     def _log_sample_controls(self, implementations: List[ControlImplementation]) -> None:
         """
         Log sample control IDs for debugging purposes.
@@ -961,6 +1418,11 @@ class ComplianceIntegration(ScannerIntegration, ABC):
 
             if impl.id in processed_impl_today and self._find_existing_assessment_cached(impl.id, self.scan_date):
                 logger.debug(f"Skipping duplicate assessment for implementation {impl.id} (already processed today)")
+                # IMPORTANT: Still update the control implementation status even when skipping assessment creation
+                # This ensures status is updated on subsequent runs
+                if self.update_control_status:
+                    logger.debug(f"Updating control implementation status for {control_id} (existing assessment)")
+                    self._update_implementation_status(impl, result)
             else:
                 self._create_control_assessment(
                     implementation=impl,
@@ -972,7 +1434,6 @@ class ComplianceIntegration(ScannerIntegration, ABC):
                 processed_impl_today.add(impl.id)
 
             self._record_control_mapping(control_id, impl.id)
-            self._map_assets_to_control_component(sec_control, items)
             return 1
         except Exception as e:  # noqa: BLE001
             logger.error(f"Error processing control assessment for '{control_id}': {e}")
@@ -987,41 +1448,31 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         """
         Find matching implementation and security control for a control ID.
 
+        Uses ControlMatcher for robust control ID matching with leading zero normalization.
+        Performance optimized with pre-built lookup cache for O(1) matching.
+
         :param str control_id: Control identifier to match
-        :param List[ControlImplementation] implementations: Available implementations
+        :param List[ControlImplementation] implementations: Available implementations (used for fallback only)
         :return: Tuple of matching implementation and security control, or (None, None)
         :rtype: tuple[Optional[ControlImplementation], Optional[SecurityControl]]
         """
-        matching_implementation = None
-        matching_security_control = None
-        for implementation in implementations:
-            try:
-                security_control = SecurityControl.get_object(object_id=implementation.controlID)
-                if not security_control:
-                    logger.debug(
-                        f"No security control found for implementation {implementation.id} with controlID: {implementation.controlID}"
-                    )
-                    continue
-                security_control_id = security_control.controlId
-                if not security_control_id:
-                    logger.debug(f"Security control {security_control.id} has no controlId")
-                    continue
+        # Generate all variations of the search control ID for matching
+        search_variations = self._control_matcher._get_control_id_variations(control_id)
+        if not search_variations:
+            logger.debug(f"Could not generate control ID variations for: {control_id}")
+            return None, None
+
+        # Try to find a match using the pre-built lookup cache (O(1) lookup)
+        for variation in search_variations:
+            if variation in self._control_lookup_cache:
+                implementation, security_control = self._control_lookup_cache[variation]
                 logger.debug(
-                    f"Comparing extracted '{control_id}' with RegScale control '{security_control_id}' (impl: {implementation.id})"
-                )
-                if self._control_ids_match(control_id, security_control_id):
-                    matching_implementation = implementation
-                    matching_security_control = security_control
-                    logger.info(
-                        f"✅ MATCH FOUND: '{security_control_id}' == '{control_id}' (implementation: {implementation.id})"
-                    )
-                    break
-            except Exception as e:  # noqa: BLE001
-                logger.error(
-                    f"Error processing implementation {implementation.id} with controlID {implementation.controlID}: {e}"
+                    f"✅ MATCH FOUND: '{security_control.controlId}' == '{control_id}' (implementation: {implementation.id})"
                 )
-                continue
-        return matching_implementation, matching_security_control
+                return implementation, security_control
+
+        # No match found in cache
+        return None, None
 
     def _log_no_match(self, control_id: str, implementations: List[ControlImplementation]) -> None:
         """
@@ -1048,7 +1499,7 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         Determine overall assessment result for a control.
 
         :param str control_id: Control identifier to check
-        :return: Assessment result ('Pass' or 'Fail')
+        :return: Assessment result ('Pass', 'Fail', or 'Not Applicable')
         :rtype: str
         """
         is_failing = (
@@ -1056,7 +1507,18 @@ class ComplianceIntegration(ScannerIntegration, ABC):
             or control_id.lower() in self.failing_controls
             or control_id.upper() in self.failing_controls
         )
-        return "Fail" if is_failing else "Pass"
+        if is_failing:
+            return "Fail"
+
+        is_not_applicable = (
+            control_id in self.not_applicable_controls
+            or control_id.lower() in self.not_applicable_controls
+            or control_id.upper() in self.not_applicable_controls
+        )
+        if is_not_applicable:
+            return self.NOT_APPLICABLE_LABEL
+
+        return "Pass"
 
     def _get_control_compliance_items(self, control_id: str) -> List[ComplianceItem]:
         """
@@ -1093,48 +1555,11 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         except Exception:
             pass
 
-    def _map_assets_to_control_component(self, sec_control: SecurityControl, items: List[ComplianceItem]) -> None:
-        """
-        Map assets to control-specific components for organization.
-
-        :param SecurityControl sec_control: Security control to create component for
-        :param List[ComplianceItem] items: Compliance items with asset references
-        :return: None
-        :rtype: None
-        """
-        try:
-            component_title = f"Control {sec_control.controlId}"
-            component = self.components_by_title.get(component_title) if hasattr(self, "components_by_title") else None
-            if not component:
-                component = regscale_models.Component(
-                    title=component_title,
-                    componentType=regscale_models.ComponentType.Hardware,
-                    securityPlansId=self.plan_id,
-                    description=component_title,
-                    componentOwnerId=self.get_assessor_id(),
-                ).get_or_create()
-                regscale_models.ComponentMapping(
-                    componentId=component.id,
-                    securityPlanId=self.plan_id,
-                ).get_or_create()
-                if hasattr(self, "components_by_title"):
-                    self.components_by_title[component_title] = component
-
-            for item in items:
-                asset = self.get_asset_by_identifier(getattr(item, "resource_id", ""))
-                if not asset:
-                    continue
-                regscale_models.AssetMapping(
-                    assetId=asset.id,
-                    componentId=component.id,
-                ).get_or_create_with_status()
-        except Exception as map_exc:  # noqa: BLE001
-            logger.debug(f"Control-to-asset mapping skipped due to: {map_exc}")
-
     @staticmethod
     def _parse_control_id(control_id: str) -> tuple[str, Optional[str]]:
         """
         Parse a control id like 'AC-2(1)', 'AC-2 (1)', 'AC-2-1' into (base, sub).
+        Normalizes leading zeros (e.g., AC-01 becomes AC-1).
 
         Returns (base, None) when no subcontrol.
 
@@ -1148,8 +1573,22 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         if not m:
             return cid.upper(), None
         base = m.group(1).upper()
+        # Normalize leading zeros in base control number (e.g., AC-01 -> AC-1)
+        if "-" in base:
+            prefix, number = base.split("-", 1)
+            try:
+                normalized_number = str(int(number))
+                base = f"{prefix}-{normalized_number}"
+            except ValueError:
+                pass  # Keep original if conversion fails
         # Subcontrol may be captured in group 2, 3, or 4 depending on the branch matched
         sub = m.group(2) or m.group(3) or m.group(4)
+        # Normalize leading zeros in subcontrol (e.g., 01 -> 1)
+        if sub:
+            try:
+                sub = str(int(sub))
+            except ValueError:
+                pass  # Keep original if conversion fails
         return base, sub
 
     @classmethod
@@ -1181,6 +1620,7 @@ class ComplianceIntegration(ScannerIntegration, ABC):
     def _normalize_control_id(control_id: str) -> tuple[str, Optional[str]]:
         """
         Normalize control id to a canonical tuple (BASE, SUB) for set membership.
+        Normalizes leading zeros (e.g., AC-01 becomes AC-1).
 
         :param str control_id: Control identifier to normalize
         :return: Tuple of (base_control, subcontrol) in canonical form
@@ -1192,7 +1632,21 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         if not m:
             return cid.upper(), None
         base = m.group(1).upper()
+        # Normalize leading zeros in base control number (e.g., AC-01 -> AC-1)
+        if "-" in base:
+            prefix, number = base.split("-", 1)
+            try:
+                normalized_number = str(int(number))
+                base = f"{prefix}-{normalized_number}"
+            except ValueError:
+                pass  # Keep original if conversion fails
         sub = m.group(2) or m.group(3) or m.group(4)
+        # Normalize leading zeros in subcontrol (e.g., 01 -> 1)
+        if sub:
+            try:
+                sub = str(int(sub))
+            except ValueError:
+                pass  # Keep original if conversion fails
         return base, sub
 
     def _create_control_assessment(
@@ -1259,8 +1713,8 @@ class ComplianceIntegration(ScannerIntegration, ABC):
                     pass
             else:
                 # Create new assessment
+                # leadAssessorId will be set automatically from the token via the Assessment model's default_factory
                 assessment = Assessment(
-                    leadAssessorId=implementation.createdById,
                     title=f"{self.title} compliance assessment for {control_id.upper()}",
                     assessmentType="Control Testing",
                     plannedStart=get_current_datetime(),
@@ -1335,9 +1789,30 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         ]  # NOSONAR
 
         if compliance_items:
-            # Group by result for summary
-            pass_count = len([item for item in compliance_items if item.compliance_result in self.PASS_STATUSES])
-            fail_count = len(compliance_items) - pass_count
+            # Group by result for more detailed summary
+            compliant_count = len([item for item in compliance_items if item.compliance_result in self.PASS_STATUSES])
+            noncompliant_count = len(
+                [item for item in compliance_items if item.compliance_result in self.FAIL_STATUSES]
+            )
+            inconclusive_count = len(
+                [item for item in compliance_items if item.compliance_result in self.INCONCLUSIVE_STATUSES]
+            )
+            not_applicable_count = len(
+                [item for item in compliance_items if item.compliance_result in self.NOT_APPLICABLE_STATUSES]
+            )
+
+            # Calculate confidence score based on evidence quality
+            total_evaluated = compliant_count + noncompliant_count + inconclusive_count
+            confidence_score = 0
+            if total_evaluated > 0:
+                # Confidence is based on the ratio of conclusive evidence (compliant + noncompliant) to total
+                conclusive_count = compliant_count + noncompliant_count
+                confidence_score = round((conclusive_count / total_evaluated) * 100, 2)
+
+            # Calculate compliance score
+            compliance_score = 0
+            if (compliant_count + noncompliant_count) > 0:
+                compliance_score = round((compliant_count / (compliant_count + noncompliant_count)) * 100, 2)
 
             # Count unique resources across all policy assessments for this control
             unique_resources = set()
@@ -1346,7 +1821,7 @@ class ComplianceIntegration(ScannerIntegration, ABC):
             for item in compliance_items:
                 unique_resources.add(item.resource_id)
                 # Get policy name for aggregation
-                if hasattr(item, "description"):
+                if hasattr(item, "description") and item.description:  # Check for non-empty description
                     unique_policies.add(
                         item.description[:50] + "..." if len(item.description) > 50 else item.description
                     )
@@ -1358,21 +1833,374 @@ class ComplianceIntegration(ScannerIntegration, ABC):
                 f"""
             <div style="margin-top: 20px;">
                 <h4>Aggregated Assessment Summary</h4>
+                <p><strong>Control {control_id} Analysis:</strong>
+                   status=<span style="color: {result_color}; font-weight: bold;">{result.upper()}</span>,
+                   score={compliance_score:.2f},
+                   confidence={confidence_score:.0f},
+                   compliant={compliant_count},
+                   noncompliant={noncompliant_count},
+                   inconclusive={inconclusive_count}
+                   {f', not_applicable={not_applicable_count}' if not_applicable_count > 0 else ''}
+                </p>
+                <hr style="border: 0; border-top: 1px solid #e0e0e0; margin: 10px 0;">
                 <p><strong>Policy Assessments:</strong> {len(compliance_items)} total</p>
                 <p><strong>Unique Policies Tested:</strong> {len(unique_policies)}</p>
                 <p><strong>Unique Resources Assessed:</strong> {len(unique_resources)}</p>
-                <p><strong>Passing Assessments:</strong> <span style="color: #2e7d32;">{pass_count}</span></p>
-                <p><strong>Failing Assessments:</strong> <span style="color: #d32f2f;">{fail_count}</span></p>
-                <p><strong>Overall Control Result:</strong> <span style="color: {result_color}; font-weight: bold;">{result}</span></p>
+                <hr style="border: 0; border-top: 1px solid #e0e0e0; margin: 10px 0;">
+                <h5>Evidence Breakdown:</h5>
+                <p style="margin-left: 20px;">
+                    <span style="color: #2e7d32;">✓ Compliant:</span> {compliant_count}<br>
+                    <span style="color: #d32f2f;">✗ Non-Compliant:</span> {noncompliant_count}<br>
+                    <span style="color: #ff9800;">⚠ Inconclusive:</span> {inconclusive_count}
+                    {f'<br><span style="color: #9e9e9e;">- Not Applicable:</span> {not_applicable_count}' if not_applicable_count > 0 else ''}
+                </p>
+                <hr style="border: 0; border-top: 1px solid #e0e0e0; margin: 10px 0;">
+                <p><strong>Compliance Score:</strong> {compliance_score:.2f}%
+                   <span style="font-size: 0.9em; color: #666;">(compliant / (compliant + noncompliant))</span></p>
+                <p><strong>Confidence Level:</strong> {confidence_score:.0f}%
+                   <span style="font-size: 0.9em; color: #666;">(conclusive evidence ratio)</span></p>
+                <p><strong>Overall Control Result:</strong>
+                   <span style="color: {result_color}; font-weight: bold;">{result}</span></p>
             </div>
              """
             )
 
+            # Add detailed failure information
+            if result == "Fail" and noncompliant_count > 0:
+                failed_items = [item for item in compliance_items if item.compliance_result in self.FAIL_STATUSES]
+                html_parts.append(self._create_failure_details_section(failed_items))
+
+        return "\n".join(html_parts)
+
+    def _create_failure_details_section(self, failed_items: List[ComplianceItem]) -> str:
+        """
+        Create detailed failure information section for failed compliance items.
+
+        :param List[ComplianceItem] failed_items: List of failed compliance items
+        :return: HTML section with detailed failure information
+        :rtype: str
+        """
+        html_parts = [self._get_failure_section_header()]
+
+        for idx, item in enumerate(failed_items, 1):
+            html_parts.append(self._process_failed_item(idx, item))
+
+        html_parts.append("</div>")
+        return "\n".join(html_parts)
+
+    def _get_failure_section_header(self) -> str:
+        """Get the HTML header for failure details section."""
+        return """
+            <div style="margin-top: 20px; padding: 15px; background-color: #fff3e0;
+                        border-left: 4px solid #ff9800; border-radius: 5px;">
+                <h4 style="color: #e65100; margin-top: 0;">Failed Evidence Details</h4>
+            """
+
+    def _process_failed_item(self, idx: int, item: ComplianceItem) -> str:
+        """
+        Process a single failed item and return HTML.
+
+        :param int idx: The index of the failed item
+        :param ComplianceItem item: The failed compliance item
+        :return: HTML for the failed item
+        :rtype: str
+        """
+        if self._has_aws_evidence(item):
+            return self._process_aws_item_with_evidence(idx, item)
+        return self._process_non_aws_item(idx, item)
+
+    def _has_aws_evidence(self, item: ComplianceItem) -> bool:
+        """Check if item has AWS Audit Manager evidence."""
+        return hasattr(item, "evidence_items") and item.evidence_items
+
+    def _process_aws_item_with_evidence(self, idx: int, item: ComplianceItem) -> str:
+        """Process AWS item with evidence details."""
+        evidence_categories = self._categorize_evidence(item)
+
+        if not evidence_categories["failed"]:
+            return ""
+
+        html_parts = []
+        html_parts.append(self._create_failed_check_header(idx, item, evidence_categories))
+        html_parts.append(self._create_failed_evidence_details(evidence_categories["failed"]))
+        html_parts.append(self._add_remediation_guidance(item))
+        html_parts.append("</div>")
+
         return "\n".join(html_parts)
 
+    def _categorize_evidence(self, item: ComplianceItem) -> Dict[str, List[Any]]:
+        """
+        Categorize evidence items by compliance status.
+
+        :param ComplianceItem item: The compliance item with evidence
+        :return: Dictionary with categorized evidence
+        :rtype: Dict[str, List[Any]]
+        """
+        categories = {"failed": [], "compliant": [], "inconclusive": []}
+
+        for evidence in item.evidence_items:
+            compliance_check = self._get_evidence_compliance_check(item, evidence)
+
+            if compliance_check == "FAILED":
+                categories["failed"].append(evidence)
+            elif compliance_check == "COMPLIANT":
+                categories["compliant"].append(evidence)
+            else:
+                categories["inconclusive"].append(evidence)
+
+        return categories
+
+    def _get_evidence_compliance_check(self, item: ComplianceItem, evidence: Any) -> Optional[str]:
+        """Get compliance check result for evidence."""
+        if hasattr(item, "_get_evidence_compliance"):
+            return item._get_evidence_compliance(evidence)
+        return None
+
+    def _create_failed_check_header(
+        self, idx: int, item: ComplianceItem, evidence_categories: Dict[str, List[Any]]
+    ) -> str:
+        """Create HTML header for failed check."""
+        return f"""
+            <div style="margin-top: 15px; padding: 10px; background-color: #ffebee; border-radius: 3px;">
+                <h5 style="color: #c62828; margin-top: 0;">
+                    Failed Check #{idx}: {item.control_id}
+                </h5>
+                <p><strong>Resource:</strong> {getattr(item, 'resource_name', item.resource_id)}</p>
+                <p><strong>Evidence Summary:</strong>
+                   {len(evidence_categories["failed"])} failed, {len(evidence_categories["compliant"])} compliant,
+                   {len(evidence_categories["inconclusive"])} inconclusive</p>
+            """
+
+    def _create_failed_evidence_details(self, failed_evidence: List[Any]) -> str:
+        """Create HTML for failed evidence details."""
+        html_parts = ['<div style="margin-top: 10px;"><strong>Failed Evidence:</strong><ul>']
+
+        # Limit to 10 failed evidence items
+        for evidence in failed_evidence[:10]:
+            html_parts.append(self._format_single_evidence(evidence))
+
+        html_parts.append("</ul></div>")
+        return "\n".join(html_parts)
+
+    def _format_single_evidence(self, evidence: Dict[str, Any]) -> str:
+        """Format a single evidence item as HTML."""
+        evidence_html = []
+        evidence_source = evidence.get("dataSource", "Unknown source")
+        evidence_id = evidence.get("id", "")[:50]
+
+        evidence_html.append(f"<li><strong>Source:</strong> {evidence_source}")
+
+        if evidence_id:
+            evidence_html.append(f"<br><strong>Evidence ID:</strong> {evidence_id}")
+
+        resources_info = self._get_resources_info(evidence)
+        if resources_info:
+            evidence_html.append(f'<br><strong>Resources:</strong><ul><li>{"</li><li>".join(resources_info)}</li></ul>')
+
+        evidence_html.append("</li>")
+        return "\n".join(evidence_html)
+
+    def _get_resources_info(self, evidence: Dict[str, Any]) -> List[str]:
+        """Extract resource information from evidence."""
+        resources_info = []
+        resources_included = evidence.get("resourcesIncluded", [])
+
+        # Limit to 5 resources per evidence
+        for resource in resources_included[:5]:
+            resource_str = self._format_resource(resource)
+            if resource_str:
+                resources_info.append(resource_str)
+
+        return resources_info
+
+    def _format_resource(self, resource: Dict[str, Any]) -> Optional[str]:
+        """Format a single resource as a string."""
+        resource_type = resource.get("type", "Unknown")
+        resource_value = resource.get("value", "")[:100]
+        resource_check = resource.get("complianceCheck", "N/A")
+
+        if resource_value:
+            return f"{resource_type}: {resource_value} (Status: {resource_check})"
+        return None
+
+    def _add_remediation_guidance(self, item: ComplianceItem) -> str:
+        """Add remediation guidance if available."""
+        if not (hasattr(item, "action_plan_instructions") and item.action_plan_instructions):
+            return ""
+
+        instructions = item.action_plan_instructions[:500]
+        truncated = "..." if len(item.action_plan_instructions) > 500 else ""
+
+        return f"""
+            <div style="margin-top: 10px; padding: 8px; background-color: #e3f2fd;
+                        border-left: 3px solid #1976d2; border-radius: 3px;">
+                <strong>Remediation Guidance:</strong><br>
+                {instructions}{truncated}
+            </div>
+            """
+
+    def _process_non_aws_item(self, idx: int, item: ComplianceItem) -> str:
+        """Process non-AWS items or items without evidence."""
+        description = self._get_item_description(item)
+
+        return f"""
+            <div style="margin-top: 15px; padding: 10px; background-color: #ffebee; border-radius: 3px;">
+                <h5 style="color: #c62828; margin-top: 0;">Failed Check #{idx}: {item.control_id}</h5>
+                <p><strong>Resource:</strong> {getattr(item, 'resource_name', item.resource_id)}</p>
+                <p><strong>Description:</strong> {description}</p>
+            </div>
+            """
+
+    def _get_item_description(self, item: ComplianceItem) -> str:
+        """Get truncated description from item."""
+        if hasattr(item, "description"):
+            return item.description[:200]
+        return "N/A"
+
+    def _get_security_plan(self) -> Optional[regscale_models.SecurityPlan]:
+        """
+        Get the security plan for this integration.
+
+        :return: SecurityPlan instance or None
+        :rtype: Optional[regscale_models.SecurityPlan]
+        """
+        if not self._security_plan_loaded:
+            self._security_plan_loaded = True  # Mark as attempted to prevent repeated calls
+            try:
+                logger.info(f"[SECURITY PLAN] Retrieving security plan with ID: {self.plan_id}")
+                self._security_plan = SecurityPlan.get_object(object_id=self.plan_id)
+                if self._security_plan:
+                    logger.info(
+                        f"[SECURITY PLAN] Retrieved security plan: {getattr(self._security_plan, 'systemName', 'N/A')}"
+                    )
+                    logger.info(
+                        f"[SECURITY PLAN] complianceSettingsId: {getattr(self._security_plan, 'complianceSettingsId', None)}"
+                    )
+                else:
+                    logger.warning(f"[SECURITY PLAN] No security plan found with ID: {self.plan_id}")
+            except Exception as e:
+                logger.error(f"[SECURITY PLAN] Error getting security plan {self.plan_id}: {e}")
+                # Don't set to None - keep as None but mark as loaded to prevent retry
+        return self._security_plan
+
+    def _get_compliance_settings(self) -> Optional[regscale_models.ComplianceSettings]:
+        """
+        Get compliance settings for the security plan.
+
+        :return: ComplianceSettings instance or None
+        :rtype: Optional[regscale_models.ComplianceSettings]
+        """
+        if not self._compliance_settings_loaded:
+            self._compliance_settings_loaded = True  # Mark as attempted to prevent repeated calls
+            try:
+                security_plan = self._get_security_plan()
+                logger.debug(f"[COMPLIANCE SETTINGS] Security plan retrieved: {security_plan is not None}")
+                if security_plan:
+                    logger.debug(
+                        f"[COMPLIANCE SETTINGS] Security plan systemName: {getattr(security_plan, 'systemName', 'N/A')}"
+                    )
+                    logger.debug(f"[COMPLIANCE SETTINGS] Security plan ID: {getattr(security_plan, 'id', 'N/A')}")
+                    logger.debug(
+                        f"[COMPLIANCE SETTINGS] Has complianceSettingsId attribute: {hasattr(security_plan, 'complianceSettingsId')}"
+                    )
+                    logger.debug(
+                        f"[COMPLIANCE SETTINGS] complianceSettingsId value: {getattr(security_plan, 'complianceSettingsId', 'None')}"
+                    )
+
+                if self._has_valid_compliance_settings_id(security_plan):
+                    self._compliance_settings = self._fetch_compliance_settings(security_plan)
+                else:
+                    self._log_missing_compliance_settings_reason(security_plan)
+            except Exception as e:
+                logger.debug(f"Error getting compliance settings: {e}")
+                import traceback
+
+                logger.debug(f"Full traceback: {traceback.format_exc()}")
+                # Don't set to None - keep as None but mark as loaded to prevent retry
+        return self._compliance_settings
+
+    def _has_valid_compliance_settings_id(self, security_plan) -> bool:
+        """Check if security plan has valid compliance settings ID."""
+        return security_plan and hasattr(security_plan, "complianceSettingsId") and security_plan.complianceSettingsId
+
+    def _fetch_compliance_settings(self, security_plan) -> Optional[regscale_models.ComplianceSettings]:
+        """Fetch and log compliance settings."""
+        logger.debug(f"Retrieving compliance settings with ID: {security_plan.complianceSettingsId}")
+        compliance_settings = ComplianceSettings.get_object(object_id=security_plan.complianceSettingsId)
+
+        if compliance_settings:
+            logger.debug(f"Using compliance settings: {compliance_settings.title}")
+            logger.debug(
+                f"Compliance settings has field groups: {bool(getattr(compliance_settings, 'complianceSettingsFieldGroups', None))}"
+            )
+        else:
+            logger.debug(f"No compliance settings found for ID: {security_plan.complianceSettingsId}")
+
+        return compliance_settings
+
+    def _log_missing_compliance_settings_reason(self, security_plan) -> None:
+        """Log specific reason why compliance settings are not available."""
+        if not security_plan:
+            logger.debug("Security plan not found")
+        elif not hasattr(security_plan, "complianceSettingsId"):
+            logger.debug("Security plan does not have complianceSettingsId attribute")
+        elif not security_plan.complianceSettingsId:
+            logger.debug("Security plan has no complianceSettingsId set")
+
+    def _get_implementation_status_from_result(self, result: str, override: Optional[str] = None) -> str:
+        """
+        Get implementation status based on assessment result using enum-based mappings.
+        Results are cached to avoid repeated calculations for the same input.
+
+        :param str result: Assessment result ('Pass', 'Fail', 'Not Applicable', etc.)
+        :param Optional[str] override: Optional override value to use instead of the mapping
+        :return: Implementation status string
+        :rtype: str
+        """
+        # Use override directly if provided
+        if override:
+            return override
+
+        # Handle None or empty result
+        if not result:
+            logger.warning("Received None or empty result for status mapping, defaulting to 'Unknown'")
+            return "Unknown"
+
+        # Check cache first to avoid repeated calculations
+        cache_key = result.lower().strip()
+        if cache_key in self._status_mapping_cache:
+            cached_status = self._status_mapping_cache[cache_key]
+            logger.debug(f"[STATUS MAPPING] Using cached mapping for '{result}': '{cached_status}'")
+            return cached_status
+
+        logger.info(f"[STATUS MAPPING] Getting implementation status for result: {result}")
+
+        # Try to use the compliance settings mapping function
+        compliance_settings = self._get_compliance_settings()
+        if compliance_settings:
+            logger.info(f"[STATUS MAPPING] Using compliance settings '{compliance_settings.title}' for mapping")
+            try:
+                mapped_status = compliance_settings.get_implementation_status_for_result(result, None)
+                logger.info(f"[STATUS MAPPING] Mapped '{result}' to '{mapped_status}' using compliance settings")
+                # Cache the result
+                self._status_mapping_cache[cache_key] = mapped_status
+                return mapped_status
+            except Exception as e:
+                logger.warning(f"[STATUS MAPPING] Error using compliance settings mapping: {e}")
+
+        # Fallback: Use the class method directly if no compliance settings instance
+        framework = self._detect_compliance_framework()
+        logger.info(f"[STATUS MAPPING] Using framework '{framework}' for fallback mapping")
+        mapped_status = ComplianceSettings.get_status_mapping(framework, result, None)
+        logger.info(f"[STATUS MAPPING] Mapped '{result}' to '{mapped_status}' using fallback")
+        # Cache the result
+        self._status_mapping_cache[cache_key] = mapped_status
+        return mapped_status
+
     def _update_implementation_status(self, implementation: ControlImplementation, result: str) -> None:
         """
         Update control implementation status based on assessment result.
+        Uses compliance settings from the security plan if available, otherwise falls back to defaults.
 
         :param ControlImplementation implementation: Control implementation to update
         :param str result: Assessment result ('Pass' or 'Fail')
@@ -1380,15 +2208,30 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         :rtype: None
         """
         try:
-            if result == "Pass":
-                new_status = "Fully Implemented"
-            else:
-                new_status = "In Remediation"
+            # Get status from compliance settings or fallback to default
+            new_status = self._get_implementation_status_from_result(result)
 
             # Update implementation status
             implementation.status = new_status
             implementation.dateLastAssessed = get_current_datetime()
             implementation.lastAssessmentResult = result
+
+            # Ensure required fields are set if empty
+            if not implementation.responsibility:
+                implementation.responsibility = ControlImplementation.get_default_responsibility(
+                    parent_id=implementation.parentId
+                )
+                logger.debug(
+                    f"Setting default responsibility for implementation {implementation.id}: {implementation.responsibility}"
+                )
+
+            if not implementation.implementation:
+                control_id = (
+                    getattr(implementation.control, "controlId", "control") if implementation.control else "control"
+                )
+                implementation.implementation = f"Implementation details for {control_id} will be documented."
+                logger.debug(f"Setting default implementation statement for implementation {implementation.id}")
+
             implementation.save()
 
             # Update objectives if they exist
@@ -1401,7 +2244,7 @@ class ComplianceIntegration(ScannerIntegration, ABC):
                 objective.status = new_status
                 objective.save()
 
-            logger.debug(f"Updated implementation status to {new_status}")
+            logger.debug(f"Updated implementation status to {new_status} (from compliance settings)")
 
         except Exception as e:
             logger.error(f"Error updating implementation status: {e}")
@@ -1558,6 +2401,8 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         """
         Create or update an issue from a finding, using cache to prevent duplicates.
 
+        Properly handles milestone creation for compliance integrations.
+
         :param str title: Issue title
         :param IntegrationFinding finding: The finding to create issue from
         :return: Created or updated issue
@@ -1568,17 +2413,21 @@ class ComplianceIntegration(ScannerIntegration, ABC):
 
         # Check for existing issue by external_id first
         external_id = finding.external_id
+        logger.debug(f"Looking for existing issue with external_id: '{external_id}'")
         existing_issue = self._find_existing_issue_cached(external_id)
 
         if existing_issue:
             logger.debug(
-                f"Found existing issue {existing_issue.id} for external_id {external_id}, updating instead of creating"
+                f"Found existing issue {existing_issue.id} (other_identifier: '{existing_issue.otherIdentifier}') for lookup external_id '{external_id}', updating instead of creating"
             )
 
+            # Store original status for milestone comparison
+            original_status = existing_issue.status
+
             # Update existing issue with new finding data
             existing_issue.title = title
             existing_issue.description = finding.description
-            existing_issue.severity = finding.severity
+            existing_issue.severityLevel = finding.severity
             existing_issue.status = finding.status
             # Ensure affectedControls is updated from the finding's control id
             try:
@@ -1598,10 +2447,46 @@ class ComplianceIntegration(ScannerIntegration, ABC):
             except Exception:
                 pass
             existing_issue.dateLastUpdated = self.scan_date
+            # Set organization ID based on Issue Owner or SSP Owner hierarchy
+            existing_issue.orgId = self.determine_issue_organization_id(existing_issue.issueOwnerId)
             existing_issue.save()
 
+            # Create milestone if status changed
+            # Reconstruct original issue state for comparison
+            original_issue = regscale_models.Issue()
+            original_issue.status = original_status
+            self._create_milestones_for_updated_issue(existing_issue, finding, original_issue)
+
             return existing_issue
         else:
             # No existing issue found, create new one using parent method
             logger.debug(f"No existing issue found for external_id {external_id}, creating new issue")
             return super().create_or_update_issue_from_finding(title, finding)
+
+    def _create_milestones_for_updated_issue(
+        self,
+        issue: regscale_models.Issue,
+        finding: IntegrationFinding,
+        original_issue: regscale_models.Issue,
+    ) -> None:
+        """
+        Create milestones for an updated issue in compliance integration.
+
+        This method handles both status transition milestones and backfilling of missing
+        creation milestones for existing issues.
+
+        :param regscale_models.Issue issue: The updated issue
+        :param IntegrationFinding finding: The finding data
+        :param regscale_models.Issue original_issue: Original state for comparison
+        """
+        milestone_manager = self.get_milestone_manager()
+
+        # First, ensure the issue has a creation milestone (backfill if missing)
+        milestone_manager.ensure_creation_milestone_exists(issue=issue, finding=finding)
+
+        # Then, handle status transition milestones
+        milestone_manager.create_milestones_for_issue(
+            issue=issue,
+            finding=finding,
+            existing_issue=original_issue,
+        )