regscale-cli 6.23.0.1__py3-none-any.whl → 6.24.0.0__py3-none-any.whl

regscale/integrations/compliance_integration.py

@@ -225,9 +225,7 @@ class ComplianceIntegration(ScannerIntegration, ABC):
             )
 
             for asset in existing_assets:
-                # Cache by external_id, identifier, and other_tracking_number for flexible lookup
-                if hasattr(asset, "externalId") and asset.externalId:
-                    self._existing_assets_cache[asset.externalId] = asset
+                # Cache by identifier and other_tracking_number for flexible lookup
                 if hasattr(asset, "identifier") and asset.identifier:
                     self._existing_assets_cache[asset.identifier] = asset
                 if hasattr(asset, "otherTrackingNumber") and asset.otherTrackingNumber:
@@ -283,13 +281,11 @@ class ComplianceIntegration(ScannerIntegration, ABC):
             wiz_issues = 0
             for issue in all_issues:
                 # Cache by external_id and other_identifier for flexible lookup
-                if hasattr(issue, "externalId") and issue.externalId:
-                    self._existing_issues_cache[issue.externalId] = issue
-                    if "wiz-policy" in issue.externalId.lower():
-                        wiz_issues += 1
-                        logger.debug(f"Cached Wiz issue: {issue.id} -> external_id: {issue.externalId}")
                 if hasattr(issue, "otherIdentifier") and issue.otherIdentifier:
                     self._existing_issues_cache[issue.otherIdentifier] = issue
+                    if "wiz-policy" in issue.otherIdentifier.lower():
+                        wiz_issues += 1
+                        logger.debug(f"Cached Wiz issue: {issue.id} -> other_identifier: {issue.otherIdentifier}")
 
             logger.debug(f"Cached {wiz_issues} Wiz policy issues out of {len(all_issues)} total issues")
 
@@ -411,17 +407,32 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         """
         logger.info("Processing compliance data...")
 
-        # Reset state to avoid double counting on repeated calls
+        self._reset_compliance_state()
+        allowed_controls = self._build_allowed_controls_set()
+        raw_compliance_data = self.fetch_compliance_data()
+
+        processing_stats = self._process_raw_compliance_items(raw_compliance_data, allowed_controls)
+        self._log_processing_summary(raw_compliance_data, processing_stats)
+
+        # Perform control-level categorization based on aggregated results
+        self._categorize_controls_by_aggregation()
+        self._log_final_results()
+
+    def _reset_compliance_state(self) -> None:
+        """Reset state to avoid double counting on repeated calls."""
         self.all_compliance_items = []
         self.failed_compliance_items = []
         self.passing_controls = {}
         self.failing_controls = {}
         self.asset_compliance_map.clear()
 
-        # Build allowed control IDs from plan/catalog controls to restrict scope
+    def _build_allowed_controls_set(self) -> set[str]:
+        """Build allowed control IDs from plan/catalog controls to restrict scope."""
         allowed_controls_normalized: set[str] = set()
         try:
             controls = self._get_controls()
+            logger.debug(f"Loaded {len(controls)} controls from plan/catalog")
+
             for ctl in controls:
                 cid = (ctl.get("controlId") or "").strip()
                 if not cid:
@@ -429,56 +440,220 @@ class ComplianceIntegration(ScannerIntegration, ABC):
                 base, sub = self._normalize_control_id(cid)
                 normalized = f"{base}({sub})" if sub else base
                 allowed_controls_normalized.add(normalized)
-        except Exception:
-            # If controls cannot be loaded, proceed without additional filtering
+
+            logger.debug(f"Built allowed_controls_normalized set with {len(allowed_controls_normalized)} entries")
+            if allowed_controls_normalized:
+                sample = sorted(allowed_controls_normalized)[:5]
+                logger.debug(f"Sample allowed controls: {sample}")
+        except Exception as e:
+            logger.warning(f"Could not load controls from plan/catalog: {e}")
             allowed_controls_normalized = set()
 
-        # Fetch raw compliance data
-        raw_compliance_data = self.fetch_compliance_data()
+        return allowed_controls_normalized
+
+    def _process_raw_compliance_items(self, raw_compliance_data: list, allowed_controls: set) -> dict:
+        """Process raw compliance items and return processing statistics.
+        :param list raw_compliance_data: Raw compliance data from external system
+        :param set allowed_controls: Allowed control IDs
+        :return: Processed compliance items
+        :rtype: dict
+        """
+        stats = {"skipped_no_control": 0, "skipped_no_resource": 0, "skipped_not_in_plan": 0, "processed_count": 0}
 
-        # Convert to ComplianceItem objects
         for raw_item in raw_compliance_data:
             try:
                 compliance_item = self.create_compliance_item(raw_item)
-                # Skip items that do not resolve to a control or resource
-                if not getattr(compliance_item, "control_id", "") or not getattr(compliance_item, "resource_id", ""):
+                if not self._process_single_compliance_item(compliance_item, allowed_controls, stats):
                     continue
-
-                # If we have an allowed set, restrict to only controls in current plan/catalog
-                if allowed_controls_normalized:
-                    base, sub = self._normalize_control_id(getattr(compliance_item, "control_id", ""))
-                    norm_item = f"{base}({sub})" if sub else base
-                    if norm_item not in allowed_controls_normalized:
-                        continue
-                self.all_compliance_items.append(compliance_item)
-
-                # Build asset mapping
-                self.asset_compliance_map[compliance_item.resource_id].append(compliance_item)
-
-                # Categorize by result
-                if compliance_item.compliance_result in self.FAIL_STATUSES:
-                    self.failed_compliance_items.append(compliance_item)
-                    # Track failing controls (control can fail if ANY asset fails)
-                    control_key = compliance_item.control_id.lower()
-                    self.failing_controls[control_key] = compliance_item
-                    # Remove from passing if it was there
-                    self.passing_controls.pop(control_key, None)
-
-                elif compliance_item.compliance_result in self.PASS_STATUSES:
-                    control_key = compliance_item.control_id.lower()
-                    # Only mark as passing if not already failing
-                    if control_key not in self.failing_controls:
-                        self.passing_controls[control_key] = compliance_item
-
             except Exception as e:
                 logger.error(f"Error processing compliance item: {e}")
                 continue
 
+        return stats
+
+    def _process_single_compliance_item(self, compliance_item: Any, allowed_controls: set, stats: dict) -> bool:
+        """Process a single compliance item and update statistics. Returns True if processed successfully."""
+        control_id = getattr(compliance_item, "control_id", "")
+        resource_id = getattr(compliance_item, "resource_id", "")
+
+        if not control_id:
+            stats["skipped_no_control"] += 1
+            return False
+        if not resource_id:
+            stats["skipped_no_resource"] += 1
+            return False
+
+        if not self._should_process_item(compliance_item, control_id, allowed_controls, stats):
+            return False
+
+        self._add_processed_item(compliance_item, stats)
+        return True
+
+    def _should_process_item(self, compliance_item: Any, control_id: str, allowed_controls: set, stats: dict) -> bool:
+        """Determine if an item should be processed based on control filtering."""
+        if not allowed_controls:
+            return True
+
+        base, sub = self._normalize_control_id(control_id)
+        norm_item = f"{base}({sub})" if sub else base
+
+        if norm_item in allowed_controls:
+            return True
+
+        # Allow PASS controls through even if they don't have existing implementations
+        if compliance_item.compliance_result in self.PASS_STATUSES:
+            return True
+
+        stats["skipped_not_in_plan"] += 1
+        if stats["skipped_not_in_plan"] <= 3:
+            logger.debug(f"Skipping control {norm_item} - not in plan (result: {compliance_item.compliance_result})")
+        return False
+
+    def _add_processed_item(self, compliance_item: Any, stats: dict) -> None:
+        """Add a processed item to collections and update statistics."""
+        self.all_compliance_items.append(compliance_item)
+        stats["processed_count"] += 1
+
+        # Build asset mapping
+        self.asset_compliance_map[compliance_item.resource_id].append(compliance_item)
+
+        # Categorize by result
+        if compliance_item.compliance_result in self.FAIL_STATUSES:
+            self.failed_compliance_items.append(compliance_item)
+
+    def _log_processing_summary(self, raw_compliance_data: list, stats: dict) -> None:
+        """Log summary of compliance data processing."""
+        logger.debug("Compliance item processing summary:")
+        logger.debug(f"  - Total raw items: {len(raw_compliance_data)}")
+        logger.debug(f"  - Skipped (no control_id): {stats['skipped_no_control']}")
+        logger.debug(f"  - Skipped (no resource_id): {stats['skipped_no_resource']}")
+        logger.debug(f"  - Skipped (not in plan): {stats['skipped_not_in_plan']}")
+        logger.debug(f"  - Processed successfully: {stats['processed_count']}")
+
+    def _log_final_results(self) -> None:
+        """Log final processing results."""
         logger.debug(
             f"Processed {len(self.all_compliance_items)} compliance items: "
             f"{len(self.all_compliance_items) - len(self.failed_compliance_items)} passing, "
             f"{len(self.failed_compliance_items)} failing"
         )
+        logger.debug(
+            f"Control categorization: {len(self.passing_controls)} passing controls, "
+            f"{len(self.failing_controls)} failing controls"
+        )
+
+    def _categorize_controls_by_aggregation(self) -> None:
+        """
+        Categorize controls as passing or failing based on aggregated results across all compliance items.
+
+        This method uses project-scoped aggregation logic instead of the previous "any fail = control fails"
+        approach. For project-scoped integrations (like Wiz), this provides more accurate control status.
+        """
+
+        # Group all compliance items by control ID
+        control_items = self._group_items_by_control()
+
+        # Analyze each control's results
+        for control_key, items in control_items.items():
+            self._categorize_single_control(control_key, items)
+
+    def _group_items_by_control(self) -> dict:
+        """Group compliance items by control ID."""
+        from collections import defaultdict
+
+        control_items = defaultdict(list)
+        for item in self.all_compliance_items:
+            control_key = item.control_id.lower()
+            control_items[control_key].append(item)
+
+        return control_items
+
+    def _categorize_single_control(self, control_key: str, items: list) -> None:
+        """Categorize a single control based on its compliance items."""
+        from collections import Counter
+
+        results = [item.compliance_result for item in items]
+        result_counts = Counter(results)
+        total_items = len(results)
+
+        fail_count, pass_count = self._count_pass_fail_results(result_counts)
+
+        if fail_count == 0 and pass_count > 0:
+            self._mark_control_as_passing(control_key, items, pass_count, fail_count)
+        elif fail_count > 0:
+            self._handle_control_with_failures(control_key, items, fail_count, pass_count, total_items)
+        else:
+            logger.debug(f"Control {control_key} has unclear results: {dict(result_counts)}")
+
+    def _count_pass_fail_results(self, result_counts: dict) -> tuple[int, int]:
+        """Count pass and fail results from result counts."""
+        fail_statuses_lower = [status.lower() for status in self.FAIL_STATUSES]
+        pass_statuses_lower = [status.lower() for status in self.PASS_STATUSES]
+
+        fail_count = 0
+        pass_count = 0
+
+        for result, count in result_counts.items():
+            result_lower = result.lower()
+            if result_lower in fail_statuses_lower:
+                fail_count += count
+            elif result_lower in pass_statuses_lower:
+                pass_count += count
+
+        return fail_count, pass_count
+
+    def _mark_control_as_passing(self, control_key: str, items: list, pass_count: int, fail_count: int) -> None:
+        """Mark a control as passing."""
+        self.passing_controls[control_key] = items[0]  # Use first item as representative
+        logger.debug(f"Control {control_key} marked as PASSING: {pass_count}P/{fail_count}F")
+
+    def _handle_control_with_failures(
+        self, control_key: str, items: list, fail_count: int, pass_count: int, total_items: int
+    ) -> None:
+        """Handle a control that has some failures."""
+        fail_ratio = fail_count / total_items
+        failure_threshold = getattr(self, "control_failure_threshold", 0.2)
+
+        if fail_ratio > failure_threshold:
+            self._mark_control_as_failing(control_key, items, pass_count, fail_count, fail_ratio, failure_threshold)
+        else:
+            self._mark_control_as_passing_with_warnings(
+                control_key, items, pass_count, fail_count, fail_ratio, failure_threshold
+            )
+
+    def _mark_control_as_failing(
+        self,
+        control_key: str,
+        items: list,
+        pass_count: int,
+        fail_count: int,
+        fail_ratio: float,
+        failure_threshold: float,
+    ) -> None:
+        """Mark a control as failing due to significant failures."""
+        fail_statuses_lower = [status.lower() for status in self.FAIL_STATUSES]
+        failing_item = next(item for item in items if item.compliance_result.lower() in fail_statuses_lower)
+        self.failing_controls[control_key] = failing_item
+        logger.debug(
+            f"Control {control_key} marked as FAILING: {pass_count}P/{fail_count}F "
+            f"({fail_ratio:.1%} fail rate > {failure_threshold:.1%} threshold)"
+        )
+
+    def _mark_control_as_passing_with_warnings(
+        self,
+        control_key: str,
+        items: list,
+        pass_count: int,
+        fail_count: int,
+        fail_ratio: float,
+        failure_threshold: float,
+    ) -> None:
+        """Mark a control as passing despite low failure rate."""
+        self.passing_controls[control_key] = items[0]
+        logger.debug(
+            f"Control {control_key} marked as PASSING (low fail rate): {pass_count}P/{fail_count}F "
+            f"({fail_ratio:.1%} fail rate < {failure_threshold:.1%} threshold)"
+        )
 
     def create_asset_from_compliance_item(self, compliance_item: ComplianceItem) -> Optional[IntegrationAsset]:
         """
@@ -660,6 +835,11 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         assets_processed = self.update_regscale_assets(iter(assets))
         self._log_asset_results(assets_processed)
 
+        # Refresh the asset map after creating/updating assets to ensure
+        # the map contains all assets for issue creation
+        logger.debug("Refreshing asset map after asset sync...")
+        self.asset_map_by_identifier.update(self.get_asset_map())
+
     def _log_asset_results(self, assets_processed: int) -> None:
         """
         Log asset processing results.
@@ -707,8 +887,32 @@ class ComplianceIntegration(ScannerIntegration, ABC):
             logger.debug("No findings to process into issues")
             return
 
-        issues_created, issues_skipped = self._process_findings_to_issues(findings)
-        self._log_issue_results(issues_created, issues_skipped)
+        # Ensure asset map is populated before processing issues
+        # This handles cases where assets were created in previous runs
+        if not self.asset_map_by_identifier:
+            logger.debug("Loading asset map before issue processing...")
+            self.asset_map_by_identifier.update(self.get_asset_map())
+
+        findings_processed, findings_skipped = self._process_findings_to_issues(findings)
+
+        # CRITICAL FIX: Flush bulk issue operations to database
+        # This ensures all issues created/updated in bulk mode are persisted
+        logger.debug(f"Calling bulk_save for {findings_processed} processed findings ({findings_skipped} skipped)...")
+        issue_results = regscale_models.Issue.bulk_save()
+        logger.debug(
+            f"Bulk save completed - created: {issue_results.get('created_count', 0)}, updated: {issue_results.get('updated_count', 0)}"
+        )
+
+        # Update result counts with actual database operations
+        if hasattr(self, "_results"):
+            if "issues" not in self._results:
+                self._results["issues"] = {}
+            self._results["issues"].update(issue_results)
+
+        # Use actual database results for logging
+        issues_created = issue_results.get("created_count", 0)
+        issues_updated = issue_results.get("updated_count", 0)
+        self._log_issue_results_accurate(issues_created, issues_updated, findings_processed, findings_skipped)
 
     def _process_findings_to_issues(self, findings: List[IntegrationFinding]) -> tuple[int, int]:
         """
@@ -720,14 +924,20 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         issues_created = 0
         issues_skipped = 0
 
-        for finding in findings:
+        logger.debug(f"Processing {len(findings)} findings into issues...")
+        for i, finding in enumerate(findings):
             try:
+                logger.debug(
+                    f"Processing finding {i + 1}/{len(findings)}: external_id='{finding.external_id}', asset_identifier='{finding.asset_identifier}"
+                )
                 if self._process_single_finding(finding):
                     issues_created += 1
+                    logger.debug(f"  -> Finding {i + 1} processed successfully")
                 else:
                     issues_skipped += 1
+                    logger.debug(f"  -> Finding {i + 1} skipped")
             except Exception as e:
-                logger.error(f"Error processing finding: {e}")
+                logger.error(f"Error processing finding {i + 1}: {e}")
                 issues_skipped += 1
 
         return issues_created, issues_skipped
@@ -739,14 +949,25 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         :param finding: Finding to process
         :return: True if issue was created/updated, False if skipped
         """
+        logger.debug(
+            f"  -> Processing finding: external_id='{finding.external_id}', asset_identifier='{finding.asset_identifier}'"
+        )
+
         asset = self._get_or_create_asset_for_finding(finding)
         if not asset:
+            logger.debug(f"  -> Asset not found/created for identifier '{finding.asset_identifier}', skipping finding")
             self._log_asset_not_found_error(finding)
             return False
 
+        logger.debug(f"  -> Found/created asset {asset.id} for identifier '{finding.asset_identifier}'")
         issue_title = self.get_issue_title(finding)
         issue = self.create_or_update_issue_from_finding(title=issue_title, finding=finding)
-        return issue is not None
+        success = issue is not None
+        if success and issue:
+            logger.debug(f"  -> Successfully processed finding -> issue {issue.id}")
+        else:
+            logger.debug("  -> Failed to create/update issue for finding")
+        return success
 
     def _get_or_create_asset_for_finding(self, finding: IntegrationFinding) -> Optional[regscale_models.Asset]:
         """
@@ -778,6 +999,7 @@ class ComplianceIntegration(ScannerIntegration, ABC):
     def _log_issue_results(self, issues_created: int, issues_skipped: int) -> None:
         """
         Log issue processing results.
+        DEPRECATED: Use _log_issue_results_accurate for accurate reporting.
 
         :param int issues_created: Number of issues created/updated
         :param int issues_skipped: Number of issues skipped
@@ -791,6 +1013,36 @@ class ComplianceIntegration(ScannerIntegration, ABC):
         else:
             logger.debug("No issues processed")
 
+    def _log_issue_results_accurate(
+        self, issues_created: int, issues_updated: int, findings_processed: int, findings_skipped: int
+    ) -> None:
+        """
+        Log accurate issue processing results based on actual database operations.
+
+        :param int issues_created: Number of new issues created in database
+        :param int issues_updated: Number of existing issues updated in database
+        :param int findings_processed: Number of findings that were processed
+        :param int findings_skipped: Number of findings that were skipped
+        :return: None
+        :rtype: None
+        """
+        total_db_operations = issues_created + issues_updated
+
+        if total_db_operations > 0:
+            logger.info(
+                f"Processed {findings_processed} findings into issues: {issues_created} new issues created, {issues_updated} existing issues updated"
+            )
+            if findings_skipped > 0:
+                logger.info(f"Skipped {findings_skipped} findings (assets not found)")
+        elif findings_skipped > 0:
+            logger.warning(
+                f"Issues processed: 0 created/updated, {findings_skipped} findings skipped (assets not found)"
+            )
+        else:
+            logger.debug(
+                f"Processed {findings_processed} findings but no database changes were needed (all issues up-to-date)"
+            )
+
     def _finalize_scan_history(self, scan_history: regscale_models.ScanHistory) -> None:
         """
         Finalize scan history with error handling.
@@ -1568,17 +1820,18 @@ class ComplianceIntegration(ScannerIntegration, ABC):
 
         # Check for existing issue by external_id first
         external_id = finding.external_id
+        logger.debug(f"Looking for existing issue with external_id: '{external_id}'")
         existing_issue = self._find_existing_issue_cached(external_id)
 
         if existing_issue:
             logger.debug(
-                f"Found existing issue {existing_issue.id} for external_id {external_id}, updating instead of creating"
+                f"Found existing issue {existing_issue.id} (other_identifier: '{existing_issue.otherIdentifier}') for lookup external_id '{external_id}', updating instead of creating"
             )
 
             # Update existing issue with new finding data
             existing_issue.title = title
             existing_issue.description = finding.description
-            existing_issue.severity = finding.severity
+            existing_issue.severityLevel = finding.severity
             existing_issue.status = finding.status
             # Ensure affectedControls is updated from the finding's control id
             try: