regscale-cli 6.23.0.0__py3-none-any.whl → 6.24.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of regscale-cli might be problematic. Click here for more details.

Files changed (44) hide show
  1. regscale/_version.py +1 -1
  2. regscale/core/app/application.py +2 -0
  3. regscale/integrations/commercial/__init__.py +1 -0
  4. regscale/integrations/commercial/sarif/sarif_converter.py +1 -1
  5. regscale/integrations/commercial/wizv2/click.py +109 -2
  6. regscale/integrations/commercial/wizv2/compliance_report.py +1485 -0
  7. regscale/integrations/commercial/wizv2/constants.py +72 -2
  8. regscale/integrations/commercial/wizv2/data_fetcher.py +61 -0
  9. regscale/integrations/commercial/wizv2/file_cleanup.py +104 -0
  10. regscale/integrations/commercial/wizv2/issue.py +775 -27
  11. regscale/integrations/commercial/wizv2/policy_compliance.py +599 -181
  12. regscale/integrations/commercial/wizv2/reports.py +243 -0
  13. regscale/integrations/commercial/wizv2/scanner.py +668 -245
  14. regscale/integrations/compliance_integration.py +304 -51
  15. regscale/integrations/due_date_handler.py +210 -0
  16. regscale/integrations/public/cci_importer.py +444 -0
  17. regscale/integrations/scanner_integration.py +718 -153
  18. regscale/models/integration_models/CCI_List.xml +1 -0
  19. regscale/models/integration_models/cisa_kev_data.json +61 -3
  20. regscale/models/integration_models/synqly_models/capabilities.json +1 -1
  21. regscale/models/integration_models/synqly_models/connectors/vulnerabilities.py +3 -3
  22. regscale/models/regscale_models/form_field_value.py +1 -1
  23. regscale/models/regscale_models/milestone.py +1 -0
  24. regscale/models/regscale_models/regscale_model.py +225 -60
  25. regscale/models/regscale_models/security_plan.py +3 -2
  26. regscale/regscale.py +7 -0
  27. {regscale_cli-6.23.0.0.dist-info → regscale_cli-6.24.0.0.dist-info}/METADATA +9 -9
  28. {regscale_cli-6.23.0.0.dist-info → regscale_cli-6.24.0.0.dist-info}/RECORD +44 -27
  29. tests/fixtures/test_fixture.py +13 -8
  30. tests/regscale/integrations/public/__init__.py +0 -0
  31. tests/regscale/integrations/public/test_alienvault.py +220 -0
  32. tests/regscale/integrations/public/test_cci.py +458 -0
  33. tests/regscale/integrations/public/test_cisa.py +1021 -0
  34. tests/regscale/integrations/public/test_emass.py +518 -0
  35. tests/regscale/integrations/public/test_fedramp.py +851 -0
  36. tests/regscale/integrations/public/test_fedramp_cis_crm.py +3661 -0
  37. tests/regscale/integrations/public/test_file_uploads.py +506 -0
  38. tests/regscale/integrations/public/test_oscal.py +453 -0
  39. tests/regscale/models/test_form_field_value_integration.py +304 -0
  40. tests/regscale/models/test_module_integration.py +582 -0
  41. {regscale_cli-6.23.0.0.dist-info → regscale_cli-6.24.0.0.dist-info}/LICENSE +0 -0
  42. {regscale_cli-6.23.0.0.dist-info → regscale_cli-6.24.0.0.dist-info}/WHEEL +0 -0
  43. {regscale_cli-6.23.0.0.dist-info → regscale_cli-6.24.0.0.dist-info}/entry_points.txt +0 -0
  44. {regscale_cli-6.23.0.0.dist-info → regscale_cli-6.24.0.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,1485 @@
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ """Wiz Compliance Report Integration for RegScale CLI."""
4
+
5
+ import csv
6
+ import gzip
7
+ import logging
8
+ import os
9
+ import re
10
+ from datetime import datetime, timedelta
11
+ from typing import Dict, Any, Optional, List
12
+
13
+ from regscale.core.app.utils.app_utils import error_and_exit
14
+ from regscale.core.app.utils.app_utils import get_current_datetime
15
+ from regscale.integrations.commercial.wizv2.file_cleanup import ReportFileCleanup
16
+ from regscale.integrations.commercial.wizv2.reports import WizReportManager
17
+ from regscale.integrations.commercial.wizv2.variables import WizVariables
18
+ from regscale.integrations.commercial.wizv2.wiz_auth import wiz_authenticate
19
+ from regscale.integrations.compliance_integration import ComplianceIntegration, ComplianceItem
20
+ from regscale.models import regscale_models
21
+ from regscale.models.regscale_models.control_implementation import ControlImplementation, ControlImplementationStatus
22
+
23
+ logger = logging.getLogger("regscale")
24
+
25
+
26
+ class WizComplianceReportItem(ComplianceItem):
27
+ """Compliance item parsed from Wiz CSV report."""
28
+
29
+ def __init__(self, csv_row: Dict[str, str]):
30
+ """
31
+ Initialize from CSV row data.
32
+
33
+ :param Dict[str, str] csv_row: Row data from CSV report
34
+ """
35
+ self.csv_data = csv_row
36
+ self._resource_name = csv_row.get("Resource Name", "") # Use _resource_name to avoid conflict with property
37
+ self.cloud_provider = csv_row.get("Cloud Provider", "")
38
+ self.cloud_provider_id = csv_row.get("Cloud Provider ID", "")
39
+ self._resource_id = csv_row.get("Resource ID", "") # Use _resource_id to avoid conflict with property
40
+ self.resource_region = csv_row.get("Resource Region", "")
41
+ self.subscription = csv_row.get("Subscription", "")
42
+ self.subscription_name = csv_row.get("Subscription Name", "")
43
+ self.policy_name = csv_row.get("Policy Name", "")
44
+ self.policy_id = csv_row.get("Policy ID", "")
45
+ self.result = csv_row.get("Result", "")
46
+ self._severity = csv_row.get("Severity", "") # Use _severity to avoid conflict with property
47
+ self.compliance_check_name = csv_row.get("Compliance Check Name (Wiz Subcategory)", "")
48
+ self._framework = csv_row.get("Framework", "") # Use _framework to avoid conflict with property
49
+ self.remediation_steps = csv_row.get("Remediation Steps", "")
50
+
51
+ # ComplianceItem abstract property implementations
52
+ @property
53
+ def resource_id(self) -> str:
54
+ """Unique identifier for the resource being assessed."""
55
+ return self.cloud_provider_id or self._resource_id or self._resource_name or "Unknown"
56
+
57
+ @property
58
+ def resource_name(self) -> str:
59
+ """Human-readable name of the resource."""
60
+ return self.get_unique_resource_name()
61
+
62
+ @property
63
+ def control_id(self) -> str:
64
+ """Control identifier (e.g., AC-3, SI-2)."""
65
+ return self.get_control_id()
66
+
67
+ @property
68
+ def compliance_result(self) -> str:
69
+ """Result of compliance check (PASS, FAIL, etc)."""
70
+ return self.result
71
+
72
+ @property
73
+ def severity(self) -> Optional[str]:
74
+ """Severity level of the compliance violation (if failed)."""
75
+ return self._severity if self._severity else None
76
+
77
+ @property
78
+ def description(self) -> str:
79
+ """Description of the compliance check."""
80
+ return self.get_finding_details()
81
+
82
+ @property
83
+ def framework(self) -> str:
84
+ """Compliance framework (e.g., NIST800-53R5, CSF)."""
85
+ if not self._framework:
86
+ return "NIST800-53R5"
87
+
88
+ # Normalize Wiz framework names to RegScale format
89
+ framework_mappings = {
90
+ "NIST SP 800-53 Revision 5": "NIST800-53R5",
91
+ "NIST SP 800-53 Rev 5": "NIST800-53R5",
92
+ "NIST SP 800-53 R5": "NIST800-53R5",
93
+ "NIST 800-53 Revision 5": "NIST800-53R5",
94
+ "NIST 800-53 Rev 5": "NIST800-53R5",
95
+ "NIST 800-53 R5": "NIST800-53R5",
96
+ }
97
+
98
+ return framework_mappings.get(self._framework, self._framework)
99
+
100
+ def get_control_id(self) -> str:
101
+ """Extract first control ID from compliance check name for compatibility."""
102
+ control_ids = self.get_all_control_ids()
103
+ return control_ids[0] if control_ids else ""
104
+
105
+ def get_all_control_ids(self) -> list:
106
+ """Extract all control IDs from compliance check name."""
107
+ if not self.compliance_check_name:
108
+ return []
109
+
110
+ # Parse control IDs from compliance check name
111
+ # Format: "AC-2(4) Account Management | Automated Audit Actions, AC-6(9) Least Privilege | Log Use of Privileged Functions"
112
+ # Use a regex that can find control IDs anywhere in the text
113
+ control_id_pattern = r"([A-Za-z]{2}-\d+)(?:\s*\(\s*(\d+)\s*\))?"
114
+
115
+ control_ids = []
116
+ for part in self.compliance_check_name.split(", "):
117
+ matches = re.findall(control_id_pattern, part.strip())
118
+ for match in matches:
119
+ base_control, enhancement = match
120
+ if enhancement:
121
+ control_ids.append(f"{base_control}({enhancement})")
122
+ else:
123
+ control_ids.append(base_control)
124
+
125
+ return control_ids
126
+
127
+ @property
128
+ def affected_controls(self) -> str:
129
+ """Get affected controls as comma-separated string for issues."""
130
+ control_ids = self.get_all_control_ids()
131
+ return ",".join(control_ids) if control_ids else self.control_id
132
+
133
+ def get_status(self) -> str:
134
+ """Get compliance status based on result."""
135
+ return "Satisfied" if self.result.lower() == "pass" else "Other Than Satisfied"
136
+
137
+ def get_implementation_status(self) -> str:
138
+ """Get implementation status based on result."""
139
+ return "Implemented" if self.result.lower() == "pass" else "In Remediation"
140
+
141
+ def get_severity(self) -> str:
142
+ """Map Wiz severity to RegScale severity."""
143
+ severity_map = {"CRITICAL": "High", "HIGH": "High", "MEDIUM": "Moderate", "LOW": "Low", "INFORMATIONAL": "Low"}
144
+ return severity_map.get(self._severity.upper(), "Low")
145
+
146
+ def get_unique_resource_name(self) -> str:
147
+ """Get a unique resource name by appending provider ID or resource ID."""
148
+ base_name = self._resource_name
149
+ if not base_name:
150
+ base_name = "Unknown Resource"
151
+
152
+ # Add region if available
153
+ if self.resource_region:
154
+ base_name = f"{base_name} ({self.resource_region})"
155
+
156
+ # Add unique identifier (prefer resource_id over cloud_provider_id)
157
+ unique_id = self._resource_id or self.cloud_provider_id
158
+ if unique_id:
159
+ # Extract just the last part of Azure resource IDs for brevity
160
+ if "/" in unique_id:
161
+ unique_suffix = unique_id.split("/")[-1]
162
+ else:
163
+ unique_suffix = unique_id
164
+
165
+ # Only append if not already in the name
166
+ if unique_suffix.lower() not in base_name.lower():
167
+ base_name = f"{base_name} [{unique_suffix[:12]}]" # Limit to 12 chars
168
+
169
+ return base_name
170
+
171
+ def get_unique_issue_identifier(self) -> str:
172
+ """Get a unique identifier for deduplication of issues."""
173
+ # Use resource_id + policy_id + control_id for uniqueness
174
+ resource_key = self._resource_id or self.cloud_provider_id or self._resource_name
175
+ policy_key = self.policy_id or self.policy_name
176
+ control_key = self.get_control_id()
177
+ return f"{resource_key}|{policy_key}|{control_key}"
178
+
179
+ def get_title(self) -> str:
180
+ """Get assessment title."""
181
+ return f"{self.get_control_id()} - {self.policy_name}"
182
+
183
+ def get_description(self) -> str:
184
+ """Get assessment description."""
185
+ return f"Wiz compliance assessment for {self.get_unique_resource_name()} - {self.policy_name}"
186
+
187
+ def get_finding_details(self) -> str:
188
+ """Get finding details for issues."""
189
+ details = f"Resource: {self.get_unique_resource_name()}\n"
190
+ details += f"Cloud Provider: {self.cloud_provider}\n"
191
+ if self.subscription_name:
192
+ details += f"Subscription: {self.subscription_name}\n"
193
+ details += f"Result: {self.result}\n"
194
+ details += f"Remediation: {self.remediation_steps}"
195
+ return details
196
+
197
+ def get_asset_identifier(self) -> str:
198
+ """Get asset identifier using cloud provider ID for issues."""
199
+ return self.cloud_provider_id or self._resource_id or self._resource_name or "Unknown"
200
+
201
+
202
+ class WizComplianceReportProcessor(ComplianceIntegration):
203
+ """Process compliance reports from Wiz and create assessments in RegScale."""
204
+
205
+ # Set the asset identifier field to match Wiz integration standard
206
+ asset_identifier_field: str = "wizId"
207
+
208
+ def __init__(
209
+ self,
210
+ plan_id: int,
211
+ wiz_project_id: str,
212
+ client_id: str,
213
+ client_secret: str,
214
+ regscale_module: str = "securityplans",
215
+ create_poams: bool = False,
216
+ report_file_path: Optional[str] = None,
217
+ bypass_control_filtering: bool = False,
218
+ max_report_age_days: int = 7,
219
+ force_fresh_report: bool = False,
220
+ **kwargs,
221
+ ):
222
+ """
223
+ Initialize the compliance report processor.
224
+
225
+ :param int plan_id: RegScale plan/SSP ID
226
+ :param str wiz_project_id: Wiz project ID
227
+ :param str client_id: Wiz client ID
228
+ :param str client_secret: Wiz client secret
229
+ :param str regscale_module: RegScale module to use
230
+ :param bool create_poams: Whether to create POAMs for failed assessments
231
+ :param Optional[str] report_file_path: Path to existing report file to use instead of creating new one
232
+ :param bool bypass_control_filtering: Skip control filtering for performance with large control sets
233
+ :param int max_report_age_days: Maximum age in days for reusing existing reports (default: 7 days)
234
+ :param bool force_fresh_report: Force creation of fresh report, ignoring existing reports
235
+ """
236
+ # Call parent constructor with ComplianceIntegration parameters
237
+ super().__init__(
238
+ plan_id=plan_id,
239
+ framework="NIST800-53R5",
240
+ create_poams=create_poams,
241
+ parent_module=regscale_module,
242
+ **kwargs,
243
+ )
244
+
245
+ # Wiz-specific attributes
246
+ self.wiz_project_id = wiz_project_id
247
+ self.client_id = client_id
248
+ self.client_secret = client_secret
249
+ self.report_file_path = report_file_path
250
+ self.bypass_control_filtering = bypass_control_filtering
251
+ self.max_report_age_days = max_report_age_days
252
+ self.force_fresh_report = force_fresh_report
253
+ self.title = "Wiz Compliance" # Required by ScannerIntegration
254
+
255
+ # Initialize Wiz authentication
256
+ access_token = wiz_authenticate(client_id, client_secret)
257
+ if not access_token:
258
+ error_and_exit("Failed to authenticate with Wiz")
259
+
260
+ self.report_manager = WizReportManager(WizVariables.wizUrl, access_token)
261
+
262
+ def parse_csv_report(self, file_path: str) -> List[WizComplianceReportItem]:
263
+ """
264
+ Parse CSV compliance report.
265
+
266
+ :param str file_path: Path to CSV report file
267
+ :return: List of compliance items
268
+ :rtype: List[WizComplianceReportItem]
269
+ """
270
+ items = []
271
+
272
+ try:
273
+ # Handle gzipped files
274
+ if file_path.endswith(".gz"):
275
+ with gzip.open(file_path, "rt", encoding="utf-8") as f:
276
+ reader = csv.DictReader(f)
277
+ for row in reader:
278
+ items.append(WizComplianceReportItem(row))
279
+ else:
280
+ with open(file_path, "r", encoding="utf-8") as f:
281
+ reader = csv.DictReader(f)
282
+ for row in reader:
283
+ items.append(WizComplianceReportItem(row))
284
+
285
+ logger.info(f"Parsed {len(items)} compliance items from report")
286
+ return items
287
+
288
+ except Exception as e:
289
+ logger.error(f"Error parsing CSV report: {e}")
290
+ return []
291
+
292
+ # ComplianceIntegration abstract method implementations
293
+ def fetch_compliance_data(self) -> List[Dict[str, str]]:
294
+ """
295
+ Fetch raw compliance data from CSV report.
296
+
297
+ :return: List of raw compliance data (CSV rows as dictionaries)
298
+ :rtype: List[Dict[str, str]]
299
+ """
300
+ # Use provided report file or get/create one
301
+ if self.report_file_path and os.path.exists(self.report_file_path):
302
+ report_file_path = self.report_file_path
303
+ else:
304
+ report_file_path = self._get_or_create_report()
305
+ if not report_file_path or not os.path.exists(report_file_path):
306
+ logger.error("Failed to get compliance report")
307
+ return []
308
+
309
+ # Read CSV file and return raw data
310
+ raw_data = []
311
+ try:
312
+ with open(report_file_path, "r", encoding="utf-8") as file:
313
+ csv_reader = csv.DictReader(file)
314
+ raw_data = list(csv_reader)
315
+
316
+ logger.info(f"Fetched {len(raw_data)} raw compliance records from CSV")
317
+ return raw_data
318
+
319
+ except Exception as e:
320
+ logger.error(f"Error reading CSV report: {e}")
321
+ return []
322
+
323
+ def create_compliance_item(self, raw_data: Dict[str, str]) -> ComplianceItem:
324
+ """
325
+ Create a ComplianceItem from raw compliance data.
326
+
327
+ :param Dict[str, str] raw_data: Raw compliance data from CSV row
328
+ :return: ComplianceItem instance
329
+ :rtype: ComplianceItem
330
+ """
331
+ return WizComplianceReportItem(raw_data)
332
+
333
+ def _map_string_severity_to_enum(self, severity_str: str) -> regscale_models.IssueSeverity:
334
+ """
335
+ Convert string severity to regscale_models.IssueSeverity enum.
336
+
337
+ :param str severity_str: String severity like "HIGH", "MEDIUM", etc.
338
+ :return: IssueSeverity enum value
339
+ :rtype: regscale_models.IssueSeverity
340
+ """
341
+ severity_mapping = {
342
+ "CRITICAL": regscale_models.IssueSeverity.Critical,
343
+ "HIGH": regscale_models.IssueSeverity.High,
344
+ "MEDIUM": regscale_models.IssueSeverity.Moderate,
345
+ "MODERATE": regscale_models.IssueSeverity.Moderate,
346
+ "LOW": regscale_models.IssueSeverity.Low,
347
+ "INFORMATIONAL": regscale_models.IssueSeverity.Low,
348
+ }
349
+ return severity_mapping.get(severity_str.upper(), regscale_models.IssueSeverity.Low)
350
+
351
+ def process_compliance_data(self) -> None:
352
+ """
353
+ Override the parent method to implement bypass logic for large control sets.
354
+ """
355
+ if self.bypass_control_filtering:
356
+ logger.info("Bypassing control filtering due to bypass_control_filtering=True")
357
+ # Call parent method but bypass the allowed_controls_normalized logic
358
+ self._process_compliance_data_without_filtering()
359
+ else:
360
+ # Use standard parent implementation
361
+ super().process_compliance_data()
362
+
363
+ def _process_compliance_data_without_filtering(self) -> None:
364
+ """
365
+ Process compliance data without control filtering for performance.
366
+ """
367
+ logger.info("Processing compliance data without control filtering...")
368
+
369
+ self._reset_compliance_state()
370
+ raw_compliance_data = self.fetch_compliance_data()
371
+ self._process_raw_compliance_items(raw_compliance_data)
372
+ self._log_processing_debug_info()
373
+ self._categorize_controls_fail_first()
374
+ self._log_processing_summary()
375
+ self._log_categorization_debug_info()
376
+
377
+ def _reset_compliance_state(self) -> None:
378
+ """Reset state to avoid double counting on repeated calls."""
379
+ self.all_compliance_items = []
380
+ self.failed_compliance_items = []
381
+ self.passing_controls = {}
382
+ self.failing_controls = {}
383
+ self.asset_compliance_map.clear()
384
+
385
+ def _process_raw_compliance_items(self, raw_compliance_data: List[Any], allowed_controls: set = None) -> dict:
386
+ """Convert raw compliance data to ComplianceItem objects.
387
+
388
+ :param List[Any] raw_compliance_data: Raw compliance data from CSV row
389
+ :param set allowed_controls: Allowed control IDs (unused in this override, provided for interface compatibility)
390
+ :return: Processing statistics dictionary (empty dict for this implementation)
391
+ :rtype: dict
392
+ """
393
+ for raw_item in raw_compliance_data:
394
+ try:
395
+ compliance_item = self.create_compliance_item(raw_item)
396
+
397
+ if not self._is_valid_compliance_item_for_processing(compliance_item):
398
+ continue
399
+
400
+ self._add_compliance_item_to_collections(compliance_item)
401
+
402
+ except Exception as e:
403
+ logger.error(f"Error processing compliance item: {e}")
404
+ continue
405
+
406
+ # Return empty stats dict for interface compatibility
407
+ return {}
408
+
409
+ def _is_valid_compliance_item_for_processing(self, compliance_item: Any) -> bool:
410
+ """Check if compliance item has required control and resource IDs.
411
+
412
+ :param Any compliance_item: Compliance item to check
413
+ :return: True if compliance item has required control and resource IDs
414
+ :rtype: bool
415
+ """
416
+ control_id = getattr(compliance_item, "control_id", "")
417
+ resource_id = getattr(compliance_item, "resource_id", "")
418
+ return bool(control_id and resource_id)
419
+
420
+ def _add_compliance_item_to_collections(self, compliance_item: Any) -> None:
421
+ """Add compliance item to appropriate collections and categorize.
422
+
423
+ :param Any compliance_item: Compliance item to add to collections
424
+ :return: None
425
+ :rtype: None
426
+ """
427
+ self.all_compliance_items.append(compliance_item)
428
+ self.asset_compliance_map[compliance_item.resource_id].append(compliance_item)
429
+
430
+ # Categorize by result - normalize to handle case variations
431
+ result_lower = compliance_item.compliance_result.lower()
432
+ fail_statuses_lower = [status.lower() for status in self.FAIL_STATUSES]
433
+
434
+ if result_lower in fail_statuses_lower:
435
+ self.failed_compliance_items.append(compliance_item)
436
+
437
+ def _log_processing_debug_info(self) -> None:
438
+ """
439
+ Log debug information before categorization.
440
+
441
+ Logs sample compliance item data and status configurations
442
+ to help with debugging categorization issues.
443
+
444
+ :return: None
445
+ :rtype: None
446
+ """
447
+ logger.debug(f"About to categorize {len(self.all_compliance_items)} compliance items")
448
+ if self.all_compliance_items:
449
+ sample_item = self.all_compliance_items[0]
450
+ logger.debug(
451
+ f"DEBUG: Sample item control_id='{sample_item.control_id}', result='{sample_item.compliance_result}'"
452
+ )
453
+ logger.debug(f"FAIL_STATUSES = {self.FAIL_STATUSES}")
454
+ logger.debug(f"PASS_STATUSES = {self.PASS_STATUSES}")
455
+
456
+ def _log_processing_summary(self, raw_compliance_data: list = None, stats: dict = None) -> None:
457
+ """
458
+ Log summary of processed compliance items.
459
+
460
+ Provides a summary count of total items, passing items, failing items,
461
+ and control categorization results for monitoring processing progress.
462
+
463
+ :param list raw_compliance_data: Raw compliance data (unused in this implementation, for interface compatibility)
464
+ :param dict stats: Processing statistics (unused in this implementation, for interface compatibility)
465
+ :return: None
466
+ :rtype: None
467
+ """
468
+ passing_count = len(self.all_compliance_items) - len(self.failed_compliance_items)
469
+ failing_count = len(self.failed_compliance_items)
470
+
471
+ logger.info(
472
+ f"Processed {len(self.all_compliance_items)} compliance items: "
473
+ f"{passing_count} passing, {failing_count} failing"
474
+ )
475
+ logger.info(
476
+ f"Control categorization: {len(self.passing_controls)} passing controls, "
477
+ f"{len(self.failing_controls)} failing controls"
478
+ )
479
+
480
+ def _log_categorization_debug_info(self) -> None:
481
+ """
482
+ Log debug information about categorized controls.
483
+
484
+ Outputs lists of passing and failing control IDs for debugging
485
+ categorization logic and identifying potential issues.
486
+
487
+ :return: None
488
+ :rtype: None
489
+ """
490
+ if self.passing_controls:
491
+ logger.debug(f"Passing control IDs: {list(self.passing_controls.keys())}")
492
+ if self.failing_controls:
493
+ logger.debug(f"Failing control IDs: {list(self.failing_controls.keys())}")
494
+ if not self.passing_controls and not self.failing_controls:
495
+ logger.error(
496
+ "DEBUG: No controls were categorized! This indicates an issue in _categorize_controls_fail_first"
497
+ )
498
+
499
+ def _categorize_controls_fail_first(self) -> None:
500
+ """
501
+ Categorize controls using fail-first logic.
502
+
503
+ If ANY compliance item for a control is failing, the entire control is marked as failing.
504
+ A control is only marked as passing if ALL instances of that control are passing.
505
+ """
506
+ logger.info("Starting fail-first control categorization...")
507
+
508
+ control_results = self._determine_control_results()
509
+ self._populate_control_collections(control_results)
510
+ self._populate_failed_compliance_items()
511
+ self._log_categorization_completion()
512
+
513
+ def _determine_control_results(self) -> Dict[str, str]:
514
+ """
515
+ Determine pass/fail status for each control based on compliance items.
516
+
517
+ Analyzes all compliance items and applies fail-first logic to determine
518
+ the overall status for each control. A control is marked as "fail" if
519
+ ANY compliance item for that control is failing.
520
+
521
+ :return: Dictionary mapping control IDs (lowercase) to "pass" or "fail"
522
+ :rtype: Dict[str, str]
523
+ """
524
+ control_results = {} # {control_id: "pass" or "fail"}
525
+
526
+ for item in self.all_compliance_items:
527
+ control_ids = self._get_control_ids_for_item(item)
528
+
529
+ for control_id in control_ids:
530
+ if not control_id:
531
+ continue
532
+
533
+ control_id_lower = control_id.lower()
534
+
535
+ if self._is_compliance_item_failing(item):
536
+ control_results[control_id_lower] = "fail"
537
+ logger.debug(f"Control {control_id} marked as FAILING due to failed item")
538
+ elif control_id_lower not in control_results:
539
+ control_results[control_id_lower] = "pass"
540
+
541
+ return control_results
542
+
543
+ def _get_control_ids_for_item(self, item: Any) -> List[str]:
544
+ """
545
+ Get all control IDs for a compliance item.
546
+
547
+ Extracts control IDs from compliance items that may reference
548
+ multiple controls (e.g., multi-control compliance checks).
549
+
550
+ :param Any item: Compliance item to extract control IDs from
551
+ :return: List of control ID strings
552
+ :rtype: List[str]
553
+ """
554
+ if hasattr(item, "get_all_control_ids"):
555
+ return item.get_all_control_ids()
556
+ else:
557
+ return [item.control_id] if item.control_id else []
558
+
559
+ def _is_compliance_item_failing(self, item: Any) -> bool:
560
+ """
561
+ Check if a compliance item is failing.
562
+
563
+ Compares the compliance result against the list of failure statuses
564
+ using case-insensitive matching.
565
+
566
+ :param Any item: Compliance item to check
567
+ :return: True if the item is failing, False otherwise
568
+ :rtype: bool
569
+ """
570
+ result_lower = item.compliance_result.lower()
571
+ fail_statuses_lower = [status.lower() for status in self.FAIL_STATUSES]
572
+ return result_lower in fail_statuses_lower
573
+
574
+ def _populate_control_collections(self, control_results: Dict[str, str]) -> None:
575
+ """
576
+ Populate passing and failing control collections.
577
+
578
+ Based on the control results dictionary, populates the passing_controls
579
+ and failing_controls collections with the appropriate compliance items.
580
+
581
+ :param Dict[str, str] control_results: Dictionary mapping control IDs to "pass" or "fail"
582
+ :return: None
583
+ :rtype: None
584
+ """
585
+ for control_id_lower, result in control_results.items():
586
+ if result == "fail":
587
+ self.failing_controls[control_id_lower] = self._get_items_for_control(control_id_lower)
588
+ else:
589
+ self.passing_controls[control_id_lower] = self._get_items_for_control(control_id_lower)
590
+
591
+ def _get_items_for_control(self, control_id_lower: str) -> List[Any]:
592
+ """
593
+ Get all compliance items that belong to a specific control.
594
+
595
+ Searches through all compliance items to find those that reference
596
+ the specified control ID (case-insensitive matching).
597
+
598
+ :param str control_id_lower: Control ID in lowercase format
599
+ :return: List of compliance items for the control
600
+ :rtype: List[Any]
601
+ """
602
+ items = []
603
+ for item in self.all_compliance_items:
604
+ item_control_ids = self._get_normalized_control_ids_for_item(item)
605
+ if control_id_lower in item_control_ids:
606
+ items.append(item)
607
+ return items
608
+
609
+ def _get_normalized_control_ids_for_item(self, item: Any) -> List[str]:
610
+ """
611
+ Get normalized (lowercase) control IDs for an item.
612
+
613
+ Extracts all control IDs from a compliance item and normalizes
614
+ them to lowercase for consistent comparison and matching.
615
+
616
+ :param Any item: Compliance item to extract control IDs from
617
+ :return: List of normalized control ID strings
618
+ :rtype: List[str]
619
+ """
620
+ if hasattr(item, "get_all_control_ids"):
621
+ return [cid.lower() for cid in item.get_all_control_ids()]
622
+ else:
623
+ return [item.control_id.lower()] if item.control_id else []
624
+
625
+ def _populate_failed_compliance_items(self) -> None:
626
+ """
627
+ Populate and deduplicate the failed compliance items list.
628
+
629
+ Collects all failing compliance items from the failing_controls
630
+ collection and removes duplicates to create a clean list of
631
+ failed items for issue processing.
632
+
633
+ :return: None
634
+ :rtype: None
635
+ """
636
+ self.failed_compliance_items.clear()
637
+
638
+ for control_id, failing_items in self.failing_controls.items():
639
+ self.failed_compliance_items.extend(failing_items)
640
+
641
+ self.failed_compliance_items = self._remove_duplicate_items(self.failed_compliance_items)
642
+
643
+ def _remove_duplicate_items(self, items: List[Any]) -> List[Any]:
644
+ """
645
+ Remove duplicate compliance items while preserving order.
646
+
647
+ Uses resource_id and control_id to create unique keys for
648
+ deduplication while maintaining the original order of items.
649
+
650
+ :param List[Any] items: List of compliance items to deduplicate
651
+ :return: List of unique compliance items
652
+ :rtype: List[Any]
653
+ """
654
+ seen = set()
655
+ unique_items = []
656
+
657
+ for item in items:
658
+ item_key = f"{getattr(item, 'resource_id', '')}-{getattr(item, 'control_id', '')}"
659
+ if item_key not in seen:
660
+ seen.add(item_key)
661
+ unique_items.append(item)
662
+
663
+ return unique_items
664
+
665
+ def _log_categorization_completion(self) -> None:
666
+ """
667
+ Log completion of control categorization.
668
+
669
+ Provides final summary statistics about the categorization process,
670
+ including counts of passing/failing controls and failed items.
671
+
672
+ :return: None
673
+ :rtype: None
674
+ """
675
+ logger.info(
676
+ f"Fail-first categorization complete: {len(self.passing_controls)} passing, "
677
+ f"{len(self.failing_controls)} failing controls"
678
+ )
679
+ logger.info(f"Populated failed_compliance_items list with {len(self.failed_compliance_items)} items")
680
+
681
+ def process_compliance_sync(self) -> None:
682
+ """
683
+ New main method using ComplianceIntegration pattern.
684
+
685
+ This replaces the old process_compliance_report method.
686
+ """
687
+ logger.info("Starting Wiz compliance sync using ComplianceIntegration pattern...")
688
+ self.sync_compliance()
689
+
690
+ def _get_or_create_report(self, max_age_hours: int = None) -> Optional[str]:
691
+ """
692
+ Get existing recent report or create a new one if needed.
693
+
694
+ :param int max_age_hours: Maximum age in hours for reusing existing reports (deprecated, use max_report_age_days)
695
+ :return: Path to report file
696
+ :rtype: Optional[str]
697
+ """
698
+ # Handle force fresh report request
699
+ if self.force_fresh_report:
700
+ logger.info("Force fresh report requested, creating new compliance report...")
701
+ return self._create_and_download_report()
702
+
703
+ # Use instance variable max_report_age_days or legacy max_age_hours
704
+ if max_age_hours is not None:
705
+ # Legacy behavior for backward compatibility
706
+ max_age_hours_to_use = max_age_hours
707
+ logger.warning("Using deprecated max_age_hours parameter. Consider using max_report_age_days instead.")
708
+ else:
709
+ # Convert days to hours for the internal method
710
+ max_age_hours_to_use = self.max_report_age_days * 24
711
+
712
+ # Check for existing recent reports
713
+ existing_report = self._find_recent_report(max_age_hours_to_use)
714
+ if existing_report:
715
+ logger.info(f"Using existing report: {existing_report}")
716
+ return existing_report
717
+
718
+ # No recent report found, create a new one
719
+ logger.info(f"No recent report found within {self.max_report_age_days} days, creating new compliance report...")
720
+ return self._create_and_download_report()
721
+
722
+ def _find_recent_report(self, max_age_hours: int = 24) -> Optional[str]:
723
+ """
724
+ Find the most recent compliance report within the specified age limit.
725
+
726
+ :param int max_age_hours: Maximum age in hours
727
+ :return: Path to recent report file or None
728
+ :rtype: Optional[str]
729
+ """
730
+ artifacts_dir = "artifacts/wiz"
731
+ if not os.path.exists(artifacts_dir):
732
+ return None
733
+
734
+ report_prefix = f"compliance_report_{self.wiz_project_id}_"
735
+ cutoff_time = datetime.now() - timedelta(hours=max_age_hours)
736
+
737
+ # Find matching files
738
+ matching_files = []
739
+ for filename in os.listdir(artifacts_dir):
740
+ if filename.startswith(report_prefix) and filename.endswith(".csv"):
741
+ file_path = os.path.join(artifacts_dir, filename)
742
+ try:
743
+ # Get file modification time
744
+ mod_time = datetime.fromtimestamp(os.path.getmtime(file_path))
745
+ if mod_time > cutoff_time:
746
+ matching_files.append((file_path, mod_time))
747
+ except (OSError, ValueError):
748
+ continue
749
+
750
+ if not matching_files:
751
+ return None
752
+
753
+ # Return the most recent file
754
+ most_recent = max(matching_files, key=lambda x: x[1])
755
+ age_hours = (datetime.now() - most_recent[1]).total_seconds() / 3600
756
+ logger.info(f"Found recent report (age: {age_hours:.1f}h): {most_recent[0]}")
757
+ return most_recent[0]
758
+
759
+ def _create_and_download_report(self) -> Optional[str]:
760
+ """
761
+ Create and download a new compliance report.
762
+
763
+ :return: Path to downloaded report file
764
+ :rtype: Optional[str]
765
+ """
766
+ logger.info(f"Creating compliance report for project: {self.wiz_project_id}")
767
+
768
+ # Create report
769
+ report_id = self.report_manager.create_compliance_report(self.wiz_project_id)
770
+ if not report_id:
771
+ logger.error("Failed to create compliance report")
772
+ return None
773
+
774
+ # Wait for completion and get download URL
775
+ download_url = self.report_manager.wait_for_report_completion(report_id)
776
+ if not download_url:
777
+ logger.error("Failed to get download URL for report")
778
+ return None
779
+
780
+ # Download report
781
+ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
782
+ output_path = f"artifacts/wiz/compliance_report_{self.wiz_project_id}_{timestamp}.csv"
783
+
784
+ # Ensure directory exists
785
+ artifacts_dir = os.path.dirname(output_path)
786
+ os.makedirs(artifacts_dir, exist_ok=True)
787
+
788
+ if self.report_manager.download_report(download_url, output_path):
789
+ # Clean up old report files
790
+ ReportFileCleanup.cleanup_old_files(
791
+ directory=artifacts_dir, file_prefix="compliance_report_", extensions=[".csv"], keep_count=5
792
+ )
793
+ return output_path
794
+ else:
795
+ logger.error("Failed to download report")
796
+ return None
797
+
798
+ def _update_passing_controls_to_implemented(self, passing_control_ids: list[str]) -> None:
799
+ """
800
+ Update passing controls to 'Implemented' status in RegScale.
801
+
802
+ :param list[str] passing_control_ids: List of control IDs that passed
803
+ """
804
+ if not passing_control_ids:
805
+ return
806
+
807
+ # Initialize Application for control implementation updates
808
+ # app = Application() # Will be used through self.app
809
+
810
+ try:
811
+ # Use the existing method that works for getting control name to implementation ID mapping
812
+ control_impl_map = ControlImplementation.get_control_label_map_by_parent(
813
+ parent_id=self.plan_id, parent_module=self.parent_module
814
+ )
815
+
816
+ logger.debug(
817
+ f"Built control implementation map with {len(control_impl_map)} entries using get_control_label_map_by_parent"
818
+ )
819
+ if control_impl_map:
820
+ sample_keys = list(control_impl_map.keys())[:10]
821
+ logger.debug(f"Sample control names in map: {sample_keys}")
822
+
823
+ logger.debug(f"Looking for passing control IDs: {passing_control_ids}")
824
+
825
+ # Prepare batch updates for passing controls
826
+ implementations_to_update = []
827
+
828
+ for control_id in passing_control_ids:
829
+ control_id_lower = control_id.lower()
830
+ logger.debug(f"Looking for control '{control_id_lower}' in implementation map")
831
+
832
+ if control_id_lower in control_impl_map:
833
+ impl_id = control_impl_map[control_id_lower]
834
+ logger.debug(f"Found matching implementation for '{control_id_lower}': {impl_id}")
835
+
836
+ # Get the ControlImplementation object
837
+ impl = ControlImplementation.get_object(object_id=impl_id)
838
+ if impl:
839
+ # Update status to Implemented
840
+ impl.status = ControlImplementationStatus.Implemented.value
841
+ impl.dateLastAssessed = get_current_datetime()
842
+ impl.lastAssessmentResult = "Pass"
843
+ impl.bStatusImplemented = True
844
+
845
+ # Set audit fields if available
846
+ user_id = self.app.config.get("userId")
847
+ if user_id:
848
+ impl.lastUpdatedById = user_id
849
+ impl.dateLastUpdated = get_current_datetime()
850
+
851
+ implementations_to_update.append(impl.dict())
852
+ logger.info(f"Marking control {control_id} as Implemented")
853
+
854
+ # Batch update all implementations
855
+ if implementations_to_update:
856
+ ControlImplementation.put_batch_implementation(self.app, implementations_to_update)
857
+ logger.info(f"Successfully updated {len(implementations_to_update)} controls to Implemented status")
858
+ else:
859
+ logger.warning("No matching control implementations found to update")
860
+
861
+ except Exception as e:
862
+ logger.error(f"Error updating control implementation status: {e}")
863
+
864
+ def _update_failing_controls_to_in_remediation(self, control_ids: List[str]) -> None:
865
+ """
866
+ Update control implementation status to In Remediation for failing controls.
867
+
868
+ :param List[str] control_ids: List of control IDs that are failing
869
+ :return: None
870
+ :rtype: None
871
+ """
872
+ if not control_ids:
873
+ return
874
+
875
+ try:
876
+ control_impl_map = self._get_control_implementation_map()
877
+ if not control_impl_map:
878
+ return
879
+
880
+ implementations_to_update, controls_not_found = self._process_failing_control_ids(
881
+ control_ids, control_impl_map
882
+ )
883
+
884
+ self._log_update_summary(implementations_to_update, controls_not_found)
885
+ self._batch_update_implementations(implementations_to_update)
886
+
887
+ except Exception as e:
888
+ logger.error(f"Error updating failing control implementation status: {e}")
889
+
890
+ def _get_control_implementation_map(self) -> dict:
891
+ """Get control implementation map and validate it exists."""
892
+
893
+ control_impl_map = ControlImplementation.get_control_label_map_by_parent(
894
+ parent_id=self.plan_id, parent_module=self.parent_module
895
+ )
896
+
897
+ if not control_impl_map:
898
+ logger.warning("No control implementation mapping found for security plan")
899
+ return {}
900
+
901
+ logger.debug(f"Control implementation map contains {len(control_impl_map)} entries")
902
+ return control_impl_map
903
+
904
+ def _process_failing_control_ids(self, control_ids: List[str], control_impl_map: dict) -> tuple[list, list]:
905
+ """Process failing control IDs and return implementations to update and controls not found."""
906
+
907
+ logger.debug(f"Looking for failing control IDs: {control_ids}")
908
+ implementations_to_update = []
909
+ controls_not_found = []
910
+
911
+ for control_id in control_ids:
912
+ control_id_normalized = control_id.lower()
913
+ logger.debug(f"Looking for control '{control_id_normalized}' in implementation map")
914
+
915
+ if control_id_normalized in control_impl_map:
916
+ impl = self._update_single_control_implementation(
917
+ control_id, control_id_normalized, control_impl_map[control_id_normalized]
918
+ )
919
+ if impl:
920
+ implementations_to_update.append(impl)
921
+ else:
922
+ controls_not_found.append(control_id)
923
+ else:
924
+ logger.debug(f"Control '{control_id_normalized}' not found in implementation map")
925
+ controls_not_found.append(control_id)
926
+
927
+ return implementations_to_update, controls_not_found
928
+
929
+ def _update_single_control_implementation(
930
+ self, control_id: str, control_id_normalized: str, impl_id: int
931
+ ) -> Optional[dict]:
932
+ """Update a single control implementation to In Remediation status.
933
+ :param str control_id: ID of the control to update
934
+ :param str control_id_normalized: ID of the control to update
935
+ :param int impl_id: ID of the implementation to update
936
+ :return: Updated implementation status if implementation exists
937
+ :rtype: Optional[dict]
938
+ """
939
+ from regscale.core.app.utils.app_utils import get_current_datetime
940
+ from regscale.models.regscale_models import ControlImplementationStatus
941
+
942
+ logger.debug(f"Found matching implementation for '{control_id_normalized}': {impl_id}")
943
+
944
+ impl = ControlImplementation.get_object(object_id=impl_id)
945
+ if not impl:
946
+ logger.warning(f"Could not retrieve implementation object for ID {impl_id}")
947
+ return None
948
+
949
+ # Update status to In Remediation
950
+ impl.status = ControlImplementationStatus.InRemediation.value
951
+ impl.dateLastAssessed = get_current_datetime()
952
+ impl.lastAssessmentResult = "Fail"
953
+ impl.bStatusImplemented = False
954
+
955
+ # Set audit fields if available
956
+ user_id = self.app.config.get("userId")
957
+ if user_id:
958
+ impl.lastUpdatedById = user_id
959
+ impl.dateLastUpdated = get_current_datetime()
960
+
961
+ logger.info(f"Marking control {control_id} as In Remediation")
962
+ return impl.dict()
963
+
964
+ def _log_update_summary(self, implementations_to_update: list, controls_not_found: list) -> None:
965
+ """Log summary of control implementation updates."""
966
+ if controls_not_found:
967
+ skipped_list = ", ".join(controls_not_found[:5])
968
+ more_indicator = "..." if len(controls_not_found) > 5 else ""
969
+ logger.info(
970
+ f"Control implementation status update summary: {len(implementations_to_update)} found, "
971
+ f"{len(controls_not_found)} not in plan (skipped: {skipped_list}{more_indicator})"
972
+ )
973
+
974
+ def _batch_update_implementations(self, implementations_to_update: list) -> None:
975
+ """Perform batch update of control implementations."""
976
+ if implementations_to_update:
977
+ ControlImplementation.put_batch_implementation(self.app, implementations_to_update)
978
+ logger.debug(f"Updated {len(implementations_to_update)} Control Implementations, Successfully!")
979
+ logger.info(f"Successfully updated {len(implementations_to_update)} controls to In Remediation status")
980
+ else:
981
+ logger.warning("No matching control implementations found to update for failing controls")
982
+
983
+ def _process_control_assessments(self) -> None:
984
+ """
985
+ Override parent method to add control implementation status updates.
986
+ """
987
+ # Call parent method to create assessments
988
+ super()._process_control_assessments()
989
+
990
+ # Update control implementation status for both passing and failing controls if enabled
991
+ if self.update_control_status:
992
+ if self.passing_controls:
993
+ passing_control_ids = list(self.passing_controls.keys())
994
+ logger.info(f"Updating control implementation status for {len(passing_control_ids)} passing controls")
995
+ self._update_passing_controls_to_implemented(passing_control_ids)
996
+
997
+ if self.failing_controls:
998
+ failing_control_ids = list(self.failing_controls.keys())
999
+ logger.info(
1000
+ f"Attempting to update control implementation status for {len(failing_control_ids)} failing controls"
1001
+ )
1002
+ self._update_failing_controls_to_in_remediation(failing_control_ids)
1003
+
1004
+ def _categorize_controls_by_aggregation(self) -> None:
1005
+ """
1006
+ Override parent method to implement "fail-first" logic for Wiz compliance.
1007
+
1008
+ In the Wiz compliance integration, we implement strict "fail-first" logic:
1009
+ - If ANY compliance item for a control is failing, the entire control is marked as failing
1010
+ - A control is only marked as passing if ALL instances of that control are passing
1011
+ - This applies to both single-control and multi-control compliance items
1012
+ """
1013
+ control_items = self._group_compliance_items_by_control()
1014
+ self._apply_fail_first_logic_to_controls(control_items)
1015
+ self._populate_failed_compliance_items_from_control_items(control_items)
1016
+ self._log_categorization_results()
1017
+
1018
+ def _group_compliance_items_by_control(self) -> dict:
1019
+ """
1020
+ Group compliance items by control ID.
1021
+
1022
+ Creates a dictionary mapping control IDs (lowercase) to lists of
1023
+ compliance items that reference those controls. Handles multi-control
1024
+ items that may reference multiple control IDs.
1025
+
1026
+ :return: Dictionary mapping control IDs to lists of compliance items
1027
+ :rtype: dict
1028
+ """
1029
+ from collections import defaultdict
1030
+
1031
+ control_items = defaultdict(list)
1032
+
1033
+ for item in self.all_compliance_items:
1034
+ control_ids = self._extract_control_ids_from_item(item)
1035
+ self._add_item_to_control_groups(item, control_ids, control_items)
1036
+
1037
+ logger.debug(
1038
+ f"Grouped {len(self.all_compliance_items)} compliance items into {len(control_items)} control groups"
1039
+ )
1040
+ return control_items
1041
+
1042
+ def _extract_control_ids_from_item(self, item) -> list:
1043
+ """
1044
+ Extract all control IDs that an item affects.
1045
+
1046
+ Checks if the item has a get_all_control_ids method for multi-control
1047
+ items, otherwise falls back to the single control_id attribute.
1048
+
1049
+ :param item: Compliance item to extract control IDs from
1050
+ :type item: Any
1051
+ :return: List of control ID strings
1052
+ :rtype: list
1053
+ """
1054
+ if hasattr(item, "get_all_control_ids") and callable(item.get_all_control_ids):
1055
+ return item.get_all_control_ids()
1056
+ return [item.control_id] if item.control_id else []
1057
+
1058
+ def _add_item_to_control_groups(self, item, control_ids: list, control_items: dict) -> None:
1059
+ """
1060
+ Add item to all control groups it affects.
1061
+
1062
+ Adds the compliance item to the appropriate control groups based on
1063
+ all the control IDs it references. Uses lowercase control IDs as keys.
1064
+
1065
+ :param item: Compliance item to add to groups
1066
+ :type item: Any
1067
+ :param list control_ids: List of control IDs the item affects
1068
+ :param dict control_items: Dictionary of control groups to update
1069
+ :return: None
1070
+ :rtype: None
1071
+ """
1072
+ for control_id in control_ids:
1073
+ if control_id:
1074
+ control_key = control_id.lower()
1075
+ control_items[control_key].append(item)
1076
+
1077
+ def _apply_fail_first_logic_to_controls(self, control_items: dict) -> None:
1078
+ """
1079
+ Apply fail-first logic to categorize each control as passing or failing.
1080
+
1081
+ For each control, determines its overall status based on all associated
1082
+ compliance items. Any failure in the items makes the control fail.
1083
+
1084
+ :param dict control_items: Dictionary mapping control IDs to compliance items
1085
+ :return: None
1086
+ :rtype: None
1087
+ """
1088
+ for control_key, items in control_items.items():
1089
+ control_status = self._determine_control_status(items)
1090
+ self._categorize_control(control_key, control_status, len(items))
1091
+
1092
+ def _determine_control_status(self, items: list) -> dict:
1093
+ """
1094
+ Determine the overall status of a control based on its items.
1095
+
1096
+ Analyzes all compliance items for a control to determine if any are
1097
+ failing or passing. Returns status indicators and representative items.
1098
+
1099
+ :param list items: List of compliance items for the control
1100
+ :return: Dictionary with status flags and representative items
1101
+ :rtype: dict
1102
+ """
1103
+ fail_statuses_lower = [status.lower() for status in self.FAIL_STATUSES]
1104
+ pass_statuses_lower = [status.lower() for status in self.PASS_STATUSES]
1105
+
1106
+ status = {"has_failure": False, "has_pass": False, "failing_item": None, "passing_item": None}
1107
+
1108
+ for item in items:
1109
+ result_lower = item.compliance_result.lower()
1110
+
1111
+ if result_lower in fail_statuses_lower:
1112
+ status["has_failure"] = True
1113
+ if not status["failing_item"]:
1114
+ status["failing_item"] = item
1115
+ elif result_lower in pass_statuses_lower:
1116
+ status["has_pass"] = True
1117
+ if not status["passing_item"]:
1118
+ status["passing_item"] = item
1119
+
1120
+ return status
1121
+
1122
+ def _categorize_control(self, control_key: str, status: dict, item_count: int) -> None:
1123
+ """
1124
+ Categorize a control as passing or failing based on its status.
1125
+
1126
+ Uses the status information to place the control in the appropriate
1127
+ passing or failing collection and logs the categorization decision.
1128
+
1129
+ :param str control_key: Control ID (lowercase)
1130
+ :param dict status: Status information from _determine_control_status
1131
+ :param int item_count: Number of items analyzed for the control
1132
+ :return: None
1133
+ :rtype: None
1134
+ """
1135
+ if status["has_failure"]:
1136
+ self.failing_controls[control_key] = status["failing_item"]
1137
+ logger.debug(f"Control {control_key} marked as FAILING: fail-first logic triggered")
1138
+ elif status["has_pass"]:
1139
+ self.passing_controls[control_key] = status["passing_item"]
1140
+ logger.debug(f"Control {control_key} marked as PASSING: all {item_count} items passed")
1141
+ else:
1142
+ logger.debug(f"Control {control_key} has unclear results - no pass or fail statuses found")
1143
+
1144
+ def _populate_failed_compliance_items_from_control_items(self, control_items: dict) -> None:
1145
+ """
1146
+ Populate the list of failed compliance items from failing controls.
1147
+
1148
+ Collects all failing compliance items from controls marked as failing,
1149
+ removes duplicates, and updates the failed_compliance_items list.
1150
+
1151
+ :param dict control_items: Dictionary mapping control IDs to compliance items
1152
+ :return: None
1153
+ :rtype: None
1154
+ """
1155
+ self.failed_compliance_items.clear()
1156
+ failing_items = self._collect_failing_items_from_controls(control_items)
1157
+ self.failed_compliance_items = self._remove_duplicate_items(failing_items)
1158
+ logger.info(f"Populated failed_compliance_items list with {len(self.failed_compliance_items)} items")
1159
+
1160
+ def _collect_failing_items_from_controls(self, control_items: dict) -> list:
1161
+ """
1162
+ Collect all failing items from controls marked as failing.
1163
+
1164
+ Iterates through controls marked as failing and collects all their
1165
+ compliance items that have failing status results.
1166
+
1167
+ :param dict control_items: Dictionary mapping control IDs to compliance items
1168
+ :return: List of failing compliance items
1169
+ :rtype: list
1170
+ """
1171
+ failing_items = []
1172
+ fail_statuses_lower = [status.lower() for status in self.FAIL_STATUSES]
1173
+
1174
+ for control_key, items in control_items.items():
1175
+ if control_key in self.failing_controls:
1176
+ for item in items:
1177
+ if item.compliance_result.lower() in fail_statuses_lower:
1178
+ failing_items.append(item)
1179
+
1180
+ return failing_items
1181
+
1182
+ def _remove_duplicate_items(self, items: list) -> list:
1183
+ """
1184
+ Remove duplicate items while preserving order.
1185
+
1186
+ Uses resource_id and control_id combinations to create unique keys
1187
+ for deduplication while maintaining the original item order.
1188
+
1189
+ :param list items: List of compliance items to deduplicate
1190
+ :return: List of unique compliance items
1191
+ :rtype: list
1192
+ """
1193
+ seen = set()
1194
+ unique_items = []
1195
+
1196
+ for item in items:
1197
+ item_key = f"{getattr(item, 'resource_id', '')}-{getattr(item, 'control_id', '')}"
1198
+ if item_key not in seen:
1199
+ seen.add(item_key)
1200
+ unique_items.append(item)
1201
+
1202
+ return unique_items
1203
+
1204
+ def _log_categorization_results(self) -> None:
1205
+ """
1206
+ Log the final results of control categorization.
1207
+
1208
+ Provides summary statistics about the fail-first categorization
1209
+ process, including counts of passing and failing controls.
1210
+
1211
+ :return: None
1212
+ :rtype: None
1213
+ """
1214
+ logger.info(
1215
+ f"Control categorization with fail-first logic: "
1216
+ f"{len(self.passing_controls)} passing controls, "
1217
+ f"{len(self.failing_controls)} failing controls"
1218
+ )
1219
+
1220
+ def fetch_findings(self, *args, **kwargs):
1221
+ """
1222
+ Override to create one finding per control rather than per compliance item.
1223
+
1224
+ This ensures that each failing control gets exactly one issue in RegScale,
1225
+ consolidating all failed compliance items for that control.
1226
+ """
1227
+ logger.info("Fetching findings from failed controls (one per control)...")
1228
+
1229
+ processed_controls = set()
1230
+ findings_created = 0
1231
+
1232
+ for compliance_item in self.failed_compliance_items:
1233
+ control_ids = self._get_control_ids_for_item(compliance_item)
1234
+
1235
+ for control_id in control_ids:
1236
+ if not control_id or self._is_control_already_processed(control_id, processed_controls):
1237
+ continue
1238
+
1239
+ control_id_normalized = control_id.upper()
1240
+ processed_controls.add(control_id.lower())
1241
+
1242
+ control_failed_items = self._get_failed_items_for_control(control_id_normalized)
1243
+ finding = self._create_consolidated_finding_for_control(
1244
+ control_id=control_id_normalized, failed_items=control_failed_items
1245
+ )
1246
+
1247
+ if finding:
1248
+ findings_created += 1
1249
+ yield finding
1250
+
1251
+ self._log_findings_generation_summary(findings_created, len(processed_controls))
1252
+
1253
+ def _is_control_already_processed(self, control_id: str, processed_controls: set) -> bool:
1254
+ """
1255
+ Check if control has already been processed to avoid duplicates.
1256
+
1257
+ Uses case-insensitive comparison to determine if a control has
1258
+ already been processed for finding generation.
1259
+
1260
+ :param str control_id: Control ID to check
1261
+ :param set processed_controls: Set of already processed control IDs
1262
+ :return: True if control has been processed, False otherwise
1263
+ :rtype: bool
1264
+ """
1265
+ return control_id.lower() in processed_controls
1266
+
1267
+ def _get_failed_items_for_control(self, control_id_normalized: str) -> List[Any]:
1268
+ """
1269
+ Get all failed compliance items for a specific control.
1270
+
1271
+ Searches through the failed compliance items to find all items
1272
+ that reference the specified control ID (case-insensitive).
1273
+
1274
+ :param str control_id_normalized: Control ID in normalized format
1275
+ :return: List of failed compliance items for the control
1276
+ :rtype: List[Any]
1277
+ """
1278
+ control_failed_items = []
1279
+
1280
+ for item in self.failed_compliance_items:
1281
+ item_control_ids = self._get_control_ids_for_item(item)
1282
+
1283
+ if any(cid.upper() == control_id_normalized for cid in item_control_ids):
1284
+ control_failed_items.append(item)
1285
+
1286
+ return control_failed_items
1287
+
1288
+ def _log_findings_generation_summary(self, findings_created: int, controls_processed: int) -> None:
1289
+ """
1290
+ Log summary of findings generation.
1291
+
1292
+ Provides statistics about the finding generation process,
1293
+ including number of findings created and controls processed.
1294
+
1295
+ :param int findings_created: Number of findings successfully created
1296
+ :param int controls_processed: Number of controls processed
1297
+ :return: None
1298
+ :rtype: None
1299
+ """
1300
+ logger.info(
1301
+ f"Generated {findings_created} findings from {controls_processed} failing controls for issue processing"
1302
+ )
1303
+
1304
+ def _create_consolidated_finding_for_control(self, control_id: str, failed_items: list) -> Optional[Any]:
1305
+ """
1306
+ Create a single consolidated finding for a control with all its failed compliance items.
1307
+
1308
+ :param str control_id: The control identifier
1309
+ :param list failed_items: List of failed compliance items for this control
1310
+ :return: IntegrationFinding or None
1311
+ """
1312
+ try:
1313
+ from regscale.integrations.scanner_integration import IntegrationFinding
1314
+
1315
+ if not failed_items:
1316
+ return None
1317
+
1318
+ representative_item = failed_items[0]
1319
+ resource_info = self._collect_resource_information(failed_items)
1320
+ severity = self._determine_highest_severity(resource_info["severities"])
1321
+ description = self._build_consolidated_description(control_id, resource_info)
1322
+
1323
+ severity_enum = self._map_string_severity_to_enum(severity)
1324
+
1325
+ return self._create_integration_finding(
1326
+ control_id=control_id,
1327
+ severity_enum=severity_enum,
1328
+ description=description,
1329
+ representative_item=representative_item,
1330
+ )
1331
+
1332
+ except Exception as e:
1333
+ logger.error(f"Error creating consolidated finding for control {control_id}: {e}")
1334
+ return None
1335
+
1336
+ def _map_severity_to_priority(self, severity: Any) -> str:
1337
+ """
1338
+ Map severity enum to priority string.
1339
+
1340
+ Converts RegScale severity enumeration values to corresponding
1341
+ priority strings used in issue creation.
1342
+
1343
+ :param Any severity: Severity enum value
1344
+ :return: Priority string (High, Moderate, Low)
1345
+ :rtype: str
1346
+ """
1347
+ # Map severity to priority
1348
+ if hasattr(severity, "value"):
1349
+ severity_value = severity.value
1350
+ else:
1351
+ severity_value = str(severity)
1352
+
1353
+ priority_map = {"Critical": "High", "High": "High", "Moderate": "Moderate", "Low": "Low"}
1354
+
1355
+ return priority_map.get(severity_value, "Low")
1356
+
1357
+ def _collect_resource_information(self, failed_items: list) -> Dict[str, Any]:
1358
+ """Collect resource information from failed compliance items.
1359
+
1360
+ :param list failed_items: List of failed compliance items to process
1361
+ :return: Dictionary with resource information including affected_resources, severities, and descriptions
1362
+ :rtype: Dict[str, Any]
1363
+ """
1364
+ affected_resources = set()
1365
+ severities = []
1366
+ descriptions = []
1367
+
1368
+ for item in failed_items:
1369
+ affected_resources.add(item.resource_name)
1370
+ if item.severity:
1371
+ severities.append(item.severity)
1372
+ descriptions.append(f"- {item.resource_name}: {item.description}")
1373
+
1374
+ return {"affected_resources": affected_resources, "severities": severities, "descriptions": descriptions}
1375
+
1376
+ def _determine_highest_severity(self, severities: List[str]) -> str:
1377
+ """Determine the highest severity from a list of severities.
1378
+
1379
+ :param List[str] severities: List of severity strings to analyze
1380
+ :return: The highest severity found in the list
1381
+ :rtype: str
1382
+ """
1383
+ severity = "HIGH" # Default
1384
+ if severities:
1385
+ severity_order = ["CRITICAL", "HIGH", "MEDIUM", "LOW", "INFORMATIONAL"]
1386
+ for sev in severity_order:
1387
+ if sev in [s.upper() for s in severities]:
1388
+ severity = sev
1389
+ break
1390
+ return severity
1391
+
1392
+ def _build_consolidated_description(self, control_id: str, resource_info: Dict[str, Any]) -> str:
1393
+ """Build consolidated description for the finding.
1394
+
1395
+ :param str control_id: The control identifier
1396
+ :param Dict[str, Any] resource_info: Dictionary with resource information
1397
+ :return: Consolidated description string for the finding
1398
+ :rtype: str
1399
+ """
1400
+ affected_resources = resource_info["affected_resources"]
1401
+ descriptions = resource_info["descriptions"]
1402
+
1403
+ description = f"Control {control_id} failed for {len(affected_resources)} resource(s):\n\n"
1404
+ description += "\n".join(descriptions[:10]) # Limit to first 10 for readability
1405
+
1406
+ if len(descriptions) > 10:
1407
+ description += f"\n... and {len(descriptions) - 10} more resources"
1408
+
1409
+ return description
1410
+
1411
+ def _create_integration_finding(
1412
+ self, control_id: str, severity_enum: Any, description: str, representative_item: Any
1413
+ ) -> Any:
1414
+ """Create the IntegrationFinding object.
1415
+
1416
+ :param str control_id: The control identifier
1417
+ :param Any severity_enum: Severity enumeration value
1418
+ :param str description: Description for the finding
1419
+ :param Any representative_item: Representative compliance item
1420
+ :return: IntegrationFinding object
1421
+ :rtype: Any
1422
+ """
1423
+ from regscale.integrations.scanner_integration import IntegrationFinding
1424
+
1425
+ return IntegrationFinding(
1426
+ control_labels=[control_id],
1427
+ title=f"Compliance Violation: {control_id}",
1428
+ category="Compliance",
1429
+ plugin_name=f"{self.title} Compliance Scanner - {control_id}",
1430
+ severity=severity_enum,
1431
+ description=description,
1432
+ status="Open",
1433
+ priority=self._map_severity_to_priority(severity_enum),
1434
+ external_id=f"{self.title.lower().replace(' ', '-')}-control-{control_id}",
1435
+ first_seen=self.scan_date,
1436
+ last_seen=self.scan_date,
1437
+ scan_date=self.scan_date,
1438
+ asset_identifier=representative_item.resource_id,
1439
+ vulnerability_type="Compliance Violation",
1440
+ rule_id=control_id,
1441
+ baseline=representative_item.framework,
1442
+ affected_controls=control_id,
1443
+ )
1444
+
1445
+ def _create_finding_from_compliance_item(self, compliance_item: ComplianceItem) -> Optional[Any]:
1446
+ """
1447
+ Override parent method to properly set affected_controls for multi-control items.
1448
+
1449
+ :param ComplianceItem compliance_item: The compliance item
1450
+ :return: Finding object or None if creation fails
1451
+ :rtype: Optional[Any]
1452
+ """
1453
+ try:
1454
+ # Get severity mapping
1455
+ severity = compliance_item.severity or "Low"
1456
+ severity_enum = self._map_string_severity_to_enum(severity)
1457
+
1458
+ # Create the finding using the parent class structure
1459
+ from regscale.integrations.scanner_integration import IntegrationFinding
1460
+
1461
+ finding = IntegrationFinding(
1462
+ control_labels=[compliance_item.control_id],
1463
+ title=f"Compliance Violation: {compliance_item.control_id}",
1464
+ category="Compliance",
1465
+ plugin_name=f"{self.title} Compliance Scanner",
1466
+ severity=severity_enum,
1467
+ description=compliance_item.description,
1468
+ status="Open", # Use string instead of enum to avoid import issues
1469
+ priority=self._map_severity_to_priority(severity_enum),
1470
+ external_id=f"{self.title.lower()}-{compliance_item.control_id}-{compliance_item.resource_id}",
1471
+ first_seen=self.scan_date,
1472
+ last_seen=self.scan_date,
1473
+ scan_date=self.scan_date,
1474
+ asset_identifier=compliance_item.resource_id,
1475
+ vulnerability_type="Compliance Violation",
1476
+ rule_id=compliance_item.control_id,
1477
+ baseline=compliance_item.framework,
1478
+ affected_controls=compliance_item.affected_controls, # Use our property with all control IDs
1479
+ )
1480
+
1481
+ return finding
1482
+
1483
+ except Exception as e:
1484
+ logger.error(f"Error creating finding from compliance item: {e}")
1485
+ return None