regscale-cli 6.21.0.0__py3-none-any.whl → 6.21.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. regscale/_version.py +1 -1
  2. regscale/core/app/application.py +7 -0
  3. regscale/integrations/commercial/__init__.py +9 -10
  4. regscale/integrations/commercial/amazon/common.py +79 -2
  5. regscale/integrations/commercial/aws/cli.py +183 -9
  6. regscale/integrations/commercial/aws/scanner.py +544 -9
  7. regscale/integrations/commercial/cpe.py +18 -1
  8. regscale/integrations/commercial/import_all/import_all_cmd.py +2 -2
  9. regscale/integrations/commercial/microsoft_defender/__init__.py +0 -0
  10. regscale/integrations/commercial/{defender.py → microsoft_defender/defender.py} +38 -612
  11. regscale/integrations/commercial/microsoft_defender/defender_api.py +286 -0
  12. regscale/integrations/commercial/microsoft_defender/defender_constants.py +80 -0
  13. regscale/integrations/commercial/microsoft_defender/defender_scanner.py +168 -0
  14. regscale/integrations/commercial/qualys/__init__.py +24 -86
  15. regscale/integrations/commercial/qualys/containers.py +2 -0
  16. regscale/integrations/commercial/qualys/scanner.py +7 -2
  17. regscale/integrations/commercial/sonarcloud.py +110 -71
  18. regscale/integrations/commercial/tenablev2/jsonl_scanner.py +2 -1
  19. regscale/integrations/commercial/wizv2/async_client.py +10 -3
  20. regscale/integrations/commercial/wizv2/click.py +105 -26
  21. regscale/integrations/commercial/wizv2/constants.py +249 -1
  22. regscale/integrations/commercial/wizv2/data_fetcher.py +401 -0
  23. regscale/integrations/commercial/wizv2/finding_processor.py +295 -0
  24. regscale/integrations/commercial/wizv2/issue.py +2 -2
  25. regscale/integrations/commercial/wizv2/parsers.py +3 -2
  26. regscale/integrations/commercial/wizv2/policy_compliance.py +3057 -0
  27. regscale/integrations/commercial/wizv2/policy_compliance_helpers.py +564 -0
  28. regscale/integrations/commercial/wizv2/scanner.py +19 -25
  29. regscale/integrations/commercial/wizv2/utils.py +258 -85
  30. regscale/integrations/commercial/wizv2/variables.py +4 -3
  31. regscale/integrations/compliance_integration.py +1607 -0
  32. regscale/integrations/public/fedramp/fedramp_five.py +93 -8
  33. regscale/integrations/public/fedramp/markdown_parser.py +7 -1
  34. regscale/integrations/scanner_integration.py +57 -6
  35. regscale/models/__init__.py +1 -1
  36. regscale/models/app_models/__init__.py +1 -0
  37. regscale/models/integration_models/cisa_kev_data.json +103 -4
  38. regscale/models/integration_models/synqly_models/capabilities.json +1 -1
  39. regscale/{integrations/commercial/wizv2/models.py → models/integration_models/wizv2.py} +4 -12
  40. regscale/models/regscale_models/file.py +4 -0
  41. regscale/models/regscale_models/issue.py +151 -8
  42. regscale/models/regscale_models/regscale_model.py +4 -2
  43. regscale/models/regscale_models/security_plan.py +1 -1
  44. regscale/utils/graphql_client.py +3 -1
  45. {regscale_cli-6.21.0.0.dist-info → regscale_cli-6.21.2.0.dist-info}/METADATA +9 -9
  46. {regscale_cli-6.21.0.0.dist-info → regscale_cli-6.21.2.0.dist-info}/RECORD +52 -44
  47. tests/regscale/core/test_version_regscale.py +5 -3
  48. tests/regscale/integrations/test_wiz_policy_compliance_affected_controls.py +154 -0
  49. tests/regscale/test_authorization.py +0 -65
  50. tests/regscale/test_init.py +0 -96
  51. {regscale_cli-6.21.0.0.dist-info → regscale_cli-6.21.2.0.dist-info}/LICENSE +0 -0
  52. {regscale_cli-6.21.0.0.dist-info → regscale_cli-6.21.2.0.dist-info}/WHEEL +0 -0
  53. {regscale_cli-6.21.0.0.dist-info → regscale_cli-6.21.2.0.dist-info}/entry_points.txt +0 -0
  54. {regscale_cli-6.21.0.0.dist-info → regscale_cli-6.21.2.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,1607 @@
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ """
4
+ Abstract Compliance Integration Base Class
5
+
6
+ This module provides a base class for implementing compliance integrations
7
+ that follow common patterns across different compliance tools (Wiz, Tenable, Sicura).
8
+ """
9
+ import logging
10
+ import re
11
+ from abc import ABC, abstractmethod
12
+ from collections import defaultdict
13
+ from typing import Dict, List, Optional, Any, Iterator
14
+
15
+ from regscale.core.app.utils.app_utils import get_current_datetime, regscale_string_to_datetime
16
+ from regscale.integrations.scanner_integration import (
17
+ ScannerIntegration,
18
+ IntegrationAsset,
19
+ IntegrationFinding,
20
+ )
21
+ from regscale.models import regscale_models
22
+ from regscale.models.regscale_models import (
23
+ Catalog,
24
+ SecurityControl,
25
+ ControlImplementation,
26
+ Assessment,
27
+ ImplementationObjective,
28
+ )
29
+
30
+ logger = logging.getLogger("regscale")
31
+
32
+ # Safer, linear-time regex for control-id parsing/normalization used across
33
+ # compliance integrations. Supports: 'AC-4', 'AC-4(2)', 'AC-4 (2)', 'AC-4-2', 'AC-4 2'
34
+ # Distinct branches ('(', '-' or whitespace) avoid ambiguous nested alternation
35
+ # and excessive backtracking that could be used for DoS.
36
+ SAFE_CONTROL_ID_RE = re.compile( # NOSONAR
37
+ r"^([A-Za-z]{2}-\d+)(?:\s*\(\s*(\d+)\s*\)|-\s*(\d+)|\s+(\d+))?$", # NOSONAR
38
+ re.IGNORECASE, # NOSONAR
39
+ )
40
+
41
+
42
+ class ComplianceItem(ABC):
43
+ """
44
+ Abstract base class representing a compliance assessment item.
45
+
46
+ This represents a single compliance check result for a specific
47
+ resource against a specific control.
48
+ """
49
+
50
+ @property
51
+ @abstractmethod
52
+ def resource_id(self) -> str:
53
+ """Unique identifier for the resource being assessed."""
54
+ pass
55
+
56
+ @property
57
+ @abstractmethod
58
+ def resource_name(self) -> str:
59
+ """Human-readable name of the resource."""
60
+ pass
61
+
62
+ @property
63
+ @abstractmethod
64
+ def control_id(self) -> str:
65
+ """Control identifier (e.g., AC-3, SI-2)."""
66
+ pass
67
+
68
+ @property
69
+ @abstractmethod
70
+ def compliance_result(self) -> str:
71
+ """Result of compliance check (PASS, FAIL, etc)."""
72
+ pass
73
+
74
+ @property
75
+ @abstractmethod
76
+ def severity(self) -> Optional[str]:
77
+ """Severity level of the compliance violation (if failed)."""
78
+ pass
79
+
80
+ @property
81
+ @abstractmethod
82
+ def description(self) -> str:
83
+ """Description of the compliance check."""
84
+ pass
85
+
86
+ @property
87
+ @abstractmethod
88
+ def framework(self) -> str:
89
+ """Compliance framework (e.g., NIST800-53R5, CSF)."""
90
+ pass
91
+
92
+
93
+ class ComplianceIntegration(ScannerIntegration, ABC):
94
+ """
95
+ Abstract base class for compliance integrations.
96
+
97
+ This class provides common patterns for:
98
+ - Processing compliance data
99
+ - Creating assets from compliance items
100
+ - Creating findings/issues for failed compliance
101
+ - Mapping compliance items to controls
102
+ - Creating assessments and updating control status
103
+ """
104
+
105
+ # Status mapping constants
106
+ PASS_STATUSES = ["PASS", "PASSED", "pass", "passed"]
107
+ FAIL_STATUSES = ["FAIL", "FAILED", "fail", "failed"]
108
+
109
+ def __init__(
110
+ self,
111
+ plan_id: int,
112
+ catalog_id: Optional[int] = None,
113
+ framework: str = "NIST800-53R5",
114
+ create_issues: bool = True,
115
+ update_control_status: bool = True,
116
+ create_poams: bool = False,
117
+ parent_module: str = "securityplans",
118
+ **kwargs,
119
+ ):
120
+ """
121
+ Initialize compliance integration.
122
+
123
+ :param int plan_id: RegScale plan ID
124
+ :param Optional[int] catalog_id: RegScale catalog ID
125
+ :param str framework: Compliance framework
126
+ :param bool create_issues: Whether to create issues for failed compliance
127
+ :param bool update_control_status: Whether to update control implementation status
128
+ :param bool create_poams: Whether to mark issues as POAMs
129
+ """
130
+ super().__init__(plan_id=plan_id, **kwargs)
131
+
132
+ self.catalog_id = catalog_id
133
+ self.framework = framework
134
+ self.create_issues = create_issues
135
+ self.update_control_status = update_control_status
136
+ self.create_poams = create_poams
137
+
138
+ # Compliance data storage
139
+ self.all_compliance_items: List[ComplianceItem] = []
140
+ self.failed_compliance_items: List[ComplianceItem] = []
141
+ self.passing_controls: Dict[str, ComplianceItem] = {}
142
+ self.failing_controls: Dict[str, ComplianceItem] = {}
143
+
144
+ # Asset mapping for compliance to asset correlation
145
+ self.asset_compliance_map: Dict[str, List[ComplianceItem]] = defaultdict(list)
146
+
147
+ # Initialize caches for existing records to prevent duplicates
148
+ self._existing_assets_cache: Dict[str, regscale_models.Asset] = {}
149
+ self._existing_issues_cache: Dict[str, regscale_models.Issue] = {}
150
+ self._existing_assessments_cache: Dict[str, regscale_models.Assessment] = {}
151
+ self._cache_loaded = False
152
+
153
+ # Mapping caches for linking issues to implementations and assessments
154
+ # Key: canonical control id string (e.g., "AC-2(1)") -> ControlImplementation.id
155
+ self._impl_id_by_control: Dict[str, int] = {}
156
+ # Key: ControlImplementation.id -> Assessment created/updated today
157
+ self._assessment_by_impl_today: Dict[int, regscale_models.Assessment] = {}
158
+ # suppress asset not found errors in non-debug modes
159
+ self.suppress_asset_not_found_errors = logger.level != logging.DEBUG
160
+ # Set scan date
161
+ self.scan_date = get_current_datetime()
162
+
163
+ def is_poam(self, finding: IntegrationFinding) -> bool: # type: ignore[override]
164
+ """
165
+ Determines if an issue should be considered a POAM for compliance integrations.
166
+
167
+ - If the integration was initialized with `create_poams=True` (e.g., via `--create-poams/-cp`),
168
+ always return True so newly created and updated issues are POAMs.
169
+ - Otherwise, defer to the generic scanner behavior.
170
+ """
171
+ try:
172
+ if getattr(self, "create_poams", False):
173
+ return True
174
+ if finding.due_date >= get_current_datetime():
175
+ return True
176
+ except Exception:
177
+ pass
178
+ return super().is_poam(finding)
179
+
180
+ def _load_existing_records_cache(self) -> None:
181
+ """
182
+ Load existing RegScale records into cache to prevent duplicates.
183
+ This method populates caches for assets, issues, and assessments.
184
+
185
+ :return: None
186
+ :rtype: None
187
+ """
188
+ if self._cache_loaded:
189
+ return
190
+
191
+ logger.info("Loading existing RegScale records to prevent duplicates...")
192
+
193
+ try:
194
+ # Load existing assets for this plan
195
+ self._load_existing_assets()
196
+
197
+ # Load existing issues for this plan
198
+ self._load_existing_issues()
199
+
200
+ # Load existing assessments for control implementations
201
+ self._load_existing_assessments()
202
+
203
+ self._cache_loaded = True
204
+ logger.info("🗄️ Loaded existing records cache to prevent duplicates:")
205
+ logger.info(f" - Assets: {len(self._existing_assets_cache)}")
206
+ logger.info(f" - Issues: {len(self._existing_issues_cache)}")
207
+ logger.info(f" - Assessments: {len(self._existing_assessments_cache)}")
208
+
209
+ except Exception as e:
210
+ logger.error(f"Error loading existing records cache: {e}")
211
+ # Continue without cache to avoid blocking the integration
212
+ self._cache_loaded = True
213
+
214
+ def _load_existing_assets(self) -> None:
215
+ """
216
+ Load existing assets into cache.
217
+
218
+ :return: None
219
+ :rtype: None
220
+ """
221
+ try:
222
+ # Get all assets for this plan
223
+ existing_assets = regscale_models.Asset.get_all_by_parent(
224
+ parent_id=self.plan_id, parent_module=self.parent_module
225
+ )
226
+
227
+ for asset in existing_assets:
228
+ # Cache by external_id, identifier, and other_tracking_number for flexible lookup
229
+ if hasattr(asset, "externalId") and asset.externalId:
230
+ self._existing_assets_cache[asset.externalId] = asset
231
+ if hasattr(asset, "identifier") and asset.identifier:
232
+ self._existing_assets_cache[asset.identifier] = asset
233
+ if hasattr(asset, "otherTrackingNumber") and asset.otherTrackingNumber:
234
+ self._existing_assets_cache[asset.otherTrackingNumber] = asset
235
+
236
+ except Exception as e:
237
+ logger.debug(f"Error loading existing assets: {e}")
238
+
239
+ def _load_existing_issues(self) -> None:
240
+ """
241
+ Load existing issues into cache.
242
+
243
+ Uses both plan-level and control-level queries to ensure all relevant issues are found.
244
+
245
+ :return: None
246
+ :rtype: None
247
+ """
248
+ try:
249
+ all_issues = set()
250
+
251
+ # Method 1: Get issues directly associated with the plan
252
+ plan_issues = regscale_models.Issue.get_all_by_parent(
253
+ parent_id=self.plan_id, parent_module=self.parent_module
254
+ )
255
+ all_issues.update(plan_issues)
256
+ logger.debug(f"🔍 Found {len(plan_issues)} issues directly under plan {self.plan_id}")
257
+
258
+ # Method 2: Get issues associated with control implementations (matches scanner integration logic)
259
+ try:
260
+ issues_by_impl = regscale_models.Issue.get_open_issues_ids_by_implementation_id(
261
+ plan_id=self.plan_id, is_component=getattr(self, "is_component", False)
262
+ )
263
+ impl_issues_count = 0
264
+ for impl_id, issue_list in issues_by_impl.items():
265
+ for issue_dict in issue_list:
266
+ # issue_dict contains issue data, need to get the actual issue object
267
+ issue_id = issue_dict.get("id")
268
+ if issue_id:
269
+ try:
270
+ issue = regscale_models.Issue.get_object(object_id=issue_id)
271
+ if issue:
272
+ all_issues.add(issue)
273
+ impl_issues_count += 1
274
+ except Exception as e:
275
+ logger.debug(f"Could not load issue {issue_id}: {e}")
276
+
277
+ logger.debug(f"🔍 Found {impl_issues_count} additional issues via control implementations")
278
+ except Exception as e:
279
+ logger.debug(f"Could not load issues by control implementation: {e}")
280
+
281
+ logger.debug(f"🔍 Total unique issues found: {len(all_issues)} for plan {self.plan_id}")
282
+
283
+ wiz_issues = 0
284
+ for issue in all_issues:
285
+ # Cache by external_id and other_identifier for flexible lookup
286
+ if hasattr(issue, "externalId") and issue.externalId:
287
+ self._existing_issues_cache[issue.externalId] = issue
288
+ if "wiz-policy" in issue.externalId.lower():
289
+ wiz_issues += 1
290
+ logger.debug(f"📋 Cached Wiz issue: {issue.id} -> external_id: {issue.externalId}")
291
+ if hasattr(issue, "otherIdentifier") and issue.otherIdentifier:
292
+ self._existing_issues_cache[issue.otherIdentifier] = issue
293
+
294
+ logger.debug(f"🔍 Cached {wiz_issues} Wiz policy issues out of {len(all_issues)} total issues")
295
+
296
+ except Exception as e:
297
+ logger.debug(f"Error loading existing issues: {e}")
298
+
299
+ def _load_existing_assessments(self) -> None:
300
+ """
301
+ Load existing assessments into cache.
302
+
303
+ :return: None
304
+ :rtype: None
305
+ """
306
+ try:
307
+ # Get control implementations for this plan to find their assessments
308
+ implementations = ControlImplementation.get_all_by_parent(
309
+ parent_id=self.plan_id, parent_module=self.parent_module
310
+ )
311
+
312
+ for implementation in implementations:
313
+ try:
314
+ # Get assessments for this implementation
315
+ assessments = regscale_models.Assessment.get_all_by_parent(
316
+ parent_id=implementation.id, parent_module="controls"
317
+ )
318
+
319
+ for assessment in assessments:
320
+ # Create cache key: impl_id + day (YYYY-MM-DD)
321
+ if hasattr(assessment, "actualFinish") and assessment.actualFinish:
322
+ try:
323
+ # actualFinish may be a string; normalize to date-only key
324
+ if hasattr(assessment.actualFinish, "date"):
325
+ day_key = assessment.actualFinish.date().isoformat()
326
+ else:
327
+ day_key = regscale_string_to_datetime(assessment.actualFinish).date().isoformat()
328
+ cache_key = f"{implementation.id}_{day_key}"
329
+ self._existing_assessments_cache[cache_key] = assessment
330
+ except Exception:
331
+ continue
332
+
333
+ except Exception as e:
334
+ logger.debug(f"Error loading assessments for implementation {implementation.id}: {e}")
335
+
336
+ except Exception as e:
337
+ logger.debug(f"Error loading existing assessments: {e}")
338
+
339
+ def _find_existing_asset_cached(self, resource_id: str) -> Optional[regscale_models.Asset]:
340
+ """
341
+ Find existing asset by resource ID using cache.
342
+
343
+ :param str resource_id: Resource identifier to search for
344
+ :return: Existing asset or None if not found
345
+ :rtype: Optional[regscale_models.Asset]
346
+ """
347
+ return self._existing_assets_cache.get(resource_id)
348
+
349
+ def _find_existing_issue_cached(self, external_id: str) -> Optional[regscale_models.Issue]:
350
+ """
351
+ Find existing issue by external ID using cache.
352
+
353
+ :param str external_id: External identifier to search for
354
+ :return: Existing issue or None if not found
355
+ :rtype: Optional[regscale_models.Issue]
356
+ """
357
+ return self._existing_issues_cache.get(external_id)
358
+
359
+ def _find_existing_assessment_cached(
360
+ self, implementation_id: int, scan_date
361
+ ) -> Optional[regscale_models.Assessment]:
362
+ """
363
+ Find existing assessment by implementation ID and date using cache.
364
+
365
+ :param int implementation_id: Control implementation ID
366
+ :param scan_date: Scan date to check against existing assessments
367
+ :return: Existing assessment or None if not found
368
+ :rtype: Optional[regscale_models.Assessment]
369
+ """
370
+ # Normalize to date-only key
371
+ try:
372
+ if hasattr(scan_date, "date"):
373
+ day_key = scan_date.date().isoformat()
374
+ else:
375
+ day_key = regscale_string_to_datetime(str(scan_date)).date().isoformat()
376
+ except Exception:
377
+ day_key = str(scan_date).split(" ")[0]
378
+ cache_key = f"{implementation_id}_{day_key}"
379
+ return self._existing_assessments_cache.get(cache_key)
380
+
381
+ @abstractmethod
382
+ def fetch_compliance_data(self) -> List[Any]:
383
+ """
384
+ Fetch raw compliance data from the external system.
385
+
386
+ :return: List of raw compliance data (will be converted to ComplianceItems)
387
+ :rtype: List[Any]
388
+ """
389
+ pass
390
+
391
+ @abstractmethod
392
+ def create_compliance_item(self, raw_data: Any) -> ComplianceItem:
393
+ """
394
+ Create a ComplianceItem from raw compliance data.
395
+
396
+ :param Any raw_data: Raw compliance data from external system
397
+ :return: ComplianceItem instance
398
+ :rtype: ComplianceItem
399
+ """
400
+ pass
401
+
402
+ def process_compliance_data(self) -> None:
403
+ """
404
+ Process compliance data and categorize items.
405
+
406
+ Separates passing and failing compliance items and builds
407
+ control status mappings.
408
+
409
+ :return: None
410
+ :rtype: None
411
+ """
412
+ logger.info("Processing compliance data...")
413
+
414
+ # Reset state to avoid double counting on repeated calls
415
+ self.all_compliance_items = []
416
+ self.failed_compliance_items = []
417
+ self.passing_controls = {}
418
+ self.failing_controls = {}
419
+ self.asset_compliance_map.clear()
420
+
421
+ # Build allowed control IDs from plan/catalog controls to restrict scope
422
+ allowed_controls_normalized: set[str] = set()
423
+ try:
424
+ controls = self._get_controls()
425
+ for ctl in controls:
426
+ cid = (ctl.get("controlId") or "").strip()
427
+ if not cid:
428
+ continue
429
+ base, sub = self._normalize_control_id(cid)
430
+ normalized = f"{base}({sub})" if sub else base
431
+ allowed_controls_normalized.add(normalized)
432
+ except Exception:
433
+ # If controls cannot be loaded, proceed without additional filtering
434
+ allowed_controls_normalized = set()
435
+
436
+ # Fetch raw compliance data
437
+ raw_compliance_data = self.fetch_compliance_data()
438
+
439
+ # Convert to ComplianceItem objects
440
+ for raw_item in raw_compliance_data:
441
+ try:
442
+ compliance_item = self.create_compliance_item(raw_item)
443
+ # Skip items that do not resolve to a control or resource
444
+ if not getattr(compliance_item, "control_id", "") or not getattr(compliance_item, "resource_id", ""):
445
+ continue
446
+
447
+ # If we have an allowed set, restrict to only controls in current plan/catalog
448
+ if allowed_controls_normalized:
449
+ base, sub = self._normalize_control_id(getattr(compliance_item, "control_id", ""))
450
+ norm_item = f"{base}({sub})" if sub else base
451
+ if norm_item not in allowed_controls_normalized:
452
+ continue
453
+ self.all_compliance_items.append(compliance_item)
454
+
455
+ # Build asset mapping
456
+ self.asset_compliance_map[compliance_item.resource_id].append(compliance_item)
457
+
458
+ # Categorize by result
459
+ if compliance_item.compliance_result in self.FAIL_STATUSES:
460
+ self.failed_compliance_items.append(compliance_item)
461
+ # Track failing controls (control can fail if ANY asset fails)
462
+ control_key = compliance_item.control_id.lower()
463
+ self.failing_controls[control_key] = compliance_item
464
+ # Remove from passing if it was there
465
+ self.passing_controls.pop(control_key, None)
466
+
467
+ elif compliance_item.compliance_result in self.PASS_STATUSES:
468
+ control_key = compliance_item.control_id.lower()
469
+ # Only mark as passing if not already failing
470
+ if control_key not in self.failing_controls:
471
+ self.passing_controls[control_key] = compliance_item
472
+
473
+ except Exception as e:
474
+ logger.error(f"Error processing compliance item: {e}")
475
+ continue
476
+
477
+ logger.debug(
478
+ f"Processed {len(self.all_compliance_items)} compliance items: "
479
+ f"{len(self.all_compliance_items) - len(self.failed_compliance_items)} passing, "
480
+ f"{len(self.failed_compliance_items)} failing"
481
+ )
482
+
483
+ def create_asset_from_compliance_item(self, compliance_item: ComplianceItem) -> Optional[IntegrationAsset]:
484
+ """
485
+ Create an IntegrationAsset from a compliance item.
486
+
487
+ :param ComplianceItem compliance_item: The compliance item
488
+ :return: IntegrationAsset or None
489
+ :rtype: Optional[IntegrationAsset]
490
+ """
491
+ try:
492
+ # Check if asset already exists
493
+ existing_asset = self._find_existing_asset_by_resource_id(compliance_item.resource_id)
494
+ if existing_asset:
495
+ return None
496
+
497
+ asset_type = self._map_resource_type_to_asset_type(compliance_item)
498
+
499
+ asset = IntegrationAsset(
500
+ name=compliance_item.resource_name,
501
+ identifier=compliance_item.resource_id,
502
+ external_id=compliance_item.resource_id,
503
+ other_tracking_number=compliance_item.resource_id, # For deduplication
504
+ asset_type=asset_type,
505
+ asset_category=regscale_models.AssetCategory.Hardware,
506
+ description=f"Asset from {self.title} compliance scan",
507
+ parent_id=self.plan_id,
508
+ parent_module=self.parent_module,
509
+ status=regscale_models.AssetStatus.Active,
510
+ date_last_updated=self.scan_date,
511
+ )
512
+
513
+ return asset
514
+
515
+ except Exception as e:
516
+ logger.error(f"Error creating asset from compliance item: {e}")
517
+ return None
518
+
519
+ def create_finding_from_compliance_item(self, compliance_item: ComplianceItem) -> Optional[IntegrationFinding]:
520
+ """
521
+ Create an IntegrationFinding from a failed compliance item.
522
+
523
+ :param ComplianceItem compliance_item: The compliance item
524
+ :return: IntegrationFinding or None
525
+ :rtype: Optional[IntegrationFinding]
526
+ """
527
+ try:
528
+ control_labels = [compliance_item.control_id] if compliance_item.control_id else []
529
+ severity = self._map_severity(compliance_item.severity)
530
+
531
+ finding = IntegrationFinding(
532
+ control_labels=control_labels,
533
+ title=f"Compliance Violation: {compliance_item.control_id}",
534
+ category="Compliance",
535
+ plugin_name=f"{self.title} Compliance Scanner",
536
+ severity=severity,
537
+ description=compliance_item.description,
538
+ status=regscale_models.IssueStatus.Open,
539
+ priority=self._map_severity_to_priority(severity),
540
+ external_id=f"{self.title.lower()}-{compliance_item.control_id}-{compliance_item.resource_id}",
541
+ first_seen=self.scan_date,
542
+ last_seen=self.scan_date,
543
+ scan_date=self.scan_date,
544
+ asset_identifier=compliance_item.resource_id,
545
+ vulnerability_type="Compliance Violation",
546
+ rule_id=compliance_item.control_id,
547
+ baseline=compliance_item.framework,
548
+ affected_controls=",".join(compliance_item.control_id),
549
+ )
550
+
551
+ # Ensure affected controls are set to the normalized control label (e.g., RA-5, AC-2(1))
552
+ if compliance_item.control_id:
553
+ base, sub = self._normalize_control_id(compliance_item.control_id)
554
+ finding.affected_controls = f"{base}({sub})" if sub else base
555
+
556
+ return finding
557
+
558
+ except Exception as e:
559
+ logger.error(f"Error creating finding from compliance item: {e}")
560
+ return None
561
+
562
+ def fetch_assets(self, *args, **kwargs) -> Iterator[IntegrationAsset]:
563
+ """
564
+ Fetch assets from compliance items, avoiding duplicates.
565
+
566
+ :param args: Variable positional arguments
567
+ :param kwargs: Variable keyword arguments
568
+ :return: Iterator of integration assets
569
+ :rtype: Iterator[IntegrationAsset]
570
+ """
571
+ logger.info("Fetching assets from compliance items...")
572
+
573
+ # Load cache if not already loaded
574
+ self._load_existing_records_cache()
575
+
576
+ processed_resources = set()
577
+ for compliance_item in self.all_compliance_items:
578
+ if compliance_item.resource_id not in processed_resources:
579
+ # Check if asset already exists in RegScale
580
+ existing_asset = self._find_existing_asset_cached(compliance_item.resource_id)
581
+ if existing_asset:
582
+ logger.debug(f"Asset already exists for resource {compliance_item.resource_id}, skipping creation")
583
+ processed_resources.add(compliance_item.resource_id)
584
+ continue
585
+
586
+ asset = self.create_asset_from_compliance_item(compliance_item)
587
+ if asset:
588
+ processed_resources.add(compliance_item.resource_id)
589
+ yield asset
590
+
591
+ def fetch_findings(self, *args, **kwargs) -> Iterator[IntegrationFinding]:
592
+ """
593
+ Fetch findings from failed compliance items.
594
+
595
+ :param args: Variable positional arguments
596
+ :param kwargs: Variable keyword arguments
597
+ :return: Iterator of integration findings
598
+ :rtype: Iterator[IntegrationFinding]
599
+ """
600
+ logger.info("Fetching findings from failed compliance items...")
601
+
602
+ total = len(self.failed_compliance_items)
603
+ task_id = self.finding_progress.add_task(
604
+ f"[#f68d1f]Creating findings from {total} failed compliance item(s)...",
605
+ total=total or None,
606
+ )
607
+
608
+ for compliance_item in self.failed_compliance_items:
609
+ finding = self.create_finding_from_compliance_item(compliance_item)
610
+ if finding:
611
+ self.finding_progress.advance(task_id, 1)
612
+ yield finding
613
+
614
+ # Ensure task completes if total is known
615
+ if total:
616
+ self.finding_progress.update(task_id, completed=total)
617
+
618
+ def sync_compliance(self) -> None:
619
+ """
620
+ Main method to sync compliance data.
621
+
622
+ This method orchestrates the entire compliance sync process:
623
+ 1. Process compliance data
624
+ 2. Create assets and findings
625
+ 3. Create/update control assessments
626
+ 4. Update control implementation status
627
+
628
+ :return: None
629
+ :rtype: None
630
+ """
631
+ logger.info(f"Starting {self.title} compliance sync...")
632
+
633
+ try:
634
+ scan_history = self.create_scan_history()
635
+ self.process_compliance_data()
636
+
637
+ self._sync_assets()
638
+ self._sync_control_assessments()
639
+ self._sync_issues()
640
+ self._finalize_scan_history(scan_history)
641
+
642
+ logger.info(f"Completed {self.title} compliance sync")
643
+
644
+ except Exception as e:
645
+ logger.error(f"Error during compliance sync: {e}")
646
+ raise
647
+
648
+ def _sync_assets(self) -> None:
649
+ """
650
+ Process and sync assets from compliance items.
651
+
652
+ :return: None
653
+ :rtype: None
654
+ """
655
+ assets = list(self.fetch_assets())
656
+ if not assets:
657
+ logger.debug("No assets generated from compliance items")
658
+ return
659
+
660
+ assets_processed = self.update_regscale_assets(iter(assets))
661
+ self._log_asset_results(assets_processed)
662
+
663
+ def _log_asset_results(self, assets_processed: int) -> None:
664
+ """
665
+ Log asset processing results.
666
+
667
+ :param int assets_processed: Number of assets processed
668
+ :return: None
669
+ :rtype: None
670
+ """
671
+ results = getattr(self, "_results", {}).get("assets", {})
672
+ created = results.get("created_count", 0)
673
+ updated = results.get("updated_count", 0)
674
+ deleted = results.get("deleted_count", 0) if isinstance(results, dict) else 0
675
+
676
+ if deleted > 0:
677
+ logger.info(
678
+ f"Assets processed: {assets_processed} (created: {created}, updated: {updated}, deleted: {deleted})"
679
+ )
680
+ elif created > 0 or updated > 0:
681
+ logger.info(f"Assets processed: {assets_processed} (created: {created}, updated: {updated})")
682
+ else:
683
+ logger.debug(f"Assets processed: {assets_processed} (no changes made)")
684
+
685
+ def _sync_control_assessments(self) -> None:
686
+ """
687
+ Process control assessments if enabled.
688
+
689
+ :return: None
690
+ :rtype: None
691
+ """
692
+ if self.update_control_status:
693
+ self._process_control_assessments()
694
+
695
+ def _sync_issues(self) -> None:
696
+ """
697
+ Process and sync issues from failed compliance items.
698
+
699
+ :return: None
700
+ :rtype: None
701
+ """
702
+ if not self.create_issues:
703
+ return
704
+
705
+ findings = list(self.fetch_findings())
706
+ if not findings:
707
+ logger.debug("No findings to process into issues")
708
+ return
709
+
710
+ issues_created, issues_skipped = self._process_findings_to_issues(findings)
711
+ self._log_issue_results(issues_created, issues_skipped)
712
+
713
+ def _process_findings_to_issues(self, findings: List[IntegrationFinding]) -> tuple[int, int]:
714
+ """
715
+ Process findings into issues and return counts.
716
+
717
+ :param findings: List of findings to process
718
+ :return: Tuple of (issues_created, issues_skipped)
719
+ """
720
+ issues_created = 0
721
+ issues_skipped = 0
722
+
723
+ for finding in findings:
724
+ try:
725
+ if self._process_single_finding(finding):
726
+ issues_created += 1
727
+ else:
728
+ issues_skipped += 1
729
+ except Exception as e:
730
+ logger.error(f"Error processing finding: {e}")
731
+ issues_skipped += 1
732
+
733
+ return issues_created, issues_skipped
734
+
735
+ def _process_single_finding(self, finding: IntegrationFinding) -> bool:
736
+ """
737
+ Process a single finding into an issue.
738
+
739
+ :param finding: Finding to process
740
+ :return: True if issue was created/updated, False if skipped
741
+ """
742
+ asset = self._get_or_create_asset_for_finding(finding)
743
+ if not asset:
744
+ self._log_asset_not_found_error(finding)
745
+ return False
746
+
747
+ issue_title = self.get_issue_title(finding)
748
+ issue = self.create_or_update_issue_from_finding(title=issue_title, finding=finding)
749
+ return issue is not None
750
+
751
+ def _get_or_create_asset_for_finding(self, finding: IntegrationFinding) -> Optional[regscale_models.Asset]:
752
+ """
753
+ Get existing asset or create one on-demand for the finding.
754
+
755
+ :param IntegrationFinding finding: Finding needing an asset
756
+ :return: Asset if found/created, None otherwise
757
+ :rtype: Optional[regscale_models.Asset]
758
+ """
759
+ asset = self.get_asset_by_identifier(finding.asset_identifier)
760
+ if not asset:
761
+ asset = self._ensure_asset_for_finding(finding)
762
+ return asset
763
+
764
+ def _log_asset_not_found_error(self, finding: IntegrationFinding) -> None:
765
+ """
766
+ Log error when asset is not found for a finding.
767
+
768
+ :param IntegrationFinding finding: Finding with missing asset
769
+ :return: None
770
+ :rtype: None
771
+ """
772
+ if not getattr(self, "suppress_asset_not_found_errors", False):
773
+ logger.error(
774
+ f"Asset not found for identifier {finding.asset_identifier} — "
775
+ "skipping issue creation for this finding"
776
+ )
777
+
778
+ def _log_issue_results(self, issues_created: int, issues_skipped: int) -> None:
779
+ """
780
+ Log issue processing results.
781
+
782
+ :param int issues_created: Number of issues created/updated
783
+ :param int issues_skipped: Number of issues skipped
784
+ :return: None
785
+ :rtype: None
786
+ """
787
+ if issues_created > 0:
788
+ logger.info(f"Issues processed: {issues_created} created/updated, {issues_skipped} skipped")
789
+ elif issues_skipped > 0:
790
+ logger.warning(f"Issues processed: 0 created, {issues_skipped} skipped (assets not found)")
791
+ else:
792
+ logger.debug("No issues processed")
793
+
794
+ def _finalize_scan_history(self, scan_history: regscale_models.ScanHistory) -> None:
795
+ """
796
+ Finalize scan history with error handling.
797
+
798
+ :param regscale_models.ScanHistory scan_history: Scan history to update
799
+ :return: None
800
+ :rtype: None
801
+ """
802
+ try:
803
+ if getattr(self, "enable_scan_history", True):
804
+ self._update_scan_history(scan_history)
805
+ except Exception:
806
+ self._update_scan_history(scan_history)
807
+
808
+ def _ensure_asset_for_finding(self, finding: IntegrationFinding) -> Optional[regscale_models.Asset]:
809
+ """
810
+ Ensure an asset exists for the given finding.
811
+
812
+ Attempts to locate the asset by identifier. If missing, it will try to
813
+ build an IntegrationAsset from the first compliance item associated with
814
+ the resource id and upsert it into RegScale, then return the created asset.
815
+
816
+ :param IntegrationFinding finding: Finding referencing the asset identifier
817
+ :return: The located or newly created Asset, or None if it cannot be created
818
+ :rtype: Optional[regscale_models.Asset]
819
+ """
820
+ try:
821
+ resource_id = getattr(finding, "asset_identifier", None)
822
+ if not resource_id:
823
+ return None
824
+
825
+ # Re-check cache/DB
826
+ asset = self.get_asset_by_identifier(resource_id)
827
+ if asset:
828
+ return asset
829
+
830
+ # Use compliance items we already processed to construct an asset
831
+ related_items = self.asset_compliance_map.get(resource_id, [])
832
+ if not related_items:
833
+ return None
834
+
835
+ candidate_item = related_items[0]
836
+ integration_asset = self.create_asset_from_compliance_item(candidate_item)
837
+ if not integration_asset:
838
+ return None
839
+
840
+ # Persist the asset and refresh lookup
841
+ _ = self.update_regscale_assets(iter([integration_asset]))
842
+ return self.get_asset_by_identifier(resource_id)
843
+
844
+ except Exception as ensure_exc:
845
+ logger.debug(
846
+ f"On-demand asset creation failed for {getattr(finding, 'asset_identifier', None)}: {ensure_exc}"
847
+ )
848
+ return None
849
+
850
+ def _process_control_assessments(self) -> None:
851
+ """
852
+ Process control assessments based on compliance results.
853
+
854
+ This follows the same pattern as the original Wiz compliance integration:
855
+ 1. Get control implementations
856
+ 2. For each implementation, get the security control using controlID
857
+ 3. Match the security control's controlId with the extracted control ID from compliance items
858
+
859
+ :return: None
860
+ :rtype: None
861
+ """
862
+ logger.info("Processing control assessments...")
863
+
864
+ # Ensure existing records cache (including assessments) is loaded to prevent duplicates
865
+ self._load_existing_records_cache()
866
+
867
+ implementations = self._get_control_implementations()
868
+ if not implementations:
869
+ logger.warning("No control implementations found for assessment processing")
870
+ return
871
+
872
+ all_control_ids = set(self.passing_controls.keys()) | set(self.failing_controls.keys())
873
+ logger.info(f"Processing assessments for {len(all_control_ids)} controls with compliance data")
874
+ logger.info(f"Control IDs with data: {sorted(list(all_control_ids))}")
875
+ self._log_sample_controls(implementations)
876
+
877
+ assessments_created = 0
878
+ processed_impl_today: set[int] = set()
879
+ for control_id in all_control_ids:
880
+ created = self._process_single_control_assessment(
881
+ control_id=control_id,
882
+ implementations=implementations,
883
+ processed_impl_today=processed_impl_today,
884
+ )
885
+ assessments_created += created
886
+
887
+ if assessments_created > 0:
888
+ logger.info(f"Successfully created {assessments_created} control assessments")
889
+ passing_assessments = len([cid for cid in all_control_ids if cid not in self.failing_controls])
890
+ failing_assessments = len([cid for cid in all_control_ids if cid in self.failing_controls])
891
+ logger.info(f"Assessment breakdown: {passing_assessments} passing, {failing_assessments} failing")
892
+ logger.debug(f"Control implementation mappings created: {len(self._impl_id_by_control)}")
893
+ if self._impl_id_by_control:
894
+ logger.debug(f"Sample mappings: {dict(list(self._impl_id_by_control.items())[:5])}")
895
+ logger.debug(f"Today's assessments by implementation: {len(self._assessment_by_impl_today)}")
896
+ if self._assessment_by_impl_today:
897
+ logger.debug(f"Sample assessment mappings: {dict(list(self._assessment_by_impl_today.items())[:5])}")
898
+
899
+ def _get_control_implementations(self) -> List[ControlImplementation]:
900
+ """
901
+ Get all control implementations for the current plan.
902
+
903
+ :return: List of control implementations
904
+ :rtype: List[ControlImplementation]
905
+ """
906
+ implementations: List[ControlImplementation] = ControlImplementation.get_all_by_parent(
907
+ parent_module=self.parent_module, parent_id=self.plan_id
908
+ )
909
+ logger.info(f"Found {len(implementations)} control implementations")
910
+ return implementations
911
+
912
+ def _log_sample_controls(self, implementations: List[ControlImplementation]) -> None:
913
+ """
914
+ Log sample control IDs for debugging purposes.
915
+
916
+ :param List[ControlImplementation] implementations: List of implementations to sample from
917
+ :return: None
918
+ :rtype: None
919
+ """
920
+ sample_regscale_controls: List[str] = []
921
+ for impl in implementations[:10]:
922
+ try:
923
+ sec_control = SecurityControl.get_object(object_id=impl.controlID)
924
+ if sec_control and sec_control.controlId:
925
+ sample_regscale_controls.append(f"{sec_control.controlId}")
926
+ else:
927
+ sample_regscale_controls.append(f"NoControlId-impl:{impl.id}-controlID:{impl.controlID}")
928
+ except Exception as e: # noqa: BLE001
929
+ sample_regscale_controls.append(f"ERROR-impl:{impl.id}-controlID:{impl.controlID}-error:{str(e)[:50]}")
930
+ logger.error(
931
+ f"Error fetching SecurityControl for implementation {impl.id} with controlID {impl.controlID}: {e}"
932
+ )
933
+ logger.info(f"Sample RegScale control IDs available: {sample_regscale_controls}")
934
+
935
+ def _process_single_control_assessment(
936
+ self,
937
+ *,
938
+ control_id: str,
939
+ implementations: List[ControlImplementation],
940
+ processed_impl_today: set[int],
941
+ ) -> int:
942
+ """
943
+ Process assessment for a single control.
944
+
945
+ :param str control_id: Control identifier to process
946
+ :param List[ControlImplementation] implementations: Available control implementations
947
+ :param set[int] processed_impl_today: Set of implementation IDs already processed today
948
+ :return: Number of assessments created (0 or 1)
949
+ :rtype: int
950
+ """
951
+ try:
952
+ logger.debug(f"Processing control assessment for '{control_id}'")
953
+ impl, sec_control = self._find_matching_implementation(control_id, implementations)
954
+ if not impl or not sec_control:
955
+ self._log_no_match(control_id, implementations)
956
+ return 0
957
+
958
+ result = self._determine_overall_result(control_id)
959
+ items = self._get_control_compliance_items(control_id)
960
+ logger.debug(f"Control '{control_id}' assessment: {result} (based on {len(items)} policy assessments)")
961
+
962
+ if impl.id in processed_impl_today and self._find_existing_assessment_cached(impl.id, self.scan_date):
963
+ logger.debug(f"Skipping duplicate assessment for implementation {impl.id} (already processed today)")
964
+ else:
965
+ self._create_control_assessment(
966
+ implementation=impl,
967
+ catalog_control={"id": sec_control.id, "controlId": sec_control.controlId},
968
+ result=result,
969
+ control_id=control_id,
970
+ compliance_items=items,
971
+ )
972
+ processed_impl_today.add(impl.id)
973
+
974
+ self._record_control_mapping(control_id, impl.id)
975
+ self._map_assets_to_control_component(sec_control, items)
976
+ return 1
977
+ except Exception as e: # noqa: BLE001
978
+ logger.error(f"Error processing control assessment for '{control_id}': {e}")
979
+ import traceback
980
+
981
+ logger.debug(traceback.format_exc())
982
+ return 0
983
+
984
+ def _find_matching_implementation(
985
+ self, control_id: str, implementations: List[ControlImplementation]
986
+ ) -> tuple[Optional[ControlImplementation], Optional[SecurityControl]]:
987
+ """
988
+ Find matching implementation and security control for a control ID.
989
+
990
+ :param str control_id: Control identifier to match
991
+ :param List[ControlImplementation] implementations: Available implementations
992
+ :return: Tuple of matching implementation and security control, or (None, None)
993
+ :rtype: tuple[Optional[ControlImplementation], Optional[SecurityControl]]
994
+ """
995
+ matching_implementation = None
996
+ matching_security_control = None
997
+ for implementation in implementations:
998
+ try:
999
+ security_control = SecurityControl.get_object(object_id=implementation.controlID)
1000
+ if not security_control:
1001
+ logger.debug(
1002
+ f"No security control found for implementation {implementation.id} with controlID: {implementation.controlID}"
1003
+ )
1004
+ continue
1005
+ security_control_id = security_control.controlId
1006
+ if not security_control_id:
1007
+ logger.debug(f"Security control {security_control.id} has no controlId")
1008
+ continue
1009
+ logger.debug(
1010
+ f"Comparing extracted '{control_id}' with RegScale control '{security_control_id}' (impl: {implementation.id})"
1011
+ )
1012
+ if self._control_ids_match(control_id, security_control_id):
1013
+ matching_implementation = implementation
1014
+ matching_security_control = security_control
1015
+ logger.info(
1016
+ f"✅ MATCH FOUND: '{security_control_id}' == '{control_id}' (implementation: {implementation.id})"
1017
+ )
1018
+ break
1019
+ except Exception as e: # noqa: BLE001
1020
+ logger.error(
1021
+ f"Error processing implementation {implementation.id} with controlID {implementation.controlID}: {e}"
1022
+ )
1023
+ continue
1024
+ return matching_implementation, matching_security_control
1025
+
1026
+ def _log_no_match(self, control_id: str, implementations: List[ControlImplementation]) -> None:
1027
+ """
1028
+ Log when no matching implementation is found for a control.
1029
+
1030
+ :param str control_id: Control identifier that couldn't be matched
1031
+ :param List[ControlImplementation] implementations: Available implementations for context
1032
+ :return: None
1033
+ :rtype: None
1034
+ """
1035
+ logger.warning(f"No matching implementation found for control ID '{control_id}'")
1036
+ sample_impl_controls = []
1037
+ for impl in implementations[:5]:
1038
+ try:
1039
+ sec_control = SecurityControl.get_object(object_id=impl.controlID)
1040
+ if sec_control and sec_control.controlId:
1041
+ sample_impl_controls.append(f"{sec_control.controlId} (impl:{impl.id})")
1042
+ except Exception:
1043
+ sample_impl_controls.append(f"Unknown (impl:{impl.id})")
1044
+ logger.debug(f"Sample implementation control IDs (first 5): {sample_impl_controls}")
1045
+
1046
+ def _determine_overall_result(self, control_id: str) -> str:
1047
+ """
1048
+ Determine overall assessment result for a control.
1049
+
1050
+ :param str control_id: Control identifier to check
1051
+ :return: Assessment result ('Pass' or 'Fail')
1052
+ :rtype: str
1053
+ """
1054
+ is_failing = (
1055
+ control_id in self.failing_controls
1056
+ or control_id.lower() in self.failing_controls
1057
+ or control_id.upper() in self.failing_controls
1058
+ )
1059
+ return "Fail" if is_failing else "Pass"
1060
+
1061
+ def _get_control_compliance_items(self, control_id: str) -> List[ComplianceItem]:
1062
+ """
1063
+ Get all compliance items for a specific control.
1064
+
1065
+ :param str control_id: Control identifier to filter by
1066
+ :return: List of compliance items for the control
1067
+ :rtype: List[ComplianceItem]
1068
+ """
1069
+ items: List[ComplianceItem] = []
1070
+ for item in self.all_compliance_items:
1071
+ if hasattr(item, "control_ids"):
1072
+ item_control_ids = getattr(item, "control_ids", [])
1073
+ if any(cid.lower() == control_id.lower() for cid in item_control_ids):
1074
+ items.append(item)
1075
+ elif hasattr(item, "control_id") and item.control_id.lower() == control_id.lower():
1076
+ items.append(item)
1077
+ return items
1078
+
1079
+ def _record_control_mapping(self, control_id: str, implementation_id: int) -> None:
1080
+ """
1081
+ Record mapping between normalized control ID and implementation ID.
1082
+
1083
+ :param str control_id: Control identifier to map
1084
+ :param int implementation_id: Implementation ID to associate
1085
+ :return: None
1086
+ :rtype: None
1087
+ """
1088
+ try:
1089
+ base, sub = self._normalize_control_id(control_id)
1090
+ canonical = f"{base}({sub})" if sub else base
1091
+ self._impl_id_by_control[canonical] = implementation_id
1092
+ logger.debug(f"Mapped control '{canonical}' -> implementation ID {implementation_id}")
1093
+ except Exception:
1094
+ pass
1095
+
1096
+ def _map_assets_to_control_component(self, sec_control: SecurityControl, items: List[ComplianceItem]) -> None:
1097
+ """
1098
+ Map assets to control-specific components for organization.
1099
+
1100
+ :param SecurityControl sec_control: Security control to create component for
1101
+ :param List[ComplianceItem] items: Compliance items with asset references
1102
+ :return: None
1103
+ :rtype: None
1104
+ """
1105
+ try:
1106
+ component_title = f"Control {sec_control.controlId}"
1107
+ component = self.components_by_title.get(component_title) if hasattr(self, "components_by_title") else None
1108
+ if not component:
1109
+ component = regscale_models.Component(
1110
+ title=component_title,
1111
+ componentType=regscale_models.ComponentType.Hardware,
1112
+ securityPlansId=self.plan_id,
1113
+ description=component_title,
1114
+ componentOwnerId=self.get_assessor_id(),
1115
+ ).get_or_create()
1116
+ regscale_models.ComponentMapping(
1117
+ componentId=component.id,
1118
+ securityPlanId=self.plan_id,
1119
+ ).get_or_create()
1120
+ if hasattr(self, "components_by_title"):
1121
+ self.components_by_title[component_title] = component
1122
+
1123
+ for item in items:
1124
+ asset = self.get_asset_by_identifier(getattr(item, "resource_id", ""))
1125
+ if not asset:
1126
+ continue
1127
+ regscale_models.AssetMapping(
1128
+ assetId=asset.id,
1129
+ componentId=component.id,
1130
+ ).get_or_create_with_status()
1131
+ except Exception as map_exc: # noqa: BLE001
1132
+ logger.debug(f"Control-to-asset mapping skipped due to: {map_exc}")
1133
+
1134
+ @staticmethod
1135
+ def _parse_control_id(control_id: str) -> tuple[str, Optional[str]]:
1136
+ """
1137
+ Parse a control id like 'AC-2(1)', 'AC-2 (1)', 'AC-2-1' into (base, sub).
1138
+
1139
+ Returns (base, None) when no subcontrol.
1140
+
1141
+ :param str control_id: Control identifier to parse
1142
+ :return: Tuple of (base_control, subcontrol) where subcontrol may be None
1143
+ :rtype: tuple[str, Optional[str]]
1144
+ """
1145
+ cid = control_id.strip()
1146
+ # Use precompiled safe regex to avoid catastrophic backtracking on crafted input
1147
+ m = SAFE_CONTROL_ID_RE.match(cid)
1148
+ if not m:
1149
+ return cid.upper(), None
1150
+ base = m.group(1).upper()
1151
+ # Subcontrol may be captured in group 2, 3, or 4 depending on the branch matched
1152
+ sub = m.group(2) or m.group(3) or m.group(4)
1153
+ return base, sub
1154
+
1155
+ @classmethod
1156
+ def _control_ids_match(cls, a: str, b: str) -> bool:
1157
+ """
1158
+ Strict match of control ids. Exact match if equal.
1159
+ If subcontrols exist on either side, both must exist and be equal.
1160
+
1161
+ :param str a: First control ID to compare
1162
+ :param str b: Second control ID to compare
1163
+ :return: True if control IDs match according to strict rules
1164
+ :rtype: bool
1165
+ """
1166
+ if not a or not b:
1167
+ return False
1168
+ if a.strip().lower() == b.strip().lower():
1169
+ return True
1170
+ base_a, sub_a = cls._parse_control_id(a)
1171
+ base_b, sub_b = cls._parse_control_id(b)
1172
+ if base_a != base_b:
1173
+ return False
1174
+ # If either has a subcontrol, require both and equality
1175
+ if sub_a or sub_b:
1176
+ return (sub_a is not None) and (sub_b is not None) and (sub_a == sub_b)
1177
+ # No subcontrols -> base equals
1178
+ return True
1179
+
1180
+ @staticmethod
1181
+ def _normalize_control_id(control_id: str) -> tuple[str, Optional[str]]:
1182
+ """
1183
+ Normalize control id to a canonical tuple (BASE, SUB) for set membership.
1184
+
1185
+ :param str control_id: Control identifier to normalize
1186
+ :return: Tuple of (base_control, subcontrol) in canonical form
1187
+ :rtype: tuple[str, Optional[str]]
1188
+ """
1189
+ cid = (control_id or "").strip()
1190
+ # Use precompiled safe regex to avoid catastrophic backtracking on crafted input
1191
+ m = SAFE_CONTROL_ID_RE.match(cid)
1192
+ if not m:
1193
+ return cid.upper(), None
1194
+ base = m.group(1).upper()
1195
+ sub = m.group(2) or m.group(3) or m.group(4)
1196
+ return base, sub
1197
+
1198
+ def _create_control_assessment(
1199
+ self,
1200
+ implementation: ControlImplementation,
1201
+ catalog_control: Dict,
1202
+ result: str,
1203
+ control_id: str,
1204
+ compliance_items: List[ComplianceItem] = None,
1205
+ ) -> None:
1206
+ """
1207
+ Create or update an assessment for a control implementation.
1208
+ If an assessment for the same day exists, update it instead of creating a duplicate.
1209
+
1210
+ :param ControlImplementation implementation: The control implementation to assess
1211
+ :param Dict catalog_control: The catalog control data dictionary
1212
+ :param str result: Assessment result ('Pass' or 'Fail')
1213
+ :param str control_id: Control identifier string
1214
+ :param List[ComplianceItem] compliance_items: Pre-aggregated compliance items for this control
1215
+ :return: None
1216
+ :rtype: None
1217
+ """
1218
+ try:
1219
+ # Use provided compliance items or get them for this control (backward compatibility)
1220
+ if compliance_items is None:
1221
+ compliance_items = []
1222
+ if (
1223
+ control_id in self.failing_controls
1224
+ or control_id.lower() in self.failing_controls
1225
+ or control_id.upper() in self.failing_controls
1226
+ ):
1227
+ compliance_items = [
1228
+ item for item in self.failed_compliance_items if item.control_id.lower() == control_id.lower()
1229
+ ]
1230
+ else:
1231
+ compliance_items = [
1232
+ item for item in self.all_compliance_items if item.control_id.lower() == control_id.lower()
1233
+ ]
1234
+
1235
+ # Create assessment report
1236
+ assessment_report = self._create_assessment_report(control_id, result, compliance_items)
1237
+
1238
+ # Check for existing assessment on the same day using cache
1239
+ existing_assessment = self._find_existing_assessment_cached(implementation.id, self.scan_date)
1240
+
1241
+ if existing_assessment:
1242
+ # Update existing assessment
1243
+ existing_assessment.assessmentResult = result
1244
+ existing_assessment.assessmentReport = assessment_report
1245
+ existing_assessment.actualFinish = get_current_datetime()
1246
+ existing_assessment.dateLastUpdated = get_current_datetime()
1247
+ existing_assessment.save()
1248
+ logger.debug(f"Updated existing assessment {existing_assessment.id} for control {control_id}")
1249
+ # Refresh cache for today
1250
+ try:
1251
+ day_key = regscale_string_to_datetime(self.scan_date).date().isoformat()
1252
+ except Exception:
1253
+ day_key = str(self.scan_date).split(" ")[0]
1254
+ self._existing_assessments_cache[f"{implementation.id}_{day_key}"] = existing_assessment
1255
+ # Track today's assessment by implementation id for linking to issues later
1256
+ try:
1257
+ self._assessment_by_impl_today[implementation.id] = existing_assessment
1258
+ except Exception:
1259
+ pass
1260
+ else:
1261
+ # Create new assessment
1262
+ assessment = Assessment(
1263
+ leadAssessorId=implementation.createdById,
1264
+ title=f"{self.title} compliance assessment for {control_id.upper()}",
1265
+ assessmentType="Control Testing",
1266
+ plannedStart=get_current_datetime(),
1267
+ plannedFinish=get_current_datetime(),
1268
+ actualFinish=get_current_datetime(),
1269
+ assessmentResult=result,
1270
+ assessmentReport=assessment_report,
1271
+ status="Complete",
1272
+ parentId=implementation.id,
1273
+ parentModule="controls",
1274
+ isPublic=True,
1275
+ ).create()
1276
+ logger.debug(f"Created new assessment {assessment.id} for control {control_id}")
1277
+ # Add to cache for today to prevent duplicate creation in subsequent processing
1278
+ try:
1279
+ day_key = regscale_string_to_datetime(self.scan_date).date().isoformat()
1280
+ except Exception:
1281
+ day_key = str(self.scan_date).split(" ")[0]
1282
+ self._existing_assessments_cache[f"{implementation.id}_{day_key}"] = assessment
1283
+ # Track today's assessment by implementation id for linking to issues later
1284
+ try:
1285
+ self._assessment_by_impl_today[implementation.id] = assessment
1286
+ except Exception:
1287
+ pass
1288
+
1289
+ # Update implementation status if needed
1290
+ if self.update_control_status:
1291
+ self._update_implementation_status(implementation, result)
1292
+
1293
+ except Exception as e:
1294
+ logger.error(f"Error creating control assessment: {e}")
1295
+
1296
+ def _find_existing_assessment(self, implementation: ControlImplementation, scan_date) -> Optional:
1297
+ """
1298
+ Find existing assessment for the same implementation on the same day.
1299
+ DEPRECATED: Use _find_existing_assessment_cached instead.
1300
+
1301
+ :param ControlImplementation implementation: The control implementation
1302
+ :param scan_date: The scan date to check
1303
+ :return: Existing assessment or None
1304
+ :rtype: Optional[regscale_models.Assessment]
1305
+ """
1306
+ logger.warning("_find_existing_assessment is deprecated, use _find_existing_assessment_cached")
1307
+ return self._find_existing_assessment_cached(implementation.id, scan_date)
1308
+
1309
+ def _create_assessment_report(self, control_id: str, result: str, compliance_items: List[ComplianceItem]) -> str:
1310
+ """
1311
+ Create HTML assessment report.
1312
+
1313
+ :param str control_id: Control identifier
1314
+ :param str result: Assessment result ('Pass' or 'Fail')
1315
+ :param List[ComplianceItem] compliance_items: Compliance items for this control
1316
+ :return: Formatted HTML report string
1317
+ :rtype: str
1318
+ """
1319
+ result_color = "#d32f2f" if result == "Fail" else "#2e7d32"
1320
+
1321
+ html_parts = [
1322
+ f"""
1323
+ <div style="margin-bottom: 20px; padding: 15px; border: 2px solid {result_color};
1324
+ border-radius: 5px; background-color: {'#ffebee' if result == 'Fail' else '#e8f5e8'};">
1325
+ <h3 style="margin: 0 0 10px 0; color: {result_color};">
1326
+ {self.title} Compliance Assessment for Control {control_id.upper()}
1327
+ </h3>
1328
+ <p><strong>Overall Result:</strong>
1329
+ <span style="color: {result_color}; font-weight: bold;">{result}</span></p>
1330
+ <p><strong>Assessment Date:</strong> {self.scan_date}</p>
1331
+ <p><strong>Framework:</strong> {self.framework}</p>
1332
+ <p><strong>Total Policy Assessments:</strong> {len(compliance_items)}</p>
1333
+ </div>
1334
+ """
1335
+ ] # NOSONAR
1336
+
1337
+ if compliance_items:
1338
+ # Group by result for summary
1339
+ pass_count = len([item for item in compliance_items if item.compliance_result in self.PASS_STATUSES])
1340
+ fail_count = len(compliance_items) - pass_count
1341
+
1342
+ # Count unique resources across all policy assessments for this control
1343
+ unique_resources = set()
1344
+ unique_policies = set()
1345
+
1346
+ for item in compliance_items:
1347
+ unique_resources.add(item.resource_id)
1348
+ # Get policy name for aggregation
1349
+ if hasattr(item, "description"):
1350
+ unique_policies.add(
1351
+ item.description[:50] + "..." if len(item.description) > 50 else item.description
1352
+ )
1353
+ elif hasattr(item, "policy") and isinstance(item.policy, dict):
1354
+ policy_name = item.policy.get("name", "Unknown Policy")
1355
+ unique_policies.add(policy_name[:50] + "..." if len(policy_name) > 50 else policy_name)
1356
+
1357
+ html_parts.append(
1358
+ f"""
1359
+ <div style="margin-top: 20px;">
1360
+ <h4>Aggregated Assessment Summary</h4>
1361
+ <p><strong>Policy Assessments:</strong> {len(compliance_items)} total</p>
1362
+ <p><strong>Unique Policies Tested:</strong> {len(unique_policies)}</p>
1363
+ <p><strong>Unique Resources Assessed:</strong> {len(unique_resources)}</p>
1364
+ <p><strong>Passing Assessments:</strong> <span style="color: #2e7d32;">{pass_count}</span></p>
1365
+ <p><strong>Failing Assessments:</strong> <span style="color: #d32f2f;">{fail_count}</span></p>
1366
+ <p><strong>Overall Control Result:</strong> <span style="color: {result_color}; font-weight: bold;">{result}</span></p>
1367
+ </div>
1368
+ """
1369
+ )
1370
+
1371
+ return "\n".join(html_parts)
1372
+
1373
+ def _update_implementation_status(self, implementation: ControlImplementation, result: str) -> None:
1374
+ """
1375
+ Update control implementation status based on assessment result.
1376
+
1377
+ :param ControlImplementation implementation: Control implementation to update
1378
+ :param str result: Assessment result ('Pass' or 'Fail')
1379
+ :return: None
1380
+ :rtype: None
1381
+ """
1382
+ try:
1383
+ if result == "Pass":
1384
+ new_status = "Fully Implemented"
1385
+ else:
1386
+ new_status = "In Remediation"
1387
+
1388
+ # Update implementation status
1389
+ implementation.status = new_status
1390
+ implementation.dateLastAssessed = get_current_datetime()
1391
+ implementation.lastAssessmentResult = result
1392
+ implementation.save()
1393
+
1394
+ # Update objectives if they exist
1395
+ objectives = ImplementationObjective.get_all_by_parent(
1396
+ parent_module=implementation.get_module_slug(),
1397
+ parent_id=implementation.id,
1398
+ )
1399
+
1400
+ for objective in objectives:
1401
+ objective.status = new_status
1402
+ objective.save()
1403
+
1404
+ logger.debug(f"Updated implementation status to {new_status}")
1405
+
1406
+ except Exception as e:
1407
+ logger.error(f"Error updating implementation status: {e}")
1408
+
1409
+ def _get_controls(self) -> List[Dict]:
1410
+ """
1411
+ Get controls from catalog or plan.
1412
+
1413
+ :return: List of control dictionaries from catalog or plan
1414
+ :rtype: List[Dict]
1415
+ """
1416
+ if self.catalog_id:
1417
+ catalog = Catalog.get_with_all_details(catalog_id=self.catalog_id)
1418
+ return catalog.get("controls", []) if catalog else []
1419
+ else:
1420
+ return SecurityControl.get_controls_by_parent_id_and_module(
1421
+ parent_module=self.parent_module, parent_id=self.plan_id, return_dicts=True
1422
+ )
1423
+
1424
+ def _find_existing_asset_by_resource_id(self, resource_id: str) -> Optional[regscale_models.Asset]:
1425
+ """
1426
+ Find existing asset by resource ID.
1427
+
1428
+ :param str resource_id: Resource identifier to search for
1429
+ :return: Existing asset or None if not found
1430
+ :rtype: Optional[regscale_models.Asset]
1431
+ """
1432
+ try:
1433
+ if hasattr(self, "asset_map_by_identifier") and self.asset_map_by_identifier:
1434
+ return self.asset_map_by_identifier.get(resource_id)
1435
+
1436
+ # Query database
1437
+ existing_assets = regscale_models.Asset.get_all_by_parent(
1438
+ parent_id=self.plan_id,
1439
+ parent_module=self.parent_module,
1440
+ )
1441
+
1442
+ for asset in existing_assets:
1443
+ if hasattr(asset, "otherTrackingNumber") and asset.otherTrackingNumber == resource_id:
1444
+ return asset
1445
+
1446
+ return None
1447
+
1448
+ except Exception as e:
1449
+ logger.error(f"Error finding existing asset: {e}")
1450
+ return None
1451
+
1452
+ def _map_resource_type_to_asset_type(self, compliance_item: ComplianceItem) -> str:
1453
+ """
1454
+ Map compliance item resource type to RegScale asset type.
1455
+
1456
+ :param ComplianceItem compliance_item: Compliance item with resource type information
1457
+ :return: Asset type string suitable for RegScale
1458
+ :rtype: str
1459
+ """
1460
+ # Default implementation - can be overridden by subclasses
1461
+ return "Cloud Resource"
1462
+
1463
+ def _map_severity(self, severity: Optional[str]) -> regscale_models.IssueSeverity:
1464
+ """
1465
+ Map compliance severity to RegScale severity.
1466
+
1467
+ :param Optional[str] severity: Severity string from compliance source
1468
+ :return: Mapped RegScale severity enum value
1469
+ :rtype: regscale_models.IssueSeverity
1470
+ """
1471
+ if not severity:
1472
+ return regscale_models.IssueSeverity.Moderate
1473
+
1474
+ severity_mapping = {
1475
+ "CRITICAL": regscale_models.IssueSeverity.Critical,
1476
+ "HIGH": regscale_models.IssueSeverity.High,
1477
+ "MEDIUM": regscale_models.IssueSeverity.Moderate,
1478
+ "LOW": regscale_models.IssueSeverity.Low,
1479
+ }
1480
+
1481
+ return severity_mapping.get(severity.upper(), regscale_models.IssueSeverity.Moderate)
1482
+
1483
+ def _map_severity_to_priority(self, severity: regscale_models.IssueSeverity) -> str:
1484
+ """
1485
+ Map severity to priority string.
1486
+
1487
+ :param regscale_models.IssueSeverity severity: Issue severity enum value
1488
+ :return: Priority string for issues
1489
+ :rtype: str
1490
+ """
1491
+ priority_mapping = {
1492
+ regscale_models.IssueSeverity.Critical: "Critical",
1493
+ regscale_models.IssueSeverity.High: "High",
1494
+ regscale_models.IssueSeverity.Moderate: "Medium",
1495
+ regscale_models.IssueSeverity.Low: "Low",
1496
+ }
1497
+
1498
+ return priority_mapping.get(severity, "Medium")
1499
+
1500
+ def _update_scan_history(self, scan_history: regscale_models.ScanHistory) -> None:
1501
+ """
1502
+ Update scan history with results.
1503
+
1504
+ :param regscale_models.ScanHistory scan_history: Scan history record to update
1505
+ :return: None
1506
+ :rtype: None
1507
+ """
1508
+ try:
1509
+ scan_history.dateLastUpdated = get_current_datetime()
1510
+ scan_history.save()
1511
+ logger.debug(f"Updated scan history {scan_history.id}")
1512
+ except Exception as e:
1513
+ logger.error(f"Error updating scan history: {e}")
1514
+
1515
+ def create_scan_history(self) -> regscale_models.ScanHistory:
1516
+ """
1517
+ Create or reuse a ScanHistory for the same day and tool.
1518
+
1519
+ If a scan history exists for this plan/module with the same
1520
+ scanning tool and scan date (day-level), update and reuse it
1521
+ instead of creating a duplicate.
1522
+
1523
+ :return: Created or reused scan history record
1524
+ :rtype: regscale_models.ScanHistory
1525
+ """
1526
+ try:
1527
+ # Load existing scans for the plan/module
1528
+ existing_scans = regscale_models.ScanHistory.get_all_by_parent(
1529
+ parent_id=self.plan_id, parent_module=self.parent_module
1530
+ )
1531
+
1532
+ # Normalize target date to date component only
1533
+ target_dt = self.scan_date
1534
+ target_date_only = target_dt.split("T")[0] if isinstance(target_dt, str) else str(target_dt)[:10]
1535
+
1536
+ # Find an existing scan for today and this tool
1537
+ for scan in existing_scans:
1538
+ try:
1539
+ if getattr(scan, "scanningTool", None) == self.title and getattr(scan, "scanDate", None):
1540
+ scan_date = str(scan.scanDate)
1541
+ scan_date_only = scan_date.split("T")[0]
1542
+ if scan_date_only == target_date_only:
1543
+ # Reuse this scan history; refresh last updated
1544
+ scan.dateLastUpdated = get_current_datetime()
1545
+ scan.save()
1546
+ return scan
1547
+ except Exception:
1548
+ # Skip any malformed scan records
1549
+ continue
1550
+
1551
+ # No existing same-day scan found, create new via base behavior
1552
+ return super().create_scan_history()
1553
+ except Exception:
1554
+ # Fallback: create new scan history
1555
+ return super().create_scan_history()
1556
+
1557
+ def create_or_update_issue_from_finding(self, title: str, finding: IntegrationFinding) -> regscale_models.Issue:
1558
+ """
1559
+ Create or update an issue from a finding, using cache to prevent duplicates.
1560
+
1561
+ :param str title: Issue title
1562
+ :param IntegrationFinding finding: The finding to create issue from
1563
+ :return: Created or updated issue
1564
+ :rtype: regscale_models.Issue
1565
+ """
1566
+ # Load cache if not already loaded
1567
+ self._load_existing_records_cache()
1568
+
1569
+ # Check for existing issue by external_id first
1570
+ external_id = finding.external_id
1571
+ existing_issue = self._find_existing_issue_cached(external_id)
1572
+
1573
+ if existing_issue:
1574
+ logger.debug(
1575
+ f"Found existing issue {existing_issue.id} for external_id {external_id}, updating instead of creating"
1576
+ )
1577
+
1578
+ # Update existing issue with new finding data
1579
+ existing_issue.title = title
1580
+ existing_issue.description = finding.description
1581
+ existing_issue.severity = finding.severity
1582
+ existing_issue.status = finding.status
1583
+ # Ensure affectedControls is updated from the finding's control id
1584
+ try:
1585
+ if getattr(finding, "control_labels", None):
1586
+ existing_issue.affectedControls = ",".join(finding.control_labels)
1587
+ else:
1588
+ # Fall back to normalized control id from rule_id/control_labels
1589
+ ctl = None
1590
+ if getattr(finding, "rule_id", None):
1591
+ ctl = finding.rule_id
1592
+ elif getattr(finding, "control_labels", None):
1593
+ labels = list(finding.control_labels)
1594
+ ctl = labels[0] if labels else None
1595
+ if ctl:
1596
+ base, sub = self._normalize_control_id(ctl)
1597
+ existing_issue.affectedControls = f"{base}({sub})" if sub else base
1598
+ except Exception:
1599
+ pass
1600
+ existing_issue.dateLastUpdated = self.scan_date
1601
+ existing_issue.save()
1602
+
1603
+ return existing_issue
1604
+ else:
1605
+ # No existing issue found, create new one using parent method
1606
+ logger.debug(f"No existing issue found for external_id {external_id}, creating new issue")
1607
+ return super().create_or_update_issue_from_finding(title, finding)