regscale-cli 6.27.3.0__py3-none-any.whl → 6.28.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of regscale-cli might be problematic. Click here for more details.

Files changed (112) hide show
  1. regscale/_version.py +1 -1
  2. regscale/core/app/utils/app_utils.py +11 -2
  3. regscale/dev/cli.py +26 -0
  4. regscale/dev/version.py +72 -0
  5. regscale/integrations/commercial/__init__.py +15 -1
  6. regscale/integrations/commercial/amazon/amazon/__init__.py +0 -0
  7. regscale/integrations/commercial/amazon/amazon/common.py +204 -0
  8. regscale/integrations/commercial/amazon/common.py +48 -58
  9. regscale/integrations/commercial/aws/audit_manager_compliance.py +2671 -0
  10. regscale/integrations/commercial/aws/cli.py +3093 -55
  11. regscale/integrations/commercial/aws/cloudtrail_control_mappings.py +333 -0
  12. regscale/integrations/commercial/aws/cloudtrail_evidence.py +501 -0
  13. regscale/integrations/commercial/aws/cloudwatch_control_mappings.py +357 -0
  14. regscale/integrations/commercial/aws/cloudwatch_evidence.py +490 -0
  15. regscale/integrations/commercial/aws/config_compliance.py +914 -0
  16. regscale/integrations/commercial/aws/conformance_pack_mappings.py +198 -0
  17. regscale/integrations/commercial/aws/evidence_generator.py +283 -0
  18. regscale/integrations/commercial/aws/guardduty_control_mappings.py +340 -0
  19. regscale/integrations/commercial/aws/guardduty_evidence.py +1053 -0
  20. regscale/integrations/commercial/aws/iam_control_mappings.py +368 -0
  21. regscale/integrations/commercial/aws/iam_evidence.py +574 -0
  22. regscale/integrations/commercial/aws/inventory/__init__.py +223 -22
  23. regscale/integrations/commercial/aws/inventory/base.py +107 -5
  24. regscale/integrations/commercial/aws/inventory/resources/audit_manager.py +513 -0
  25. regscale/integrations/commercial/aws/inventory/resources/cloudtrail.py +315 -0
  26. regscale/integrations/commercial/aws/inventory/resources/cloudtrail_logs_metadata.py +476 -0
  27. regscale/integrations/commercial/aws/inventory/resources/cloudwatch.py +191 -0
  28. regscale/integrations/commercial/aws/inventory/resources/compute.py +66 -9
  29. regscale/integrations/commercial/aws/inventory/resources/config.py +464 -0
  30. regscale/integrations/commercial/aws/inventory/resources/containers.py +74 -9
  31. regscale/integrations/commercial/aws/inventory/resources/database.py +106 -31
  32. regscale/integrations/commercial/aws/inventory/resources/guardduty.py +286 -0
  33. regscale/integrations/commercial/aws/inventory/resources/iam.py +470 -0
  34. regscale/integrations/commercial/aws/inventory/resources/inspector.py +476 -0
  35. regscale/integrations/commercial/aws/inventory/resources/integration.py +175 -61
  36. regscale/integrations/commercial/aws/inventory/resources/kms.py +447 -0
  37. regscale/integrations/commercial/aws/inventory/resources/networking.py +103 -67
  38. regscale/integrations/commercial/aws/inventory/resources/s3.py +394 -0
  39. regscale/integrations/commercial/aws/inventory/resources/security.py +268 -72
  40. regscale/integrations/commercial/aws/inventory/resources/securityhub.py +473 -0
  41. regscale/integrations/commercial/aws/inventory/resources/storage.py +53 -29
  42. regscale/integrations/commercial/aws/inventory/resources/systems_manager.py +657 -0
  43. regscale/integrations/commercial/aws/inventory/resources/vpc.py +655 -0
  44. regscale/integrations/commercial/aws/kms_control_mappings.py +288 -0
  45. regscale/integrations/commercial/aws/kms_evidence.py +879 -0
  46. regscale/integrations/commercial/aws/ocsf/__init__.py +7 -0
  47. regscale/integrations/commercial/aws/ocsf/constants.py +115 -0
  48. regscale/integrations/commercial/aws/ocsf/mapper.py +435 -0
  49. regscale/integrations/commercial/aws/org_control_mappings.py +286 -0
  50. regscale/integrations/commercial/aws/org_evidence.py +666 -0
  51. regscale/integrations/commercial/aws/s3_control_mappings.py +356 -0
  52. regscale/integrations/commercial/aws/s3_evidence.py +632 -0
  53. regscale/integrations/commercial/aws/scanner.py +851 -206
  54. regscale/integrations/commercial/aws/security_hub.py +319 -0
  55. regscale/integrations/commercial/aws/session_manager.py +282 -0
  56. regscale/integrations/commercial/aws/ssm_control_mappings.py +291 -0
  57. regscale/integrations/commercial/aws/ssm_evidence.py +492 -0
  58. regscale/integrations/compliance_integration.py +308 -38
  59. regscale/integrations/due_date_handler.py +3 -0
  60. regscale/integrations/scanner_integration.py +399 -84
  61. regscale/models/integration_models/cisa_kev_data.json +34 -4
  62. regscale/models/integration_models/synqly_models/capabilities.json +1 -1
  63. regscale/models/integration_models/synqly_models/connectors/vulnerabilities.py +17 -9
  64. regscale/models/regscale_models/assessment.py +2 -1
  65. regscale/models/regscale_models/control_objective.py +74 -5
  66. regscale/models/regscale_models/file.py +2 -0
  67. regscale/models/regscale_models/issue.py +2 -5
  68. {regscale_cli-6.27.3.0.dist-info → regscale_cli-6.28.0.0.dist-info}/METADATA +1 -1
  69. {regscale_cli-6.27.3.0.dist-info → regscale_cli-6.28.0.0.dist-info}/RECORD +112 -33
  70. tests/regscale/integrations/commercial/aws/__init__.py +0 -0
  71. tests/regscale/integrations/commercial/aws/test_audit_manager_compliance.py +1304 -0
  72. tests/regscale/integrations/commercial/aws/test_audit_manager_evidence_aggregation.py +341 -0
  73. tests/regscale/integrations/commercial/aws/test_aws_audit_manager_collector.py +1155 -0
  74. tests/regscale/integrations/commercial/aws/test_aws_cloudtrail_collector.py +534 -0
  75. tests/regscale/integrations/commercial/aws/test_aws_config_collector.py +400 -0
  76. tests/regscale/integrations/commercial/aws/test_aws_guardduty_collector.py +315 -0
  77. tests/regscale/integrations/commercial/aws/test_aws_iam_collector.py +458 -0
  78. tests/regscale/integrations/commercial/aws/test_aws_inspector_collector.py +353 -0
  79. tests/regscale/integrations/commercial/aws/test_aws_inventory_integration.py +530 -0
  80. tests/regscale/integrations/commercial/aws/test_aws_kms_collector.py +919 -0
  81. tests/regscale/integrations/commercial/aws/test_aws_s3_collector.py +722 -0
  82. tests/regscale/integrations/commercial/aws/test_aws_scanner_integration.py +722 -0
  83. tests/regscale/integrations/commercial/aws/test_aws_securityhub_collector.py +792 -0
  84. tests/regscale/integrations/commercial/aws/test_aws_systems_manager_collector.py +918 -0
  85. tests/regscale/integrations/commercial/aws/test_aws_vpc_collector.py +996 -0
  86. tests/regscale/integrations/commercial/aws/test_cli_evidence.py +431 -0
  87. tests/regscale/integrations/commercial/aws/test_cloudtrail_control_mappings.py +452 -0
  88. tests/regscale/integrations/commercial/aws/test_cloudtrail_evidence.py +788 -0
  89. tests/regscale/integrations/commercial/aws/test_config_compliance.py +298 -0
  90. tests/regscale/integrations/commercial/aws/test_conformance_pack_mappings.py +200 -0
  91. tests/regscale/integrations/commercial/aws/test_evidence_generator.py +386 -0
  92. tests/regscale/integrations/commercial/aws/test_guardduty_control_mappings.py +564 -0
  93. tests/regscale/integrations/commercial/aws/test_guardduty_evidence.py +1041 -0
  94. tests/regscale/integrations/commercial/aws/test_iam_control_mappings.py +718 -0
  95. tests/regscale/integrations/commercial/aws/test_iam_evidence.py +1375 -0
  96. tests/regscale/integrations/commercial/aws/test_kms_control_mappings.py +656 -0
  97. tests/regscale/integrations/commercial/aws/test_kms_evidence.py +1163 -0
  98. tests/regscale/integrations/commercial/aws/test_ocsf_mapper.py +370 -0
  99. tests/regscale/integrations/commercial/aws/test_org_control_mappings.py +546 -0
  100. tests/regscale/integrations/commercial/aws/test_org_evidence.py +1240 -0
  101. tests/regscale/integrations/commercial/aws/test_s3_control_mappings.py +672 -0
  102. tests/regscale/integrations/commercial/aws/test_s3_evidence.py +987 -0
  103. tests/regscale/integrations/commercial/aws/test_scanner_evidence.py +373 -0
  104. tests/regscale/integrations/commercial/aws/test_security_hub_config_filtering.py +539 -0
  105. tests/regscale/integrations/commercial/aws/test_session_manager.py +516 -0
  106. tests/regscale/integrations/commercial/aws/test_ssm_control_mappings.py +588 -0
  107. tests/regscale/integrations/commercial/aws/test_ssm_evidence.py +735 -0
  108. tests/regscale/integrations/commercial/test_aws.py +55 -56
  109. {regscale_cli-6.27.3.0.dist-info → regscale_cli-6.28.0.0.dist-info}/LICENSE +0 -0
  110. {regscale_cli-6.27.3.0.dist-info → regscale_cli-6.28.0.0.dist-info}/WHEEL +0 -0
  111. {regscale_cli-6.27.3.0.dist-info → regscale_cli-6.28.0.0.dist-info}/entry_points.txt +0 -0
  112. {regscale_cli-6.27.3.0.dist-info → regscale_cli-6.28.0.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,666 @@
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ """AWS Organizations Evidence Integration for RegScale CLI."""
4
+
5
+ import gzip
6
+ import json
7
+ import logging
8
+ import os
9
+ import time
10
+ from datetime import datetime, timedelta
11
+ from io import BytesIO
12
+ from typing import Any, Dict, List, Optional
13
+
14
+ import boto3
15
+ from botocore.exceptions import ClientError
16
+
17
+ from regscale.core.app.api import Api
18
+ from regscale.core.app.utils.app_utils import get_current_datetime
19
+ from regscale.integrations.commercial.aws.org_control_mappings import OrgControlMapper
20
+ from regscale.integrations.compliance_integration import ComplianceIntegration, ComplianceItem
21
+ from regscale.models import regscale_models
22
+ from regscale.models.regscale_models.evidence import Evidence
23
+ from regscale.models.regscale_models.evidence_mapping import EvidenceMapping
24
+ from regscale.models.regscale_models.file import File
25
+
26
+ logger = logging.getLogger("regscale")
27
+
28
+ # Constants
29
+ ORG_CACHE_FILE = os.path.join("artifacts", "aws", "org_data.json")
30
+ CACHE_TTL_SECONDS = 4 * 60 * 60 # 4 hours
31
+
32
+ # HTML constants
33
+ HTML_STRONG_OPEN = "<strong>"
34
+ HTML_STRONG_CLOSE = "</strong>"
35
+ HTML_P_OPEN = "<p>"
36
+ HTML_P_CLOSE = "</p>"
37
+ HTML_UL_OPEN = "<ul>"
38
+ HTML_UL_CLOSE = "</ul>"
39
+ HTML_LI_OPEN = "<li>"
40
+ HTML_LI_CLOSE = "</li>"
41
+ HTML_H2_OPEN = "<h2>"
42
+ HTML_H2_CLOSE = "</h2>"
43
+ HTML_H3_OPEN = "<h3>"
44
+ HTML_H3_CLOSE = "</h3>"
45
+ HTML_BR = "<br>"
46
+
47
+
48
+ class OrgComplianceItem(ComplianceItem):
49
+ """Compliance item representing AWS Organizations assessment."""
50
+
51
+ def __init__(self, org_data: Dict[str, Any], control_mapper: OrgControlMapper):
52
+ """
53
+ Initialize Organizations compliance item.
54
+
55
+ :param Dict[str, Any] org_data: Organization structure and metadata
56
+ :param OrgControlMapper control_mapper: Control mapper for compliance assessment
57
+ """
58
+ self.org_data = org_data
59
+ self.control_mapper = control_mapper
60
+
61
+ # Extract organization attributes
62
+ self._org_id = org_data.get("Id", "")
63
+ self._org_arn = org_data.get("Arn", "")
64
+ self._master_account_id = org_data.get("MasterAccountId", "")
65
+ self._accounts = org_data.get("accounts", [])
66
+ self._ous = org_data.get("organizational_units", [])
67
+ self._scps = org_data.get("service_control_policies", [])
68
+
69
+ # Assess compliance
70
+ self._compliance_results = control_mapper.assess_organization_compliance(org_data)
71
+
72
+ @property
73
+ def resource_id(self) -> str:
74
+ """Unique identifier for the organization."""
75
+ return self._org_id
76
+
77
+ @property
78
+ def resource_name(self) -> str:
79
+ """Human-readable name of the organization."""
80
+ return f"AWS Organization {self._org_id[:12]}..."
81
+
82
+ @property
83
+ def control_id(self) -> str:
84
+ """Primary control identifier."""
85
+ # Return first failing control, or first control if all pass
86
+ for control_id, result in self._compliance_results.items():
87
+ if result == "FAIL":
88
+ return control_id
89
+ return list(self._compliance_results.keys())[0] if self._compliance_results else "AC-1"
90
+
91
+ @property
92
+ def compliance_result(self) -> str:
93
+ """Overall compliance result."""
94
+ if not self._compliance_results:
95
+ return "PASS"
96
+ if "FAIL" in self._compliance_results.values():
97
+ return "FAIL"
98
+ return "PASS"
99
+
100
+ @property
101
+ def severity(self) -> Optional[str]:
102
+ """Severity level based on which controls are failing."""
103
+ if self.compliance_result == "PASS":
104
+ return None
105
+
106
+ # AC-1, PM-9, AC-6 failures are HIGH severity (governance/policy issues)
107
+ if self._compliance_results.get("AC-1") == "FAIL" or self._compliance_results.get("PM-9") == "FAIL":
108
+ return "HIGH"
109
+
110
+ if self._compliance_results.get("AC-6") == "FAIL":
111
+ return "MEDIUM"
112
+
113
+ # AC-2 failures are MEDIUM severity (account management)
114
+ if self._compliance_results.get("AC-2") == "FAIL":
115
+ return "MEDIUM"
116
+
117
+ return "MEDIUM"
118
+
119
+ @property
120
+ def description(self) -> str:
121
+ """Detailed description of the Organizations compliance assessment."""
122
+ desc_parts = self._build_org_summary()
123
+ desc_parts.extend(self._build_compliance_results())
124
+
125
+ if self.compliance_result == "FAIL":
126
+ desc_parts.extend(self._build_remediation_guidance())
127
+
128
+ return "\n".join(desc_parts)
129
+
130
+ def _build_org_summary(self) -> List[str]:
131
+ """Build organization summary section."""
132
+ return [
133
+ f"{HTML_H3_OPEN}AWS Organizations Governance Assessment{HTML_H3_CLOSE}",
134
+ HTML_P_OPEN,
135
+ f"{HTML_STRONG_OPEN}Organization ID:{HTML_STRONG_CLOSE} {self._org_id}{HTML_BR}",
136
+ f"{HTML_STRONG_OPEN}Organization ARN:{HTML_STRONG_CLOSE} {self._org_arn}{HTML_BR}",
137
+ f"{HTML_STRONG_OPEN}Master Account:{HTML_STRONG_CLOSE} {self._master_account_id}{HTML_BR}",
138
+ f"{HTML_STRONG_OPEN}Total Accounts:{HTML_STRONG_CLOSE} {len(self._accounts)}{HTML_BR}",
139
+ f"{HTML_STRONG_OPEN}Organizational Units:{HTML_STRONG_CLOSE} {len(self._ous)}{HTML_BR}",
140
+ f"{HTML_STRONG_OPEN}Service Control Policies:{HTML_STRONG_CLOSE} {len(self._scps)}",
141
+ HTML_P_CLOSE,
142
+ ]
143
+
144
+ def _build_compliance_results(self) -> List[str]:
145
+ """Build compliance results section."""
146
+ results = [
147
+ f"{HTML_H3_OPEN}Control Compliance Results{HTML_H3_CLOSE}",
148
+ HTML_UL_OPEN,
149
+ ]
150
+
151
+ for control_id, result in self._compliance_results.items():
152
+ results.append(self._format_control_result(control_id, result))
153
+
154
+ results.append(HTML_UL_CLOSE)
155
+ return results
156
+
157
+ def _format_control_result(self, control_id: str, result: str) -> str:
158
+ """Format a single control compliance result."""
159
+ result_color = "#d32f2f" if result == "FAIL" else "#2e7d32"
160
+ control_desc = self.control_mapper.get_control_description(control_id)
161
+ return (
162
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}{control_id}:{HTML_STRONG_CLOSE} "
163
+ f"<span style='color: {result_color};'>{result}</span> - {control_desc}{HTML_LI_CLOSE}"
164
+ )
165
+
166
+ def _build_remediation_guidance(self) -> List[str]:
167
+ """Build remediation guidance for failed controls."""
168
+ guidance = [
169
+ f"{HTML_H3_OPEN}Remediation Guidance{HTML_H3_CLOSE}",
170
+ HTML_UL_OPEN,
171
+ ]
172
+
173
+ guidance.extend(self._get_ac1_remediation())
174
+ guidance.extend(self._get_pm9_remediation())
175
+ guidance.extend(self._get_ac2_remediation())
176
+ guidance.extend(self._get_ac6_remediation())
177
+
178
+ guidance.append(HTML_UL_CLOSE)
179
+ return guidance
180
+
181
+ def _get_ac1_remediation(self) -> List[str]:
182
+ """Get AC-1 control remediation steps."""
183
+ items = []
184
+ if self._compliance_results.get("AC-1") == "FAIL":
185
+ if len(self._ous) < 2:
186
+ items.append(f"{HTML_LI_OPEN}Create organizational units (OUs) for governance structure{HTML_LI_CLOSE}")
187
+
188
+ restrictive_scps = [scp for scp in self._scps if scp.get("Name") != "FullAWSAccess"]
189
+ if not restrictive_scps:
190
+ items.append(f"{HTML_LI_OPEN}Attach Service Control Policies to enforce access controls{HTML_LI_CLOSE}")
191
+ return items
192
+
193
+ def _get_pm9_remediation(self) -> List[str]:
194
+ """Get PM-9 control remediation steps."""
195
+ items = []
196
+ if self._compliance_results.get("PM-9") == "FAIL":
197
+ items.append(
198
+ f"{HTML_LI_OPEN}Organize accounts by risk profile (prod, dev, sandbox) using OUs{HTML_LI_CLOSE}"
199
+ )
200
+ items.append(f"{HTML_LI_OPEN}Implement restrictive SCPs for security guardrails{HTML_LI_CLOSE}")
201
+ return items
202
+
203
+ def _get_ac2_remediation(self) -> List[str]:
204
+ """Get AC-2 control remediation steps."""
205
+ items = []
206
+ if self._compliance_results.get("AC-2") == "FAIL":
207
+ non_active = [acc for acc in self._accounts if acc.get("Status") != "ACTIVE"]
208
+ if non_active:
209
+ items.append(
210
+ f"{HTML_LI_OPEN}Review and activate or remove {len(non_active)} suspended accounts{HTML_LI_CLOSE}"
211
+ )
212
+ return items
213
+
214
+ def _get_ac6_remediation(self) -> List[str]:
215
+ """Get AC-6 control remediation steps."""
216
+ items = []
217
+ if self._compliance_results.get("AC-6") == "FAIL":
218
+ items.append(
219
+ f"{HTML_LI_OPEN}Implement least privilege SCPs (deny unnecessary services/actions){HTML_LI_CLOSE}"
220
+ )
221
+ return items
222
+
223
+ @property
224
+ def framework(self) -> str:
225
+ """Compliance framework used for assessment."""
226
+ return self.control_mapper.framework
227
+
228
+
229
+ class AWSOrganizationsEvidenceIntegration(ComplianceIntegration):
230
+ """Process AWS Organizations data and create evidence/compliance records in RegScale."""
231
+
232
+ def __init__(
233
+ self,
234
+ plan_id: int,
235
+ region: str = "us-east-1",
236
+ framework: str = "NIST800-53R5",
237
+ create_issues: bool = True,
238
+ update_control_status: bool = True,
239
+ create_poams: bool = False,
240
+ parent_module: str = "securityplans",
241
+ collect_evidence: bool = False,
242
+ evidence_as_attachments: bool = True,
243
+ evidence_control_ids: Optional[List[str]] = None,
244
+ evidence_frequency: int = 30,
245
+ force_refresh: bool = False,
246
+ **kwargs,
247
+ ):
248
+ """
249
+ Initialize AWS Organizations evidence integration.
250
+
251
+ :param int plan_id: RegScale plan ID
252
+ :param str region: AWS region
253
+ :param str framework: Compliance framework
254
+ :param bool create_issues: Create issues for non-compliant organization
255
+ :param bool update_control_status: Update control implementation status
256
+ :param bool create_poams: Mark issues as POAMs
257
+ :param str parent_module: RegScale parent module
258
+ :param bool collect_evidence: Collect evidence artifacts
259
+ :param bool evidence_as_attachments: Attach evidence to SSP vs create Evidence records
260
+ :param Optional[List[str]] evidence_control_ids: Specific control IDs for evidence
261
+ :param int evidence_frequency: Evidence update frequency in days
262
+ :param bool force_refresh: Force refresh by bypassing cache
263
+ :param kwargs: Additional parameters including AWS credentials
264
+ """
265
+ super().__init__(
266
+ plan_id=plan_id,
267
+ framework=framework,
268
+ create_issues=create_issues,
269
+ update_control_status=update_control_status,
270
+ create_poams=create_poams,
271
+ parent_module=parent_module,
272
+ **kwargs,
273
+ )
274
+
275
+ # Initialize API for file operations
276
+ self.api = Api()
277
+
278
+ self.region = region
279
+ self.title = "AWS Organizations"
280
+ self.collect_evidence = collect_evidence
281
+ self.evidence_as_attachments = evidence_as_attachments
282
+ self.evidence_control_ids = evidence_control_ids
283
+ self.evidence_frequency = evidence_frequency
284
+ self.force_refresh = force_refresh
285
+
286
+ # Initialize control mapper
287
+ self.control_mapper = OrgControlMapper(framework=framework)
288
+
289
+ # Extract AWS credentials
290
+ profile = kwargs.get("profile")
291
+ aws_access_key_id = kwargs.get("aws_access_key_id")
292
+ aws_secret_access_key = kwargs.get("aws_secret_access_key")
293
+ aws_session_token = kwargs.get("aws_session_token")
294
+
295
+ if aws_access_key_id and aws_secret_access_key:
296
+ logger.info("Initializing AWS Organizations client with explicit credentials")
297
+ self.session = boto3.Session(
298
+ region_name=region,
299
+ aws_access_key_id=aws_access_key_id,
300
+ aws_secret_access_key=aws_secret_access_key,
301
+ aws_session_token=aws_session_token,
302
+ )
303
+ else:
304
+ logger.info(f"Initializing AWS Organizations client with profile: {profile if profile else 'default'}")
305
+ self.session = boto3.Session(profile_name=profile, region_name=region)
306
+
307
+ try:
308
+ self.client = self.session.client("organizations")
309
+ logger.info("Successfully created AWS Organizations client")
310
+ except Exception as e:
311
+ logger.error(f"Failed to create AWS Organizations client: {e}")
312
+ raise
313
+
314
+ # Store raw org data
315
+ self.raw_org_data: Dict[str, Any] = {}
316
+
317
+ def _is_cache_valid(self) -> bool:
318
+ """Check if cache is valid."""
319
+ if not os.path.exists(ORG_CACHE_FILE):
320
+ return False
321
+ file_age = time.time() - os.path.getmtime(ORG_CACHE_FILE)
322
+ is_valid = file_age < CACHE_TTL_SECONDS
323
+ if is_valid:
324
+ logger.info(f"Using cached Organizations data (age: {file_age / 3600:.1f} hours)")
325
+ return is_valid
326
+
327
+ def _load_cached_data(self) -> Dict[str, Any]:
328
+ """Load Organizations data from cache."""
329
+ try:
330
+ with open(ORG_CACHE_FILE, encoding="utf-8") as file:
331
+ data = json.load(file)
332
+
333
+ # Validate cache format - must be a dict
334
+ if not isinstance(data, dict):
335
+ logger.warning("Invalid cache format detected (not a dict). Invalidating cache.")
336
+ return {}
337
+
338
+ return data
339
+ except (json.JSONDecodeError, IOError) as e:
340
+ logger.warning(f"Error reading cache: {e}")
341
+ return {}
342
+
343
+ def _save_to_cache(self, org_data: Dict[str, Any]) -> None:
344
+ """Save Organizations data to cache."""
345
+ try:
346
+ os.makedirs(os.path.dirname(ORG_CACHE_FILE), exist_ok=True)
347
+ with open(ORG_CACHE_FILE, "w", encoding="utf-8") as file:
348
+ json.dump(org_data, file, indent=2, default=str)
349
+ logger.info(f"Cached Organizations data to {ORG_CACHE_FILE}")
350
+ except IOError as e:
351
+ logger.warning(f"Error writing cache: {e}")
352
+
353
+ def _fetch_fresh_org_data(self) -> Dict[str, Any]:
354
+ """Fetch fresh Organizations data from AWS."""
355
+ logger.info("Fetching Organizations data from AWS...")
356
+
357
+ org_data = {}
358
+
359
+ try:
360
+ # Get organization details
361
+ org_response = self.client.describe_organization()
362
+ org_data.update(org_response.get("Organization", {}))
363
+
364
+ # Get all accounts
365
+ accounts = []
366
+ paginator = self.client.get_paginator("list_accounts")
367
+ for page in paginator.paginate():
368
+ accounts.extend(page.get("Accounts", []))
369
+ org_data["accounts"] = accounts
370
+ logger.info(f"Found {len(accounts)} accounts in organization")
371
+
372
+ # Get organizational units
373
+ ous = self._list_organizational_units()
374
+ org_data["organizational_units"] = ous
375
+ logger.info(f"Found {len(ous)} organizational units")
376
+
377
+ # Get service control policies
378
+ scps = self._list_service_control_policies()
379
+ org_data["service_control_policies"] = scps
380
+ logger.info(f"Found {len(scps)} service control policies")
381
+
382
+ except ClientError as e:
383
+ logger.error(f"Error fetching Organizations data: {e}")
384
+ return {}
385
+
386
+ return org_data
387
+
388
+ def _list_organizational_units(self) -> List[Dict[str, Any]]:
389
+ """List all organizational units recursively."""
390
+ ous = []
391
+
392
+ def traverse_ous(parent_id: str):
393
+ try:
394
+ paginator = self.client.get_paginator("list_organizational_units_for_parent")
395
+ for page in paginator.paginate(ParentId=parent_id):
396
+ for ou in page.get("OrganizationalUnits", []):
397
+ ous.append(ou)
398
+ # Recursively get child OUs
399
+ traverse_ous(ou["Id"])
400
+ except ClientError as e:
401
+ logger.debug(f"Error listing OUs for parent {parent_id}: {e}")
402
+
403
+ try:
404
+ # Start from root
405
+ roots = self.client.list_roots().get("Roots", [])
406
+ for root in roots:
407
+ traverse_ous(root["Id"])
408
+ except ClientError as e:
409
+ logger.error(f"Error getting roots: {e}")
410
+
411
+ return ous
412
+
413
+ def _list_service_control_policies(self) -> List[Dict[str, Any]]:
414
+ """List all service control policies with their content."""
415
+ scps = []
416
+ try:
417
+ paginator = self.client.get_paginator("list_policies")
418
+ for page in paginator.paginate(Filter="SERVICE_CONTROL_POLICY"):
419
+ for policy_summary in page.get("Policies", []):
420
+ # Get full policy details including content
421
+ try:
422
+ policy_detail = self.client.describe_policy(PolicyId=policy_summary["Id"])
423
+ scps.append(policy_detail.get("Policy", {}))
424
+ except ClientError as e:
425
+ logger.debug(f"Error describing policy {policy_summary['Id']}: {e}")
426
+ except ClientError as e:
427
+ logger.error(f"Error listing SCPs: {e}")
428
+
429
+ return scps
430
+
431
+ def fetch_compliance_data(self) -> List[Dict[str, Any]]:
432
+ """Fetch raw Organizations data."""
433
+ if not self.force_refresh and self._is_cache_valid():
434
+ cached_data = self._load_cached_data()
435
+ if cached_data:
436
+ self.raw_org_data = cached_data
437
+ return [cached_data]
438
+
439
+ if self.force_refresh:
440
+ logger.info("Force refresh requested, fetching fresh Organizations data...")
441
+
442
+ try:
443
+ org_data = self._fetch_fresh_org_data()
444
+ self.raw_org_data = org_data
445
+ self._save_to_cache(org_data)
446
+ return [org_data] if org_data else []
447
+ except ClientError as e:
448
+ logger.error(f"Error fetching Organizations data: {e}")
449
+ return []
450
+
451
+ def create_compliance_item(self, raw_data: Dict[str, Any]) -> ComplianceItem:
452
+ """Create compliance item from Organizations data."""
453
+ return OrgComplianceItem(raw_data, self.control_mapper)
454
+
455
+ def sync_compliance(self) -> None:
456
+ """Main method to sync Organizations compliance data."""
457
+ super().sync_compliance()
458
+
459
+ if self.collect_evidence:
460
+ logger.info("Evidence collection enabled, starting evidence collection...")
461
+ self._collect_org_evidence()
462
+
463
+ def _collect_org_evidence(self) -> None:
464
+ """Collect Organizations evidence."""
465
+ if not self.raw_org_data:
466
+ logger.warning("No Organizations data available for evidence collection")
467
+ return
468
+
469
+ scan_date = get_current_datetime(dt_format="%Y-%m-%d")
470
+
471
+ if self.evidence_as_attachments:
472
+ logger.info("Creating SSP file attachment with Organizations evidence...")
473
+ self._create_ssp_attachment(scan_date)
474
+ else:
475
+ logger.info("Creating Evidence record with Organizations evidence...")
476
+ self._create_evidence_record(scan_date)
477
+
478
+ def _create_ssp_attachment(self, scan_date: str) -> None:
479
+ """Create SSP file attachment with Organizations evidence."""
480
+ try:
481
+ # Check for existing evidence to avoid duplicates
482
+ date_str = datetime.now().strftime("%Y%m%d")
483
+ file_name_pattern = f"org_evidence_{date_str}"
484
+
485
+ if self.check_for_existing_evidence(file_name_pattern):
486
+ logger.info(
487
+ "Evidence file for Organizations already exists for today. Skipping upload to avoid duplicates."
488
+ )
489
+ return
490
+
491
+ # Add timestamp to make filename unique if run multiple times per day
492
+ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
493
+ file_name = f"org_evidence_{timestamp}.jsonl.gz"
494
+
495
+ # Prepare JSONL content
496
+ compliance_item = self.create_compliance_item(self.raw_org_data)
497
+ evidence_entry = {
498
+ **self.raw_org_data,
499
+ "compliance_assessment": {
500
+ "overall_result": compliance_item.compliance_result,
501
+ "control_results": compliance_item._compliance_results,
502
+ "assessed_controls": list(compliance_item._compliance_results.keys()),
503
+ "assessment_date": scan_date,
504
+ },
505
+ }
506
+ jsonl_content = json.dumps(evidence_entry, default=str)
507
+
508
+ # Compress
509
+ compressed_buffer = BytesIO()
510
+ with gzip.open(compressed_buffer, "wt", encoding="utf-8", compresslevel=9) as gz_file:
511
+ gz_file.write(jsonl_content)
512
+
513
+ compressed_data = compressed_buffer.getvalue()
514
+
515
+ # Upload
516
+ api = Api()
517
+ success = File.upload_file_to_regscale(
518
+ file_name=file_name,
519
+ parent_id=self.plan_id,
520
+ parent_module=self.parent_module,
521
+ api=api,
522
+ file_data=compressed_data,
523
+ tags="aws,organizations,governance,automated",
524
+ )
525
+
526
+ if success:
527
+ logger.info(f"Successfully uploaded Organizations evidence file: {file_name}")
528
+ else:
529
+ logger.error("Failed to upload Organizations evidence file")
530
+
531
+ except Exception as e:
532
+ logger.error(f"Error creating SSP attachment: {e}", exc_info=True)
533
+
534
+ def _create_evidence_record(self, scan_date: str) -> None:
535
+ """Create Evidence record with Organizations evidence."""
536
+ try:
537
+ title = f"AWS Organizations Evidence - {scan_date}"
538
+ description = self._build_evidence_description(scan_date)
539
+ due_date = (datetime.now() + timedelta(days=self.evidence_frequency)).isoformat()
540
+
541
+ evidence = Evidence(
542
+ title=title,
543
+ description=description,
544
+ status="Collected",
545
+ updateFrequency=self.evidence_frequency,
546
+ dueDate=due_date,
547
+ )
548
+
549
+ created_evidence = evidence.create()
550
+ if not created_evidence or not created_evidence.id:
551
+ logger.error("Failed to create evidence record")
552
+ return
553
+
554
+ logger.info(f"Created evidence record {created_evidence.id}: {title}")
555
+
556
+ # Upload evidence file
557
+ self._upload_evidence_file(created_evidence.id, scan_date)
558
+
559
+ # Link to SSP
560
+ self._link_evidence_to_ssp(created_evidence.id)
561
+
562
+ # Link to controls if specified
563
+ if self.evidence_control_ids:
564
+ self._link_evidence_to_controls(created_evidence.id, is_attachment=False)
565
+
566
+ except Exception as e:
567
+ logger.error(f"Error creating evidence record: {e}", exc_info=True)
568
+
569
+ def _build_evidence_description(self, scan_date: str) -> str:
570
+ """Build HTML evidence description."""
571
+ accounts = self.raw_org_data.get("accounts", [])
572
+ ous = self.raw_org_data.get("organizational_units", [])
573
+ scps = self.raw_org_data.get("service_control_policies", [])
574
+
575
+ compliance_item = self.create_compliance_item(self.raw_org_data)
576
+ control_stats = dict(compliance_item._compliance_results.items())
577
+
578
+ desc_parts = [
579
+ "<h1>AWS Organizations Governance Evidence</h1>",
580
+ f"{HTML_P_OPEN}{HTML_STRONG_OPEN}Assessment Date:{HTML_STRONG_CLOSE} {scan_date}{HTML_P_CLOSE}",
581
+ f"{HTML_H2_OPEN}Organization Summary{HTML_H2_CLOSE}",
582
+ HTML_UL_OPEN,
583
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}Total Accounts:{HTML_STRONG_CLOSE} {len(accounts)}{HTML_LI_CLOSE}",
584
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}Organizational Units:{HTML_STRONG_CLOSE} {len(ous)}{HTML_LI_CLOSE}",
585
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}Service Control Policies:{HTML_STRONG_CLOSE} {len(scps)}{HTML_LI_CLOSE}",
586
+ HTML_UL_CLOSE,
587
+ f"{HTML_H2_OPEN}Control Compliance Results{HTML_H2_CLOSE}",
588
+ HTML_UL_OPEN,
589
+ ]
590
+
591
+ for control_id, result in control_stats.items():
592
+ control_desc = self.control_mapper.get_control_description(control_id)
593
+ result_color = "#d32f2f" if result == "FAIL" else "#2e7d32"
594
+ desc_parts.append(
595
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}{control_id}:{HTML_STRONG_CLOSE} "
596
+ f"<span style='color: {result_color};'>{result}</span> - {control_desc}{HTML_LI_CLOSE}"
597
+ )
598
+
599
+ desc_parts.append(HTML_UL_CLOSE)
600
+ return "\n".join(desc_parts)
601
+
602
+ def _upload_evidence_file(self, evidence_id: int, scan_date: str) -> None:
603
+ """Upload evidence file to Evidence record."""
604
+ try:
605
+ compliance_item = self.create_compliance_item(self.raw_org_data)
606
+ evidence_entry = {
607
+ **self.raw_org_data,
608
+ "compliance_assessment": {
609
+ "overall_result": compliance_item.compliance_result,
610
+ "control_results": compliance_item._compliance_results,
611
+ "assessed_controls": list(compliance_item._compliance_results.keys()),
612
+ "assessment_date": scan_date,
613
+ },
614
+ }
615
+ jsonl_content = json.dumps(evidence_entry, default=str)
616
+
617
+ compressed_buffer = BytesIO()
618
+ with gzip.open(compressed_buffer, "wt", encoding="utf-8", compresslevel=9) as gz_file:
619
+ gz_file.write(jsonl_content)
620
+
621
+ compressed_data = compressed_buffer.getvalue()
622
+ file_name = f"org_evidence_{scan_date}.jsonl.gz"
623
+
624
+ api = Api()
625
+ success = File.upload_file_to_regscale(
626
+ file_name=file_name,
627
+ parent_id=evidence_id,
628
+ parent_module="evidence",
629
+ api=api,
630
+ file_data=compressed_data,
631
+ tags="aws,organizations,governance",
632
+ )
633
+
634
+ if success:
635
+ logger.info(f"Uploaded Organizations evidence file to Evidence {evidence_id}")
636
+ else:
637
+ logger.warning(f"Failed to upload evidence file to Evidence {evidence_id}")
638
+
639
+ except Exception as e:
640
+ logger.error(f"Error uploading evidence file: {e}", exc_info=True)
641
+
642
+ def _link_evidence_to_ssp(self, evidence_id: int) -> None:
643
+ """Link evidence to Security Plan."""
644
+ try:
645
+ mapping = EvidenceMapping(evidenceID=evidence_id, mappedID=self.plan_id, mappingType=self.parent_module)
646
+ mapping.create()
647
+ logger.info(f"Linked evidence {evidence_id} to SSP {self.plan_id}")
648
+ except Exception as ex:
649
+ logger.warning(f"Failed to link evidence to SSP: {ex}")
650
+
651
+ def _link_evidence_to_controls(self, evidence_id: int, is_attachment: bool = False) -> None:
652
+ """
653
+ Link evidence to specified control IDs.
654
+
655
+ :param int evidence_id: Evidence or attachment ID
656
+ :param bool is_attachment: True if linking attachment, False for evidence record
657
+ """
658
+ try:
659
+ for control_id in self.evidence_control_ids:
660
+ if is_attachment:
661
+ self.api.link_ssp_attachment_to_control(self.plan_id, evidence_id, control_id)
662
+ else:
663
+ self.api.link_evidence_to_control(evidence_id, control_id)
664
+ logger.info(f"Linked evidence {evidence_id} to control {control_id}")
665
+ except Exception as e:
666
+ logger.error(f"Failed to link evidence to controls: {e}", exc_info=True)