regscale-cli 6.21.2.0__py3-none-any.whl → 6.28.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. regscale/_version.py +1 -1
  2. regscale/airflow/hierarchy.py +2 -2
  3. regscale/core/app/api.py +5 -2
  4. regscale/core/app/application.py +36 -6
  5. regscale/core/app/internal/control_editor.py +73 -21
  6. regscale/core/app/internal/evidence.py +727 -204
  7. regscale/core/app/internal/login.py +4 -2
  8. regscale/core/app/internal/model_editor.py +219 -64
  9. regscale/core/app/utils/app_utils.py +86 -12
  10. regscale/core/app/utils/catalog_utils/common.py +1 -1
  11. regscale/core/login.py +21 -4
  12. regscale/core/utils/async_graphql_client.py +363 -0
  13. regscale/core/utils/date.py +77 -1
  14. regscale/dev/cli.py +26 -0
  15. regscale/dev/code_gen.py +109 -24
  16. regscale/dev/version.py +72 -0
  17. regscale/integrations/commercial/__init__.py +30 -2
  18. regscale/integrations/commercial/aws/audit_manager_compliance.py +3908 -0
  19. regscale/integrations/commercial/aws/cli.py +3107 -54
  20. regscale/integrations/commercial/aws/cloudtrail_control_mappings.py +333 -0
  21. regscale/integrations/commercial/aws/cloudtrail_evidence.py +501 -0
  22. regscale/integrations/commercial/aws/cloudwatch_control_mappings.py +357 -0
  23. regscale/integrations/commercial/aws/cloudwatch_evidence.py +490 -0
  24. regscale/integrations/commercial/{amazon → aws}/common.py +71 -19
  25. regscale/integrations/commercial/aws/config_compliance.py +914 -0
  26. regscale/integrations/commercial/aws/conformance_pack_mappings.py +198 -0
  27. regscale/integrations/commercial/aws/control_compliance_analyzer.py +439 -0
  28. regscale/integrations/commercial/aws/evidence_generator.py +283 -0
  29. regscale/integrations/commercial/aws/guardduty_control_mappings.py +340 -0
  30. regscale/integrations/commercial/aws/guardduty_evidence.py +1053 -0
  31. regscale/integrations/commercial/aws/iam_control_mappings.py +368 -0
  32. regscale/integrations/commercial/aws/iam_evidence.py +574 -0
  33. regscale/integrations/commercial/aws/inventory/__init__.py +338 -22
  34. regscale/integrations/commercial/aws/inventory/base.py +107 -5
  35. regscale/integrations/commercial/aws/inventory/resources/analytics.py +390 -0
  36. regscale/integrations/commercial/aws/inventory/resources/applications.py +234 -0
  37. regscale/integrations/commercial/aws/inventory/resources/audit_manager.py +513 -0
  38. regscale/integrations/commercial/aws/inventory/resources/cloudtrail.py +315 -0
  39. regscale/integrations/commercial/aws/inventory/resources/cloudtrail_logs_metadata.py +476 -0
  40. regscale/integrations/commercial/aws/inventory/resources/cloudwatch.py +191 -0
  41. regscale/integrations/commercial/aws/inventory/resources/compute.py +328 -9
  42. regscale/integrations/commercial/aws/inventory/resources/config.py +464 -0
  43. regscale/integrations/commercial/aws/inventory/resources/containers.py +74 -9
  44. regscale/integrations/commercial/aws/inventory/resources/database.py +481 -31
  45. regscale/integrations/commercial/aws/inventory/resources/developer_tools.py +253 -0
  46. regscale/integrations/commercial/aws/inventory/resources/guardduty.py +286 -0
  47. regscale/integrations/commercial/aws/inventory/resources/iam.py +470 -0
  48. regscale/integrations/commercial/aws/inventory/resources/inspector.py +476 -0
  49. regscale/integrations/commercial/aws/inventory/resources/integration.py +175 -61
  50. regscale/integrations/commercial/aws/inventory/resources/kms.py +447 -0
  51. regscale/integrations/commercial/aws/inventory/resources/machine_learning.py +358 -0
  52. regscale/integrations/commercial/aws/inventory/resources/networking.py +390 -67
  53. regscale/integrations/commercial/aws/inventory/resources/s3.py +394 -0
  54. regscale/integrations/commercial/aws/inventory/resources/security.py +268 -72
  55. regscale/integrations/commercial/aws/inventory/resources/securityhub.py +473 -0
  56. regscale/integrations/commercial/aws/inventory/resources/storage.py +288 -29
  57. regscale/integrations/commercial/aws/inventory/resources/systems_manager.py +657 -0
  58. regscale/integrations/commercial/aws/inventory/resources/vpc.py +655 -0
  59. regscale/integrations/commercial/aws/kms_control_mappings.py +288 -0
  60. regscale/integrations/commercial/aws/kms_evidence.py +879 -0
  61. regscale/integrations/commercial/aws/ocsf/__init__.py +7 -0
  62. regscale/integrations/commercial/aws/ocsf/constants.py +115 -0
  63. regscale/integrations/commercial/aws/ocsf/mapper.py +435 -0
  64. regscale/integrations/commercial/aws/org_control_mappings.py +286 -0
  65. regscale/integrations/commercial/aws/org_evidence.py +666 -0
  66. regscale/integrations/commercial/aws/s3_control_mappings.py +356 -0
  67. regscale/integrations/commercial/aws/s3_evidence.py +632 -0
  68. regscale/integrations/commercial/aws/scanner.py +1072 -205
  69. regscale/integrations/commercial/aws/security_hub.py +319 -0
  70. regscale/integrations/commercial/aws/session_manager.py +282 -0
  71. regscale/integrations/commercial/aws/ssm_control_mappings.py +291 -0
  72. regscale/integrations/commercial/aws/ssm_evidence.py +492 -0
  73. regscale/integrations/commercial/jira.py +489 -153
  74. regscale/integrations/commercial/microsoft_defender/defender.py +326 -5
  75. regscale/integrations/commercial/microsoft_defender/defender_api.py +348 -14
  76. regscale/integrations/commercial/microsoft_defender/defender_constants.py +157 -0
  77. regscale/integrations/commercial/qualys/__init__.py +167 -68
  78. regscale/integrations/commercial/qualys/scanner.py +305 -39
  79. regscale/integrations/commercial/sarif/sairf_importer.py +432 -0
  80. regscale/integrations/commercial/sarif/sarif_converter.py +67 -0
  81. regscale/integrations/commercial/sicura/api.py +79 -42
  82. regscale/integrations/commercial/sicura/commands.py +8 -2
  83. regscale/integrations/commercial/sicura/scanner.py +83 -44
  84. regscale/integrations/commercial/stigv2/ckl_parser.py +5 -5
  85. regscale/integrations/commercial/synqly/assets.py +133 -16
  86. regscale/integrations/commercial/synqly/edr.py +2 -8
  87. regscale/integrations/commercial/synqly/query_builder.py +536 -0
  88. regscale/integrations/commercial/synqly/ticketing.py +27 -0
  89. regscale/integrations/commercial/synqly/vulnerabilities.py +165 -28
  90. regscale/integrations/commercial/tenablev2/cis_parsers.py +453 -0
  91. regscale/integrations/commercial/tenablev2/cis_scanner.py +447 -0
  92. regscale/integrations/commercial/tenablev2/commands.py +146 -5
  93. regscale/integrations/commercial/tenablev2/scanner.py +1 -3
  94. regscale/integrations/commercial/tenablev2/stig_parsers.py +113 -57
  95. regscale/integrations/commercial/wizv2/WizDataMixin.py +1 -1
  96. regscale/integrations/commercial/wizv2/click.py +191 -76
  97. regscale/integrations/commercial/wizv2/compliance/__init__.py +15 -0
  98. regscale/integrations/commercial/wizv2/{policy_compliance_helpers.py → compliance/helpers.py} +78 -60
  99. regscale/integrations/commercial/wizv2/compliance_report.py +1592 -0
  100. regscale/integrations/commercial/wizv2/core/__init__.py +133 -0
  101. regscale/integrations/commercial/wizv2/{async_client.py → core/client.py} +7 -3
  102. regscale/integrations/commercial/wizv2/{constants.py → core/constants.py} +92 -89
  103. regscale/integrations/commercial/wizv2/core/file_operations.py +237 -0
  104. regscale/integrations/commercial/wizv2/fetchers/__init__.py +11 -0
  105. regscale/integrations/commercial/wizv2/{data_fetcher.py → fetchers/policy_assessment.py} +66 -9
  106. regscale/integrations/commercial/wizv2/file_cleanup.py +104 -0
  107. regscale/integrations/commercial/wizv2/issue.py +776 -28
  108. regscale/integrations/commercial/wizv2/models/__init__.py +0 -0
  109. regscale/integrations/commercial/wizv2/parsers/__init__.py +34 -0
  110. regscale/integrations/commercial/wizv2/{parsers.py → parsers/main.py} +1 -1
  111. regscale/integrations/commercial/wizv2/processors/__init__.py +11 -0
  112. regscale/integrations/commercial/wizv2/{finding_processor.py → processors/finding.py} +1 -1
  113. regscale/integrations/commercial/wizv2/reports.py +243 -0
  114. regscale/integrations/commercial/wizv2/sbom.py +1 -1
  115. regscale/integrations/commercial/wizv2/scanner.py +1031 -441
  116. regscale/integrations/commercial/wizv2/utils/__init__.py +48 -0
  117. regscale/integrations/commercial/wizv2/{utils.py → utils/main.py} +116 -61
  118. regscale/integrations/commercial/wizv2/variables.py +89 -3
  119. regscale/integrations/compliance_integration.py +1036 -151
  120. regscale/integrations/control_matcher.py +432 -0
  121. regscale/integrations/due_date_handler.py +333 -0
  122. regscale/integrations/milestone_manager.py +291 -0
  123. regscale/integrations/public/__init__.py +14 -0
  124. regscale/integrations/public/cci_importer.py +834 -0
  125. regscale/integrations/public/csam/__init__.py +0 -0
  126. regscale/integrations/public/csam/csam.py +938 -0
  127. regscale/integrations/public/csam/csam_agency_defined.py +179 -0
  128. regscale/integrations/public/csam/csam_common.py +154 -0
  129. regscale/integrations/public/csam/csam_controls.py +432 -0
  130. regscale/integrations/public/csam/csam_poam.py +124 -0
  131. regscale/integrations/public/fedramp/click.py +77 -6
  132. regscale/integrations/public/fedramp/docx_parser.py +10 -1
  133. regscale/integrations/public/fedramp/fedramp_cis_crm.py +675 -289
  134. regscale/integrations/public/fedramp/fedramp_five.py +1 -1
  135. regscale/integrations/public/fedramp/poam/scanner.py +75 -7
  136. regscale/integrations/public/fedramp/poam_export_v5.py +888 -0
  137. regscale/integrations/scanner_integration.py +1961 -430
  138. regscale/models/integration_models/CCI_List.xml +1 -0
  139. regscale/models/integration_models/aqua.py +2 -2
  140. regscale/models/integration_models/cisa_kev_data.json +805 -11
  141. regscale/models/integration_models/flat_file_importer/__init__.py +5 -8
  142. regscale/models/integration_models/nexpose.py +36 -10
  143. regscale/models/integration_models/qualys.py +3 -4
  144. regscale/models/integration_models/synqly_models/capabilities.json +1 -1
  145. regscale/models/integration_models/synqly_models/connectors/vulnerabilities.py +87 -18
  146. regscale/models/integration_models/synqly_models/filter_parser.py +332 -0
  147. regscale/models/integration_models/synqly_models/ocsf_mapper.py +124 -25
  148. regscale/models/integration_models/synqly_models/synqly_model.py +89 -16
  149. regscale/models/locking.py +12 -8
  150. regscale/models/platform.py +4 -2
  151. regscale/models/regscale_models/__init__.py +7 -0
  152. regscale/models/regscale_models/assessment.py +2 -1
  153. regscale/models/regscale_models/catalog.py +1 -1
  154. regscale/models/regscale_models/compliance_settings.py +251 -1
  155. regscale/models/regscale_models/component.py +1 -0
  156. regscale/models/regscale_models/control_implementation.py +236 -41
  157. regscale/models/regscale_models/control_objective.py +74 -5
  158. regscale/models/regscale_models/file.py +2 -0
  159. regscale/models/regscale_models/form_field_value.py +5 -3
  160. regscale/models/regscale_models/inheritance.py +44 -0
  161. regscale/models/regscale_models/issue.py +301 -102
  162. regscale/models/regscale_models/milestone.py +33 -14
  163. regscale/models/regscale_models/organization.py +3 -0
  164. regscale/models/regscale_models/regscale_model.py +310 -73
  165. regscale/models/regscale_models/security_plan.py +4 -2
  166. regscale/models/regscale_models/vulnerability.py +3 -3
  167. regscale/regscale.py +25 -4
  168. regscale/templates/__init__.py +0 -0
  169. regscale/utils/threading/threadhandler.py +20 -15
  170. regscale/validation/record.py +23 -1
  171. {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/METADATA +17 -33
  172. {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/RECORD +310 -111
  173. tests/core/__init__.py +0 -0
  174. tests/core/utils/__init__.py +0 -0
  175. tests/core/utils/test_async_graphql_client.py +472 -0
  176. tests/fixtures/test_fixture.py +13 -8
  177. tests/regscale/core/test_login.py +171 -4
  178. tests/regscale/integrations/commercial/__init__.py +0 -0
  179. tests/regscale/integrations/commercial/aws/__init__.py +0 -0
  180. tests/regscale/integrations/commercial/aws/test_audit_manager_compliance.py +1304 -0
  181. tests/regscale/integrations/commercial/aws/test_audit_manager_evidence_aggregation.py +341 -0
  182. tests/regscale/integrations/commercial/aws/test_aws_analytics_collector.py +260 -0
  183. tests/regscale/integrations/commercial/aws/test_aws_applications_collector.py +242 -0
  184. tests/regscale/integrations/commercial/aws/test_aws_audit_manager_collector.py +1155 -0
  185. tests/regscale/integrations/commercial/aws/test_aws_cloudtrail_collector.py +534 -0
  186. tests/regscale/integrations/commercial/aws/test_aws_config_collector.py +400 -0
  187. tests/regscale/integrations/commercial/aws/test_aws_developer_tools_collector.py +203 -0
  188. tests/regscale/integrations/commercial/aws/test_aws_guardduty_collector.py +315 -0
  189. tests/regscale/integrations/commercial/aws/test_aws_iam_collector.py +458 -0
  190. tests/regscale/integrations/commercial/aws/test_aws_inspector_collector.py +353 -0
  191. tests/regscale/integrations/commercial/aws/test_aws_inventory_integration.py +530 -0
  192. tests/regscale/integrations/commercial/aws/test_aws_kms_collector.py +919 -0
  193. tests/regscale/integrations/commercial/aws/test_aws_machine_learning_collector.py +237 -0
  194. tests/regscale/integrations/commercial/aws/test_aws_s3_collector.py +722 -0
  195. tests/regscale/integrations/commercial/aws/test_aws_scanner_integration.py +722 -0
  196. tests/regscale/integrations/commercial/aws/test_aws_securityhub_collector.py +792 -0
  197. tests/regscale/integrations/commercial/aws/test_aws_systems_manager_collector.py +918 -0
  198. tests/regscale/integrations/commercial/aws/test_aws_vpc_collector.py +996 -0
  199. tests/regscale/integrations/commercial/aws/test_cli_evidence.py +431 -0
  200. tests/regscale/integrations/commercial/aws/test_cloudtrail_control_mappings.py +452 -0
  201. tests/regscale/integrations/commercial/aws/test_cloudtrail_evidence.py +788 -0
  202. tests/regscale/integrations/commercial/aws/test_config_compliance.py +298 -0
  203. tests/regscale/integrations/commercial/aws/test_conformance_pack_mappings.py +200 -0
  204. tests/regscale/integrations/commercial/aws/test_control_compliance_analyzer.py +375 -0
  205. tests/regscale/integrations/commercial/aws/test_datetime_parsing.py +223 -0
  206. tests/regscale/integrations/commercial/aws/test_evidence_generator.py +386 -0
  207. tests/regscale/integrations/commercial/aws/test_guardduty_control_mappings.py +564 -0
  208. tests/regscale/integrations/commercial/aws/test_guardduty_evidence.py +1041 -0
  209. tests/regscale/integrations/commercial/aws/test_iam_control_mappings.py +718 -0
  210. tests/regscale/integrations/commercial/aws/test_iam_evidence.py +1375 -0
  211. tests/regscale/integrations/commercial/aws/test_kms_control_mappings.py +656 -0
  212. tests/regscale/integrations/commercial/aws/test_kms_evidence.py +1163 -0
  213. tests/regscale/integrations/commercial/aws/test_ocsf_mapper.py +370 -0
  214. tests/regscale/integrations/commercial/aws/test_org_control_mappings.py +546 -0
  215. tests/regscale/integrations/commercial/aws/test_org_evidence.py +1240 -0
  216. tests/regscale/integrations/commercial/aws/test_s3_control_mappings.py +672 -0
  217. tests/regscale/integrations/commercial/aws/test_s3_evidence.py +987 -0
  218. tests/regscale/integrations/commercial/aws/test_scanner_evidence.py +373 -0
  219. tests/regscale/integrations/commercial/aws/test_security_hub_config_filtering.py +539 -0
  220. tests/regscale/integrations/commercial/aws/test_session_manager.py +516 -0
  221. tests/regscale/integrations/commercial/aws/test_ssm_control_mappings.py +588 -0
  222. tests/regscale/integrations/commercial/aws/test_ssm_evidence.py +735 -0
  223. tests/regscale/integrations/commercial/conftest.py +28 -0
  224. tests/regscale/integrations/commercial/microsoft_defender/__init__.py +1 -0
  225. tests/regscale/integrations/commercial/microsoft_defender/test_defender.py +1517 -0
  226. tests/regscale/integrations/commercial/microsoft_defender/test_defender_api.py +1748 -0
  227. tests/regscale/integrations/commercial/microsoft_defender/test_defender_constants.py +327 -0
  228. tests/regscale/integrations/commercial/microsoft_defender/test_defender_scanner.py +487 -0
  229. tests/regscale/integrations/commercial/test_aws.py +3742 -0
  230. tests/regscale/integrations/commercial/test_burp.py +48 -0
  231. tests/regscale/integrations/commercial/test_crowdstrike.py +49 -0
  232. tests/regscale/integrations/commercial/test_dependabot.py +341 -0
  233. tests/regscale/integrations/commercial/test_gcp.py +1543 -0
  234. tests/regscale/integrations/commercial/test_gitlab.py +549 -0
  235. tests/regscale/integrations/commercial/test_ip_mac_address_length.py +84 -0
  236. tests/regscale/integrations/commercial/test_jira.py +2204 -0
  237. tests/regscale/integrations/commercial/test_npm_audit.py +42 -0
  238. tests/regscale/integrations/commercial/test_okta.py +1228 -0
  239. tests/regscale/integrations/commercial/test_sarif_converter.py +251 -0
  240. tests/regscale/integrations/commercial/test_sicura.py +349 -0
  241. tests/regscale/integrations/commercial/test_snow.py +423 -0
  242. tests/regscale/integrations/commercial/test_sonarcloud.py +394 -0
  243. tests/regscale/integrations/commercial/test_sqlserver.py +186 -0
  244. tests/regscale/integrations/commercial/test_stig.py +33 -0
  245. tests/regscale/integrations/commercial/test_stig_mapper.py +153 -0
  246. tests/regscale/integrations/commercial/test_stigv2.py +406 -0
  247. tests/regscale/integrations/commercial/test_wiz.py +1365 -0
  248. tests/regscale/integrations/commercial/test_wiz_inventory.py +256 -0
  249. tests/regscale/integrations/commercial/wizv2/__init__.py +339 -0
  250. tests/regscale/integrations/commercial/wizv2/compliance/__init__.py +1 -0
  251. tests/regscale/integrations/commercial/wizv2/compliance/test_helpers.py +903 -0
  252. tests/regscale/integrations/commercial/wizv2/core/__init__.py +1 -0
  253. tests/regscale/integrations/commercial/wizv2/core/test_auth.py +701 -0
  254. tests/regscale/integrations/commercial/wizv2/core/test_client.py +1037 -0
  255. tests/regscale/integrations/commercial/wizv2/core/test_file_operations.py +989 -0
  256. tests/regscale/integrations/commercial/wizv2/fetchers/__init__.py +1 -0
  257. tests/regscale/integrations/commercial/wizv2/fetchers/test_policy_assessment.py +805 -0
  258. tests/regscale/integrations/commercial/wizv2/parsers/__init__.py +1 -0
  259. tests/regscale/integrations/commercial/wizv2/parsers/test_main.py +1153 -0
  260. tests/regscale/integrations/commercial/wizv2/processors/__init__.py +1 -0
  261. tests/regscale/integrations/commercial/wizv2/processors/test_finding.py +671 -0
  262. tests/regscale/integrations/commercial/wizv2/test_WizDataMixin.py +537 -0
  263. tests/regscale/integrations/commercial/wizv2/test_click_comprehensive.py +851 -0
  264. tests/regscale/integrations/commercial/wizv2/test_compliance_report_comprehensive.py +910 -0
  265. tests/regscale/integrations/commercial/wizv2/test_compliance_report_normalization.py +138 -0
  266. tests/regscale/integrations/commercial/wizv2/test_file_cleanup.py +283 -0
  267. tests/regscale/integrations/commercial/wizv2/test_file_operations.py +260 -0
  268. tests/regscale/integrations/commercial/wizv2/test_issue.py +343 -0
  269. tests/regscale/integrations/commercial/wizv2/test_issue_comprehensive.py +1203 -0
  270. tests/regscale/integrations/commercial/wizv2/test_reports.py +497 -0
  271. tests/regscale/integrations/commercial/wizv2/test_sbom.py +643 -0
  272. tests/regscale/integrations/commercial/wizv2/test_scanner_comprehensive.py +805 -0
  273. tests/regscale/integrations/commercial/wizv2/test_wiz_click_client_id.py +165 -0
  274. tests/regscale/integrations/commercial/wizv2/test_wiz_compliance_report.py +1394 -0
  275. tests/regscale/integrations/commercial/wizv2/test_wiz_compliance_unit.py +341 -0
  276. tests/regscale/integrations/commercial/wizv2/test_wiz_control_normalization.py +138 -0
  277. tests/regscale/integrations/commercial/wizv2/test_wiz_findings_comprehensive.py +364 -0
  278. tests/regscale/integrations/commercial/wizv2/test_wiz_inventory_comprehensive.py +644 -0
  279. tests/regscale/integrations/commercial/wizv2/test_wiz_status_mapping.py +149 -0
  280. tests/regscale/integrations/commercial/wizv2/test_wizv2.py +1218 -0
  281. tests/regscale/integrations/commercial/wizv2/test_wizv2_utils.py +519 -0
  282. tests/regscale/integrations/commercial/wizv2/utils/__init__.py +1 -0
  283. tests/regscale/integrations/commercial/wizv2/utils/test_main.py +1523 -0
  284. tests/regscale/integrations/public/__init__.py +0 -0
  285. tests/regscale/integrations/public/fedramp/__init__.py +1 -0
  286. tests/regscale/integrations/public/fedramp/test_gen_asset_list.py +150 -0
  287. tests/regscale/integrations/public/fedramp/test_poam_export_v5.py +1293 -0
  288. tests/regscale/integrations/public/test_alienvault.py +220 -0
  289. tests/regscale/integrations/public/test_cci.py +1053 -0
  290. tests/regscale/integrations/public/test_cisa.py +1021 -0
  291. tests/regscale/integrations/public/test_emass.py +518 -0
  292. tests/regscale/integrations/public/test_fedramp.py +1152 -0
  293. tests/regscale/integrations/public/test_fedramp_cis_crm.py +3661 -0
  294. tests/regscale/integrations/public/test_file_uploads.py +506 -0
  295. tests/regscale/integrations/public/test_oscal.py +453 -0
  296. tests/regscale/integrations/test_compliance_status_mapping.py +406 -0
  297. tests/regscale/integrations/test_control_matcher.py +1421 -0
  298. tests/regscale/integrations/test_control_matching.py +155 -0
  299. tests/regscale/integrations/test_milestone_manager.py +408 -0
  300. tests/regscale/models/test_control_implementation.py +118 -3
  301. tests/regscale/models/test_form_field_value_integration.py +304 -0
  302. tests/regscale/models/test_issue.py +378 -1
  303. tests/regscale/models/test_module_integration.py +582 -0
  304. tests/regscale/models/test_tenable_integrations.py +811 -105
  305. regscale/integrations/commercial/wizv2/policy_compliance.py +0 -3057
  306. regscale/integrations/public/fedramp/mappings/fedramp_r4_parts.json +0 -7388
  307. regscale/integrations/public/fedramp/mappings/fedramp_r5_parts.json +0 -9605
  308. regscale/integrations/public/fedramp/parts_mapper.py +0 -107
  309. /regscale/integrations/commercial/{amazon → sarif}/__init__.py +0 -0
  310. /regscale/integrations/commercial/wizv2/{wiz_auth.py → core/auth.py} +0 -0
  311. {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/LICENSE +0 -0
  312. {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/WHEEL +0 -0
  313. {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/entry_points.txt +0 -0
  314. {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,3908 @@
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ """AWS Audit Manager Compliance Integration for RegScale CLI."""
4
+
5
+ import json
6
+ import logging
7
+ import os
8
+ import time
9
+ from dataclasses import dataclass
10
+ from datetime import datetime, timedelta, timezone
11
+ from typing import Any, Dict, List, Optional
12
+
13
+ import boto3
14
+ from botocore.exceptions import ClientError
15
+
16
+ from regscale.core.app.utils.app_utils import create_progress_object, get_current_datetime
17
+ from regscale.integrations.compliance_integration import ComplianceIntegration, ComplianceItem
18
+ from regscale.models import regscale_models
19
+
20
+ # Import ControlComplianceAnalyzer for enhanced evidence analysis
21
+ from regscale.integrations.commercial.aws.control_compliance_analyzer import ControlComplianceAnalyzer
22
+
23
+ logger = logging.getLogger("regscale")
24
+
25
+ # Constants for file paths and cache TTL
26
+ AUDIT_MANAGER_CACHE_FILE = os.path.join("artifacts", "aws", "audit_manager_assessments.json")
27
+ CACHE_TTL_SECONDS = 4 * 60 * 60 # 4 hours in seconds
28
+
29
+ # AWS Audit Manager IAM permission constants
30
+ IAM_PERMISSION_LIST_ASSESSMENTS = "auditmanager:ListAssessments"
31
+ IAM_PERMISSION_GET_ASSESSMENT = "auditmanager:GetAssessment"
32
+ IAM_PERMISSION_GET_EVIDENCE_FOLDERS = "auditmanager:GetEvidenceFoldersByAssessmentControl"
33
+
34
+ # HTML tag constants to avoid duplication
35
+ HTML_STRONG_OPEN = "<strong>"
36
+ HTML_STRONG_CLOSE = "</strong>"
37
+ HTML_P_OPEN = "<p>"
38
+ HTML_P_CLOSE = "</p>"
39
+ HTML_UL_OPEN = "<ul>"
40
+ HTML_UL_CLOSE = "</ul>"
41
+ HTML_LI_OPEN = "<li>"
42
+ HTML_LI_CLOSE = "</li>"
43
+ HTML_H4_OPEN = "<h4>"
44
+ HTML_H4_CLOSE = "</h4>"
45
+ HTML_BR = "<br>"
46
+
47
+
48
+ class AWSAuditManagerComplianceItem(ComplianceItem):
49
+ """
50
+ Compliance item from AWS Audit Manager assessment.
51
+
52
+ IMPORTANT: Evidence-Based Compliance Determination
53
+ ---------------------------------------------------
54
+ This integration uses evidence items to determine control compliance status:
55
+
56
+ 1. Control 'status' field (REVIEWED/UNDER_REVIEW/INACTIVE) is workflow tracking only
57
+ 2. Actual compliance is determined by aggregating evidence items' complianceCheck fields
58
+ 3. Evidence complianceCheck values (normalized internally):
59
+ Success values (→ "COMPLIANT"):
60
+ - "COMPLIANT" (AWS Config)
61
+ - "PASS" (AWS Security Hub)
62
+ Failure values (→ "FAILED"):
63
+ - "NON_COMPLIANT" / "Non-compliant" (AWS Config)
64
+ - "FAILED" / "FAIL" / "Fail" (AWS Security Hub)
65
+ Other:
66
+ - "NOT_APPLICABLE": Evidence not applicable to this control
67
+ - None/missing: No compliance check available
68
+
69
+ Aggregation Logic:
70
+ - ANY evidence with failure values (FAILED, FAIL, NON_COMPLIANT, etc.) → Control FAILS
71
+ - ALL evidence with success values (COMPLIANT, PASS) → Control PASSES
72
+ - NOT_APPLICABLE evidence is tracked separately and doesn't affect compliance
73
+ - No evidence or only inconclusive/not applicable evidence → Returns None (control not updated)
74
+
75
+ The None return value signals the integration framework to skip updating the control
76
+ status, preventing false positive/negative results when evidence is unavailable.
77
+ """
78
+
79
+ def __init__(
80
+ self,
81
+ assessment_data: Dict[str, Any],
82
+ control_data: Dict[str, Any],
83
+ evidence_items: Optional[List[Dict]] = None,
84
+ use_enhanced_analyzer: bool = False,
85
+ ):
86
+ """
87
+ Initialize from AWS Audit Manager assessment and control data.
88
+
89
+ :param Dict[str, Any] assessment_data: Assessment metadata
90
+ :param Dict[str, Any] control_data: Control assessment result
91
+ :param Optional[List[Dict]] evidence_items: Evidence items with complianceCheck fields.
92
+ REQUIRED for accurate compliance determination.
93
+ Without evidence, control status will not be updated.
94
+ :param bool use_enhanced_analyzer: Use enhanced ControlComplianceAnalyzer for evidence analysis
95
+ """
96
+ self.assessment_data = assessment_data
97
+ self.control_data = control_data
98
+ self.evidence_items = evidence_items or []
99
+ self.use_enhanced_analyzer = use_enhanced_analyzer
100
+
101
+ # Extract assessment metadata
102
+ self.assessment_name = assessment_data.get("name", "")
103
+ self.assessment_id = assessment_data.get("arn", "")
104
+ self.framework_name = assessment_data.get("framework", {}).get("metadata", {}).get("name", "")
105
+ self.framework_type = assessment_data.get("framework", {}).get("type", "")
106
+ self.compliance_type = assessment_data.get("complianceType", "")
107
+ self.aws_account = assessment_data.get("awsAccount", {})
108
+
109
+ # Extract control metadata
110
+ # AWS Audit Manager embeds the control ID in the 'name' field
111
+ # Format: "AC-2 - Control Name" or "AC-2(1) - Control Name with Enhancement"
112
+ control_name = control_data.get("name", "")
113
+ self._control_name = control_name
114
+
115
+ # Extract control ID from name field (before the hyphen)
116
+ # Example: "AC-2 - Access Control" -> "AC-2"
117
+ self._control_id = self._extract_control_id_from_name(control_name)
118
+
119
+ self.control_status = control_data.get("status", "UNDER_REVIEW")
120
+ self.control_response = control_data.get("response", "")
121
+ self.control_comments = control_data.get("comments", [])
122
+
123
+ # Log extracted control ID for debugging
124
+ logger.debug(f"Extracted control ID: '{self._control_id}' from name: '{control_name}'")
125
+
126
+ # Extract evidence counts
127
+ self.evidence_count = control_data.get("evidenceCount", 0)
128
+ self.assessment_report_evidence_count = control_data.get("assessmentReportEvidenceCount", 0)
129
+
130
+ # Initialize enhanced analyzer attributes
131
+ self.compliance_score = None
132
+ self.confidence_level = None
133
+ self.compliance_details = None
134
+
135
+ # Extract remediation and testing guidance
136
+ self.action_plan_title = control_data.get("actionPlanTitle", "")
137
+ self.action_plan_instructions = control_data.get("actionPlanInstructions", "")
138
+ self.testing_information = control_data.get("testingInformation", "")
139
+
140
+ # Resource information (from evidence sources)
141
+ self._resource_id = None
142
+ self._resource_name = None
143
+ self._severity = "MEDIUM"
144
+
145
+ # Cache for aggregated compliance result
146
+ self._aggregated_compliance_result = None
147
+
148
+ @property
149
+ def resource_id(self) -> str:
150
+ """Unique identifier for the resource being assessed."""
151
+ if self._resource_id:
152
+ return self._resource_id
153
+ return self.aws_account.get("id", "")
154
+
155
+ @property
156
+ def resource_name(self) -> str:
157
+ """Human-readable name of the resource."""
158
+ if self._resource_name:
159
+ return self._resource_name
160
+ account_name = self.aws_account.get("name", "")
161
+ account_id = self.aws_account.get("id", "")
162
+ if account_name:
163
+ return f"{account_name} ({account_id})"
164
+ return account_id
165
+
166
+ @property
167
+ def resource_arns(self) -> str:
168
+ """
169
+ Extract all failing resource ARNs from evidence, newline-separated.
170
+
171
+ Returns newline-separated ARNs for use in issue.assetIdentifier field.
172
+ This provides full resource identification instead of just account ID.
173
+ All FAILED evidence from AWS Security Hub and AWS Config includes resource ARNs.
174
+
175
+ :return: Newline-separated ARNs of failing resources
176
+ :rtype: str
177
+ """
178
+ failing_resources = self._extract_failing_resources(self.evidence_items)
179
+ if failing_resources:
180
+ # Extract unique ARNs, filtering out any "Unknown ARN" fallback values
181
+ arns = [res["arn"] for res in failing_resources if res.get("arn") and res["arn"] != "Unknown ARN"]
182
+ if arns:
183
+ return "\n".join(arns)
184
+ # Return empty string if no ARNs found (should never happen for FAILED evidence)
185
+ return ""
186
+
187
+ def _try_extract_with_pattern(self, control_name: str, pattern: str) -> Optional[str]:
188
+ """
189
+ Try to extract control ID using a specific regex pattern.
190
+
191
+ :param str control_name: Full control name from AWS
192
+ :param str pattern: Regex pattern to match
193
+ :return: Extracted control ID or None
194
+ :rtype: Optional[str]
195
+ """
196
+ import re
197
+
198
+ match = re.match(pattern, control_name)
199
+ return match.group(1).strip() if match else None
200
+
201
+ def _extract_control_id_from_name(self, control_name: str) -> str:
202
+ """
203
+ Extract control ID from AWS Audit Manager control name.
204
+
205
+ Supports multiple control ID formats:
206
+ - NIST (colon): "AC-2: Access Control (NIST-SP-800-53-r5)", "AC-2(1): Enhancement (NIST-SP-800-53-r5)"
207
+ - NIST (hyphen): "AC-2 - Access Control", "AC-2(1) - Access Control Enhancement"
208
+ - SOC 2: "CC1.1 COSO Principle 1...", "PI1.5 The entity implements..."
209
+ - CIS: "1.1 Ensure...", "1.1.1 Ensure..."
210
+ - ISO: "A.5.1 Policies for...", "A.5.1.1 Policies..."
211
+
212
+ :param str control_name: Full control name from AWS
213
+ :return: Extracted control ID
214
+ :rtype: str
215
+ """
216
+ if not control_name:
217
+ return ""
218
+
219
+ # Define patterns in order of specificity
220
+ patterns = [
221
+ r"^([A-Z]{2,3}-\d+(?:\(\d+\))?):\s*", # NIST with colon
222
+ r"^([A-Z]{2,3}-\d+(?:\(\d+\))?)\s*-\s*", # NIST with hyphen
223
+ r"^([A-Z]{1,3}\d+\.\d+)\s+", # SOC 2
224
+ r"^(\d+(?:\.\d+){1,3})\s+", # CIS
225
+ r"^([A-Z]\.\d+(?:\.\d+){1,2})\s+", # ISO
226
+ r"^([A-Z]+\d+(?:\.\d+)*)\s+", # Generic alphanumeric with dots
227
+ ]
228
+
229
+ for pattern in patterns:
230
+ result = self._try_extract_with_pattern(control_name, pattern)
231
+ if result:
232
+ return result
233
+
234
+ logger.warning(f"Could not extract control ID from name: '{control_name}'")
235
+ return ""
236
+
237
+ def _parse_remediation_from_attributes(self, attributes: Dict[str, Any]) -> Dict[str, str]:
238
+ """
239
+ Parse remediation information from evidence attributes.
240
+
241
+ Security Hub evidence includes remediation info in attributes.findingRemediation
242
+ as a JSON string: {"recommendation": {"text": "...", "url": "..."}}
243
+
244
+ :param Dict[str, Any] attributes: Evidence attributes dictionary
245
+ :return: Dictionary with 'url' and 'text' keys, or empty dict if not available
246
+ :rtype: Dict[str, str]
247
+ """
248
+ import json
249
+
250
+ remediation_str = attributes.get("findingRemediation")
251
+ if not remediation_str:
252
+ return {}
253
+
254
+ try:
255
+ # Parse the JSON string
256
+ remediation_data = json.loads(remediation_str)
257
+ recommendation = remediation_data.get("recommendation", {})
258
+
259
+ return {"url": recommendation.get("url", ""), "text": recommendation.get("text", "")}
260
+ except (json.JSONDecodeError, AttributeError, TypeError) as ex:
261
+ logger.debug(f"Failed to parse remediation from attributes: {ex}")
262
+ return {}
263
+
264
+ def _parse_severity_from_attributes(self, attributes: Dict[str, Any]) -> Dict[str, Any]:
265
+ """
266
+ Parse severity information from evidence attributes.
267
+
268
+ Security Hub evidence includes severity in attributes.findingSeverity
269
+ as a JSON string: {"label": "CRITICAL", "normalized": 90, "original": "CRITICAL"}
270
+
271
+ :param Dict[str, Any] attributes: Evidence attributes dictionary
272
+ :return: Dictionary with 'label', 'normalized', and 'original' keys, or empty dict
273
+ :rtype: Dict[str, Any]
274
+ """
275
+ import json
276
+
277
+ severity_str = attributes.get("findingSeverity")
278
+ if not severity_str:
279
+ return {}
280
+
281
+ try:
282
+ # Parse the JSON string
283
+ severity_data = json.loads(severity_str)
284
+ return {
285
+ "label": severity_data.get("label", ""),
286
+ "normalized": severity_data.get("normalized", 0),
287
+ "original": severity_data.get("original", ""),
288
+ }
289
+ except (json.JSONDecodeError, AttributeError, TypeError) as ex:
290
+ logger.debug(f"Failed to parse severity from attributes: {ex}")
291
+ return {}
292
+
293
+ def _extract_control_name_from_attributes(self, attributes: Dict[str, Any], data_source: str) -> str:
294
+ """
295
+ Extract control or rule name from evidence attributes.
296
+
297
+ For Security Hub: extract and clean findingTitle
298
+ For Config: extract configRuleName or managedRuleIdentifier
299
+
300
+ :param Dict[str, Any] attributes: Evidence attributes dictionary
301
+ :param str data_source: Evidence data source (e.g., "AWS Security Hub", "AWS Config")
302
+ :return: Control/rule name, or empty string if not available
303
+ :rtype: str
304
+ """
305
+ import json
306
+
307
+ if "Security Hub" in data_source:
308
+ # Extract Security Hub control title
309
+ title_str = attributes.get("findingTitle", "")
310
+ if title_str:
311
+ # Remove surrounding quotes if present
312
+ try:
313
+ # Try to parse as JSON string first (may be quoted)
314
+ title = json.loads(title_str) if title_str.startswith('"') else title_str
315
+ return title.strip()
316
+ except (json.JSONDecodeError, AttributeError):
317
+ return title_str.strip().strip('"')
318
+
319
+ elif "Config" in data_source:
320
+ # Extract Config rule name
321
+ rule_name = attributes.get("configRuleName", "")
322
+ if rule_name:
323
+ # Remove surrounding quotes if present
324
+ rule_name = rule_name.strip().strip('"')
325
+ return rule_name
326
+
327
+ # Fall back to managed rule identifier
328
+ managed_id = attributes.get("managedRuleIdentifier", "")
329
+ if managed_id:
330
+ managed_id = managed_id.strip().strip('"')
331
+ return managed_id
332
+
333
+ return ""
334
+
335
+ def _extract_failing_resources(self, evidence_items: List[Dict[str, Any]]) -> List[Dict[str, str]]:
336
+ """
337
+ Extract all failing resources from evidence items.
338
+
339
+ Returns list of resources with FAILED or NON_COMPLIANT status,
340
+ including their ARN, source (control/rule name), and severity.
341
+
342
+ :param List[Dict[str, Any]] evidence_items: List of evidence items
343
+ :return: List of failing resource dictionaries
344
+ :rtype: List[Dict[str, str]]
345
+ """
346
+ failing_resources = []
347
+
348
+ for evidence in evidence_items:
349
+ # Get compliance check result
350
+ compliance = self._get_evidence_compliance(evidence)
351
+ if compliance != "FAILED":
352
+ continue
353
+
354
+ # Extract data source and attributes
355
+ data_source = evidence.get("dataSource", "Unknown")
356
+ attributes = evidence.get("attributes", {})
357
+
358
+ # Get control/rule name
359
+ control_name = self._extract_control_name_from_attributes(attributes, data_source)
360
+
361
+ # Get severity
362
+ severity_info = self._parse_severity_from_attributes(attributes)
363
+ severity_label = severity_info.get("label", "MEDIUM") if severity_info else "MEDIUM"
364
+
365
+ # Extract failing resources from resourcesIncluded
366
+ resources_included = evidence.get("resourcesIncluded", [])
367
+ for resource in resources_included:
368
+ resource_check = resource.get("complianceCheck")
369
+ # Check if this specific resource failed
370
+ if resource_check in ("FAILED", "FAIL", "NON_COMPLIANT", "Non-compliant"):
371
+ arn = resource.get("arn", "Unknown ARN")
372
+ failing_resources.append(
373
+ {"arn": arn, "source": f"{data_source}: {control_name}", "severity": severity_label}
374
+ )
375
+
376
+ return failing_resources
377
+
378
+ @property
379
+ def control_id(self) -> str:
380
+ """Control identifier (e.g., AC-3, SI-2)."""
381
+ # The control ID is already in the correct format from the name field
382
+ # Just return it directly
383
+ return self._control_id if self._control_id else ""
384
+
385
+ def _extract_resource_compliance(self, resources_included: List[Dict[str, Any]]) -> Optional[str]:
386
+ """
387
+ Extract compliance status from resourcesIncluded list.
388
+
389
+ :param List[Dict[str, Any]] resources_included: List of resources
390
+ :return: "COMPLIANT", "FAILED", "NOT_APPLICABLE", or None
391
+ :rtype: Optional[str]
392
+ """
393
+ if not resources_included:
394
+ return None
395
+
396
+ resource_checks = [r.get("complianceCheck") for r in resources_included]
397
+
398
+ # Check for any failure values (case-insensitive)
399
+ failure_values = {"FAILED", "FAIL", "Fail", "NON_COMPLIANT", "Non-compliant"}
400
+ if any(check in failure_values for check in resource_checks):
401
+ return "FAILED"
402
+
403
+ # Check for any success values (case-insensitive)
404
+ if any(check in {"COMPLIANT", "PASS", "Pass"} for check in resource_checks):
405
+ return "COMPLIANT"
406
+
407
+ if any(check == "NOT_APPLICABLE" for check in resource_checks):
408
+ return "NOT_APPLICABLE"
409
+
410
+ return None
411
+
412
+ def _normalize_compliance_value(self, compliance_check: str, evidence: Dict[str, Any]) -> Optional[str]:
413
+ """
414
+ Normalize compliance check value to standard format.
415
+
416
+ :param str compliance_check: Raw compliance check value
417
+ :param Dict[str, Any] evidence: Evidence item for logging context
418
+ :return: Normalized compliance value or None
419
+ :rtype: Optional[str]
420
+ """
421
+ if not isinstance(compliance_check, str):
422
+ logger.warning(
423
+ f"Control {self.control_id}: Invalid complianceCheck type {type(compliance_check).__name__}: "
424
+ f"{compliance_check}. Treating as inconclusive."
425
+ )
426
+ return None
427
+
428
+ compliance_upper = compliance_check.upper().replace("-", "_").replace(" ", "_")
429
+
430
+ # Normalize failure values
431
+ if compliance_upper in {"FAILED", "FAIL", "NON_COMPLIANT"}:
432
+ if compliance_check != "FAILED":
433
+ logger.debug(
434
+ f"Control {self.control_id}: Normalized '{compliance_check}' to FAILED "
435
+ f"(source: {evidence.get('dataSource', 'unknown')})"
436
+ )
437
+ return "FAILED"
438
+
439
+ # Normalize success values
440
+ if compliance_upper == "PASS":
441
+ logger.debug(
442
+ f"Control {self.control_id}: Normalized PASS to COMPLIANT "
443
+ f"(source: {evidence.get('dataSource', 'unknown')})"
444
+ )
445
+ return "COMPLIANT"
446
+
447
+ return compliance_check
448
+
449
+ def _get_evidence_compliance(self, evidence: Dict[str, Any]) -> Optional[str]:
450
+ """
451
+ Extract compliance check result from a single evidence item.
452
+
453
+ Checks both root-level and resource-level complianceCheck fields.
454
+
455
+ AWS Audit Manager uses different values depending on the source:
456
+ Success values (normalized to "COMPLIANT"):
457
+ - "COMPLIANT" - from AWS Config
458
+ - "PASS" - from AWS Security Hub
459
+
460
+ Failure values (normalized to "FAILED"):
461
+ - "NON_COMPLIANT" / "Non-compliant" - from AWS Config
462
+ - "FAILED" / "FAIL" / "Fail" - from AWS Security Hub
463
+
464
+ :param Dict[str, Any] evidence: Evidence item
465
+ :return: "COMPLIANT", "FAILED", "NOT_APPLICABLE", or None
466
+ :rtype: Optional[str]
467
+ """
468
+ # Check root-level complianceCheck first
469
+ compliance_check = evidence.get("complianceCheck")
470
+
471
+ # If no root-level check, look in resourcesIncluded
472
+ if compliance_check is None:
473
+ resources_included = evidence.get("resourcesIncluded", [])
474
+ compliance_check = self._extract_resource_compliance(resources_included)
475
+
476
+ # Normalize all variations to standard values
477
+ if compliance_check:
478
+ compliance_check = self._normalize_compliance_value(compliance_check, evidence)
479
+
480
+ return compliance_check
481
+
482
+ def _log_inconclusive_status(self, not_applicable_count: int, inconclusive_count: int, total_evidence: int) -> None:
483
+ """Log inconclusive status with appropriate message."""
484
+ if not_applicable_count > 0 and inconclusive_count == 0:
485
+ logger.info(
486
+ f"Control {self.control_id}: All {not_applicable_count} evidence item(s) marked as NOT_APPLICABLE. "
487
+ "Control status will not be updated."
488
+ )
489
+ elif not_applicable_count > 0:
490
+ logger.info(
491
+ f"Control {self.control_id}: {not_applicable_count} NOT_APPLICABLE, "
492
+ f"{inconclusive_count} inconclusive evidence item(s). Control status will not be updated."
493
+ )
494
+ else:
495
+ logger.debug(
496
+ f"Control {self.control_id}: Evidence collected ({total_evidence} item(s)) but no valid compliance checks found. "
497
+ "Control status will not be updated. This may occur when evidence lacks complianceCheck fields."
498
+ )
499
+
500
+ def _count_evidence_by_status(self) -> tuple:
501
+ """
502
+ Count evidence items by compliance status.
503
+
504
+ :return: Tuple of (compliant_count, failed_count, inconclusive_count, not_applicable_count)
505
+ :rtype: tuple
506
+ """
507
+ compliant_count = 0
508
+ failed_count = 0
509
+ inconclusive_count = 0
510
+ not_applicable_count = 0
511
+
512
+ for evidence in self.evidence_items:
513
+ compliance_check = self._get_evidence_compliance(evidence)
514
+
515
+ if compliance_check == "FAILED":
516
+ failed_count += 1
517
+ elif compliance_check == "COMPLIANT":
518
+ compliant_count += 1
519
+ elif compliance_check == "NOT_APPLICABLE":
520
+ not_applicable_count += 1
521
+ else:
522
+ inconclusive_count += 1
523
+
524
+ return compliant_count, failed_count, inconclusive_count, not_applicable_count
525
+
526
+ def _determine_compliance_status(self, counts: tuple) -> Optional[str]:
527
+ """
528
+ Determine compliance status based on evidence counts.
529
+
530
+ :param tuple counts: Tuple of (compliant, failed, inconclusive, not_applicable) counts
531
+ :return: "PASS", "FAIL", "NOT_APPLICABLE", or None
532
+ :rtype: Optional[str]
533
+ """
534
+ compliant_count, failed_count, inconclusive_count, not_applicable_count = counts
535
+ total_evidence = len(self.evidence_items)
536
+
537
+ logger.debug(
538
+ f"Control {self.control_id} evidence summary: "
539
+ f"{failed_count} FAILED, {compliant_count} COMPLIANT, "
540
+ f"{not_applicable_count} NOT_APPLICABLE, {inconclusive_count} inconclusive out of {total_evidence} total"
541
+ )
542
+
543
+ # If ANY evidence failed, the control fails
544
+ if failed_count > 0:
545
+ logger.info(
546
+ f"Control {self.control_id} FAILS: {failed_count} failed evidence item(s) out of {total_evidence}"
547
+ )
548
+ return "FAIL"
549
+
550
+ # If we have compliant evidence and no failures, control passes
551
+ if compliant_count > 0:
552
+ if inconclusive_count > 0 or not_applicable_count > 0:
553
+ logger.info(
554
+ f"Control {self.control_id} PASSES: {compliant_count} compliant, "
555
+ f"{not_applicable_count} not applicable, {inconclusive_count} inconclusive (no failures)"
556
+ )
557
+ else:
558
+ logger.info(f"Control {self.control_id} PASSES: All {compliant_count} evidence items compliant")
559
+ return "PASS"
560
+
561
+ # If all evidence is not applicable, return NOT_APPLICABLE status
562
+ if not_applicable_count > 0 and inconclusive_count == 0:
563
+ logger.info(
564
+ f"Control {self.control_id}: All {not_applicable_count} evidence item(s) marked NOT_APPLICABLE. "
565
+ "Control will be marked as Not Applicable."
566
+ )
567
+ return "NOT_APPLICABLE"
568
+
569
+ # If all evidence is inconclusive or mix of not applicable and inconclusive
570
+ self._log_inconclusive_status(not_applicable_count, inconclusive_count, total_evidence)
571
+ return None
572
+
573
+ def _aggregate_evidence_compliance(self) -> Optional[str]:
574
+ """
575
+ Aggregate evidence complianceCheck fields to determine overall control compliance.
576
+
577
+ AWS Audit Manager evidence items contain a complianceCheck field with values that vary by source.
578
+ All values are normalized before aggregation:
579
+
580
+ Success values (normalized to "COMPLIANT"):
581
+ - "COMPLIANT" (AWS Config)
582
+ - "PASS" (AWS Security Hub)
583
+
584
+ Failure values (normalized to "FAILED"):
585
+ - "NON_COMPLIANT" / "Non-compliant" (AWS Config)
586
+ - "FAILED" / "FAIL" / "Fail" (AWS Security Hub)
587
+
588
+ Other values:
589
+ - "NOT_APPLICABLE": Evidence is not applicable to this control
590
+ - null/None: No compliance check available for this evidence
591
+
592
+ Evidence can have compliance checks in TWO locations:
593
+ 1. Root level: evidence["complianceCheck"]
594
+ 2. Resource level: evidence["resourcesIncluded"][*]["complianceCheck"]
595
+
596
+ This method checks BOTH locations to ensure accurate compliance determination.
597
+
598
+ Aggregation Logic (after normalization):
599
+ 1. If ANY evidence shows "FAILED" (any failure value) → Control FAILS
600
+ 2. If ALL evidence shows "COMPLIANT" (any success value) → Control PASSES
601
+ 3. NOT_APPLICABLE evidence is tracked separately and doesn't affect compliance
602
+ 4. If NO compliance checks available (all null/NOT_APPLICABLE) → INCONCLUSIVE
603
+ 5. If mixed (some COMPLIANT, some null, no FAILED) → PASS with warning
604
+
605
+ :return: "PASS", "FAIL", or None (if inconclusive/no evidence)
606
+ :rtype: Optional[str]
607
+ """
608
+ if not self.evidence_items:
609
+ logger.debug(f"Control {self.control_id}: No evidence items available for aggregation")
610
+ return None
611
+
612
+ # Use enhanced analyzer if enabled
613
+ if self.use_enhanced_analyzer:
614
+ logger.debug(f"Control {self.control_id}: Using enhanced ControlComplianceAnalyzer")
615
+ return self._aggregate_evidence_with_analyzer()
616
+
617
+ # Count evidence by status
618
+ counts = self._count_evidence_by_status()
619
+
620
+ # Determine compliance based on counts
621
+ return self._determine_compliance_status(counts)
622
+
623
+ def _aggregate_evidence_with_analyzer(self) -> Optional[str]:
624
+ """
625
+ Use enhanced ControlComplianceAnalyzer to determine control compliance.
626
+
627
+ This method leverages the ControlComplianceAnalyzer for more sophisticated
628
+ evidence analysis, providing compliance scores and confidence levels.
629
+
630
+ :return: "PASS", "FAIL", "NOT_APPLICABLE", or None
631
+ :rtype: Optional[str]
632
+ """
633
+ analyzer = ControlComplianceAnalyzer(control_id=self.control_id)
634
+
635
+ # Add each evidence item to the analyzer
636
+ for evidence in self.evidence_items:
637
+ analyzer.add_evidence_insight(evidence)
638
+
639
+ # Get the compliance determination
640
+ status, details = analyzer.determine_control_status()
641
+
642
+ # Store enhanced analysis results
643
+ self.compliance_score = analyzer.get_compliance_score()
644
+ self.confidence_level = analyzer.get_confidence_level()
645
+ self.compliance_details = details
646
+
647
+ # Get comprehensive analysis
648
+ analysis = analyzer.get_compliance_analysis()
649
+
650
+ # Log detailed analysis
651
+ logger.info(
652
+ f"Control {self.control_id} analysis: "
653
+ f"status={status}, score={self.compliance_score:.2f}, "
654
+ f"confidence={self.confidence_level * 100:.0f}, "
655
+ f"compliant={analysis.compliant_evidence_count}, "
656
+ f"noncompliant={analysis.noncompliant_evidence_count}, "
657
+ f"inconclusive={analysis.inconclusive_evidence_count}"
658
+ )
659
+
660
+ # Note: The ComplianceIntegration base class will automatically map this status
661
+ # to the appropriate value based on the security plan's compliance settings.
662
+ # For example:
663
+ # - DoD/RMF: PASS → "Implemented", FAIL → "Planned"
664
+ # - FedRAMP: PASS → "Fully Implemented", FAIL → "In Remediation"
665
+ logger.debug(
666
+ f"Control {self.control_id}: Result '{status}' will be mapped to compliance-specific status "
667
+ "by ComplianceIntegration._get_implementation_status_from_result()"
668
+ )
669
+
670
+ # Map analyzer status to expected return values
671
+ if status == "PASS":
672
+ return "PASS"
673
+ elif status == "FAIL":
674
+ return "FAIL"
675
+ elif status == "NOT_APPLICABLE":
676
+ return "NOT_APPLICABLE"
677
+ elif status == "INCONCLUSIVE":
678
+ logger.debug(
679
+ f"Control {self.control_id}: Inconclusive evidence. Details: {details.get('reason', 'Unknown')}"
680
+ )
681
+ return None
682
+ else: # NO_DATA
683
+ logger.debug(f"Control {self.control_id}: No evidence data available for assessment")
684
+ return None
685
+
686
+ @property
687
+ def compliance_result(self) -> Optional[str]:
688
+ """
689
+ Result of compliance check (PASS, FAIL, etc).
690
+
691
+ IMPORTANT: AWS Audit Manager control 'status' (REVIEWED/UNDER_REVIEW/INACTIVE) is a
692
+ WORKFLOW STATUS, not a compliance result. The actual compliance determination requires
693
+ analyzing the individual evidence items' 'complianceCheck' fields.
694
+
695
+ This property aggregates evidence to determine actual compliance:
696
+ 1. Collects all evidence items' complianceCheck fields (COMPLIANT/FAILED)
697
+ 2. Determines overall control compliance (if ANY evidence FAILED -> control FAILS)
698
+ 3. Returns PASS if all evidence is compliant, FAIL if any failures
699
+
700
+ If no evidence is available, returns None. The control status should NOT be updated
701
+ when evidence is unavailable - this signals the integration to skip the control.
702
+
703
+ :return: "PASS", "FAIL", or None (if no evidence available)
704
+ :rtype: Optional[str]
705
+ """
706
+ # Use cached result if available (including None)
707
+ if self._aggregated_compliance_result is not None or hasattr(self, "_result_was_cached"):
708
+ return self._aggregated_compliance_result
709
+
710
+ # Aggregate evidence compliance checks
711
+ result = self._aggregate_evidence_compliance()
712
+
713
+ if result is None:
714
+ # No evidence or no compliance checks available
715
+ # Return None to signal that control should not be updated
716
+ if len(self.evidence_items) > 0:
717
+ logger.debug(
718
+ f"Control {self.control_id}: Evidence items collected ({len(self.evidence_items)}) but cannot determine "
719
+ f"compliance status (no valid complianceCheck values found). Control status will not be updated."
720
+ )
721
+ else:
722
+ logger.debug(
723
+ f"Control {self.control_id}: No evidence items available for compliance determination. "
724
+ f"Control status will not be updated. Metadata evidence count: {self.evidence_count}"
725
+ )
726
+
727
+ # Cache the result (including None)
728
+ self._aggregated_compliance_result = result
729
+ self._result_was_cached = True
730
+ return result
731
+
732
+ @property
733
+ def severity(self) -> Optional[str]:
734
+ """
735
+ Severity level of the compliance violation (if failed).
736
+
737
+ Returns the highest severity from failing evidence items, or None if control passed.
738
+ """
739
+ if self.compliance_result != "FAIL":
740
+ return None
741
+
742
+ # Use highest_severity from evidence if available
743
+ return self.highest_severity
744
+
745
+ @property
746
+ def remediation_urls(self) -> List[str]:
747
+ """
748
+ List of unique remediation URLs from failing evidence.
749
+
750
+ Extracts remediation URLs from Security Hub findings in evidence attributes.
751
+ """
752
+ urls = set()
753
+ for evidence in self.evidence_items:
754
+ # Only process failing evidence
755
+ if self._get_evidence_compliance(evidence) != "FAILED":
756
+ continue
757
+
758
+ attributes = evidence.get("attributes", {})
759
+ remediation_info = self._parse_remediation_from_attributes(attributes)
760
+ if remediation_info.get("url"):
761
+ urls.add(remediation_info["url"])
762
+
763
+ return sorted(urls)
764
+
765
+ @property
766
+ def remediation_info(self) -> List[Dict[str, str]]:
767
+ """
768
+ List of remediation information from failing evidence.
769
+
770
+ Returns list of dicts with 'url' and 'text' keys.
771
+ """
772
+ remediation_list = []
773
+ seen_urls = set()
774
+
775
+ for evidence in self.evidence_items:
776
+ # Only process failing evidence
777
+ if self._get_evidence_compliance(evidence) != "FAILED":
778
+ continue
779
+
780
+ attributes = evidence.get("attributes", {})
781
+ remediation_info = self._parse_remediation_from_attributes(attributes)
782
+
783
+ # Only add if we have a URL and haven't seen it before
784
+ url = remediation_info.get("url")
785
+ if url and url not in seen_urls:
786
+ remediation_list.append(remediation_info)
787
+ seen_urls.add(url)
788
+
789
+ return remediation_list
790
+
791
+ @property
792
+ def failing_resources(self) -> List[Dict[str, str]]:
793
+ """
794
+ List of resources that failed compliance checks.
795
+
796
+ Returns list of dicts with 'arn', 'source', and 'severity' keys.
797
+ """
798
+ return self._extract_failing_resources(self.evidence_items)
799
+
800
+ @property
801
+ def underlying_checks(self) -> List[str]:
802
+ """
803
+ List of underlying Security Hub controls and Config rules that were evaluated.
804
+
805
+ Returns list of control/rule names from failing evidence.
806
+ """
807
+ checks = set()
808
+
809
+ for evidence in self.evidence_items:
810
+ # Only process failing evidence
811
+ if self._get_evidence_compliance(evidence) != "FAILED":
812
+ continue
813
+
814
+ data_source = evidence.get("dataSource", "")
815
+ attributes = evidence.get("attributes", {})
816
+ control_name = self._extract_control_name_from_attributes(attributes, data_source)
817
+
818
+ if control_name:
819
+ checks.add(f"{data_source}: {control_name}")
820
+
821
+ return sorted(checks)
822
+
823
+ @property
824
+ def highest_severity(self) -> str:
825
+ """
826
+ Highest severity level from failing evidence items.
827
+
828
+ Returns CRITICAL, HIGH, MEDIUM, LOW, or INFORMATIONAL.
829
+ Defaults to MEDIUM if no severity information available.
830
+ """
831
+ severity_rankings = {"CRITICAL": 5, "HIGH": 4, "MEDIUM": 3, "LOW": 2, "INFORMATIONAL": 1}
832
+
833
+ highest = "MEDIUM" # Default
834
+ highest_rank = severity_rankings.get(highest, 0)
835
+
836
+ for evidence in self.evidence_items:
837
+ # Only process failing evidence
838
+ if self._get_evidence_compliance(evidence) != "FAILED":
839
+ continue
840
+
841
+ attributes = evidence.get("attributes", {})
842
+ severity_info = self._parse_severity_from_attributes(attributes)
843
+ severity_label = severity_info.get("label", "").upper()
844
+
845
+ if severity_label in severity_rankings:
846
+ rank = severity_rankings[severity_label]
847
+ if rank > highest_rank:
848
+ highest = severity_label
849
+ highest_rank = rank
850
+
851
+ return highest
852
+
853
+ @property
854
+ def severity_score(self) -> int:
855
+ """
856
+ Highest normalized severity score (0-100) from failing evidence items.
857
+
858
+ Returns 0-100 where CRITICAL=90, HIGH=70, MEDIUM=40, LOW=30, INFORMATIONAL=10.
859
+ """
860
+ highest_score = 0
861
+
862
+ for evidence in self.evidence_items:
863
+ # Only process failing evidence
864
+ if self._get_evidence_compliance(evidence) != "FAILED":
865
+ continue
866
+
867
+ attributes = evidence.get("attributes", {})
868
+ severity_info = self._parse_severity_from_attributes(attributes)
869
+ normalized = severity_info.get("normalized", 0)
870
+
871
+ if normalized > highest_score:
872
+ highest_score = normalized
873
+
874
+ return highest_score
875
+
876
+ def _add_compliance_assessment_section(self, desc_parts: list) -> None:
877
+ """Add compliance assessment section to description."""
878
+ compliance_result = self.compliance_result
879
+ compliant_count = sum(1 for e in self.evidence_items if self._get_evidence_compliance(e) == "COMPLIANT")
880
+ failed_count = sum(1 for e in self.evidence_items if self._get_evidence_compliance(e) == "FAILED")
881
+
882
+ desc_parts.append(f"{HTML_H4_OPEN}Compliance Assessment{HTML_H4_CLOSE}")
883
+ desc_parts.append(HTML_P_OPEN)
884
+
885
+ if compliance_result == "FAIL":
886
+ desc_parts.append(
887
+ f"<span style='color: red;'>{HTML_STRONG_OPEN}Result: FAILED{HTML_STRONG_CLOSE}</span>{HTML_BR}"
888
+ )
889
+ desc_parts.append(
890
+ f"This control has {HTML_STRONG_OPEN}{failed_count} failed evidence item(s){HTML_STRONG_CLOSE} "
891
+ f"out of {len(self.evidence_items)} total.{HTML_BR}"
892
+ )
893
+ if compliant_count > 0:
894
+ desc_parts.append(f"{compliant_count} evidence item(s) are compliant. ")
895
+ elif compliance_result == "PASS":
896
+ desc_parts.append(
897
+ f"<span style='color: green;'>{HTML_STRONG_OPEN}Result: PASSED{HTML_STRONG_CLOSE}</span>{HTML_BR}"
898
+ )
899
+ desc_parts.append(f"All {compliant_count} evidence item(s) with compliance checks are compliant.")
900
+ else:
901
+ desc_parts.append(f"{HTML_STRONG_OPEN}Result: INCONCLUSIVE{HTML_STRONG_CLOSE}{HTML_BR}")
902
+ desc_parts.append("Evidence collected but compliance status could not be determined.")
903
+
904
+ desc_parts.append(HTML_P_CLOSE)
905
+
906
+ def _add_remediation_section(self, desc_parts: list) -> None:
907
+ """Add remediation section to description."""
908
+ if not (self.action_plan_title or self.action_plan_instructions):
909
+ return
910
+
911
+ desc_parts.append(f"{HTML_H4_OPEN}Remediation{HTML_H4_CLOSE}")
912
+ if self.action_plan_title:
913
+ desc_parts.append(
914
+ f"{HTML_P_OPEN}{HTML_STRONG_OPEN}Action Plan:{HTML_STRONG_CLOSE} {self.action_plan_title}"
915
+ f"{HTML_P_CLOSE}"
916
+ )
917
+ if self.action_plan_instructions:
918
+ desc_parts.append(
919
+ f"{HTML_P_OPEN}{HTML_STRONG_OPEN}Remediation Steps:{HTML_STRONG_CLOSE}{HTML_BR}"
920
+ f"{self.action_plan_instructions}{HTML_P_CLOSE}"
921
+ )
922
+
923
+ def _add_comments_section(self, desc_parts: list) -> None:
924
+ """Add assessor comments section to description."""
925
+ if not self.control_comments:
926
+ return
927
+
928
+ desc_parts.append(f"{HTML_H4_OPEN}Assessor Comments{HTML_H4_CLOSE}")
929
+ desc_parts.append(HTML_UL_OPEN)
930
+ for comment in self.control_comments[:5]: # Show up to 5 comments
931
+ author = comment.get("authorName", "Unknown")
932
+ posted_date = comment.get("postedDate", "")
933
+ comment_body = comment.get("commentBody", "")
934
+ desc_parts.append(
935
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}{author}{HTML_STRONG_CLOSE} ({posted_date}): {comment_body} {HTML_LI_CLOSE}"
936
+ )
937
+ desc_parts.append(HTML_UL_CLOSE)
938
+
939
+ def _add_failed_resources_section(self, desc_parts: list) -> None:
940
+ """Add failed resources section with ARNs, sources, and severity."""
941
+ failing_resources = self.failing_resources
942
+ if not failing_resources:
943
+ return
944
+
945
+ desc_parts.append(f"{HTML_H4_OPEN}Failed Resources{HTML_H4_CLOSE}")
946
+ desc_parts.append(f"{HTML_P_OPEN}The following resources failed compliance checks:{HTML_P_CLOSE}")
947
+
948
+ # Create HTML table
949
+ desc_parts.append('<table style="border-collapse: collapse; width: 100%;">')
950
+ desc_parts.append(
951
+ '<tr style="background-color: #f2f2f2;">'
952
+ '<th style="border: 1px solid #ddd; padding: 8px; text-align: left;">Resource ARN</th>'
953
+ '<th style="border: 1px solid #ddd; padding: 8px; text-align: left;">Source</th>'
954
+ '<th style="border: 1px solid #ddd; padding: 8px; text-align: left;">Severity</th>'
955
+ "</tr>"
956
+ )
957
+
958
+ for resource in failing_resources:
959
+ arn = resource.get("arn", "Unknown")
960
+ source = resource.get("source", "Unknown")
961
+ severity = resource.get("severity", "MEDIUM")
962
+
963
+ # Color-code severity
964
+ severity_colors = {
965
+ "CRITICAL": "#d32f2f",
966
+ "HIGH": "#f57c00",
967
+ "MEDIUM": "#fbc02d",
968
+ "LOW": "#7cb342",
969
+ "INFORMATIONAL": "#1976d2",
970
+ }
971
+ severity_color = severity_colors.get(severity, "#757575")
972
+
973
+ desc_parts.append(
974
+ f'<tr><td style="border: 1px solid #ddd; padding: 8px;">{arn}</td>'
975
+ f'<td style="border: 1px solid #ddd; padding: 8px;">{source}</td>'
976
+ f'<td style="border: 1px solid #ddd; padding: 8px; color: {severity_color}; font-weight: bold;">{severity}</td></tr>'
977
+ )
978
+
979
+ desc_parts.append("</table>")
980
+
981
+ def _transform_remediation_url(self, url: str) -> str:
982
+ """
983
+ Transform AWS documentation URLs from console format to proper user guide format.
984
+
985
+ AWS Audit Manager provides remediation URLs in this format:
986
+ https://docs.aws.amazon.com/console/securityhub/Config.1/remediation
987
+
988
+ But these should be transformed to the actual documentation:
989
+ https://docs.aws.amazon.com/securityhub/latest/userguide/securityhub-awsconfig-controls.html#securityhub-awsconfig-1
990
+
991
+ :param str url: Original URL from AWS Audit Manager
992
+ :return: Transformed URL to actual AWS documentation
993
+ :rtype: str
994
+ """
995
+ import re
996
+
997
+ if not url:
998
+ return url
999
+
1000
+ # Pattern: https://docs.aws.amazon.com/console/securityhub/<ControlID>/remediation
1001
+ console_pattern = r"https://docs\.aws\.amazon\.com/console/securityhub/([^/]+)/remediation"
1002
+ match = re.match(console_pattern, url)
1003
+
1004
+ if match:
1005
+ control_id = match.group(1) # e.g., "Config.1" or "EC2.1"
1006
+
1007
+ # Parse control prefix and number
1008
+ # Examples: "Config.1" -> ("Config", "1"), "EC2.15" -> ("EC2", "15")
1009
+ control_match = re.match(r"([A-Za-z0-9]+)\.(\d+)", control_id)
1010
+
1011
+ if control_match:
1012
+ prefix = control_match.group(1).lower() # "config" or "ec2"
1013
+ number = control_match.group(2) # "1" or "15"
1014
+
1015
+ # Map service prefixes to documentation pages
1016
+ # Security Hub organizes controls by AWS service
1017
+ service_doc_map = {
1018
+ "config": "awsconfig",
1019
+ "ec2": "ec2",
1020
+ "iam": "iam",
1021
+ "s3": "s3",
1022
+ "rds": "rds",
1023
+ "lambda": "lambda",
1024
+ "cloudtrail": "cloudtrail",
1025
+ "guardduty": "guardduty",
1026
+ "securityhub": "securityhub",
1027
+ "kms": "kms",
1028
+ "elb": "elb",
1029
+ "elbv2": "elbv2",
1030
+ "cloudwatch": "cloudwatch",
1031
+ "sns": "sns",
1032
+ "sqs": "sqs",
1033
+ "autoscaling": "autoscaling",
1034
+ "codebuild": "codebuild",
1035
+ "dms": "dms",
1036
+ "dynamodb": "dynamodb",
1037
+ "ecs": "ecs",
1038
+ "efs": "efs",
1039
+ "elasticache": "elasticache",
1040
+ "elasticsearch": "elasticsearch",
1041
+ "emr": "emr",
1042
+ "redshift": "redshift",
1043
+ "sagemaker": "sagemaker",
1044
+ "secretsmanager": "secretsmanager",
1045
+ "ssm": "ssm",
1046
+ "waf": "waf",
1047
+ }
1048
+
1049
+ doc_prefix = service_doc_map.get(prefix, prefix)
1050
+
1051
+ # Construct the proper documentation URL
1052
+ transformed_url = (
1053
+ f"https://docs.aws.amazon.com/securityhub/latest/userguide/"
1054
+ f"securityhub-{doc_prefix}-controls.html#securityhub-{doc_prefix}-{number}"
1055
+ )
1056
+
1057
+ logger.debug(f"Transformed remediation URL: {url} -> {transformed_url}")
1058
+ return transformed_url
1059
+
1060
+ # If pattern doesn't match or control ID can't be parsed, return original URL
1061
+ logger.debug(f"Could not transform remediation URL (returning original): {url}")
1062
+ return url
1063
+
1064
+ def _add_remediation_resources_section(self, desc_parts: list) -> None:
1065
+ """Add remediation resources section with links."""
1066
+ remediation_info = self.remediation_info
1067
+ if not remediation_info:
1068
+ return
1069
+
1070
+ desc_parts.append(f"{HTML_H4_OPEN}Remediation Resources{HTML_H4_CLOSE}")
1071
+ desc_parts.append(HTML_UL_OPEN)
1072
+
1073
+ for remediation in remediation_info:
1074
+ url = remediation.get("url", "")
1075
+ text = remediation.get("text", "See remediation guidance")
1076
+
1077
+ if url:
1078
+ # Transform console URLs to proper documentation URLs
1079
+ transformed_url = self._transform_remediation_url(url)
1080
+ desc_parts.append(
1081
+ f'{HTML_LI_OPEN}<a href="{transformed_url}" target="_blank">{text}</a>{HTML_LI_CLOSE}'
1082
+ )
1083
+
1084
+ desc_parts.append(HTML_UL_CLOSE)
1085
+
1086
+ def _add_underlying_checks_section(self, desc_parts: list) -> None:
1087
+ """Add underlying Security Hub controls and Config rules section."""
1088
+ underlying_checks = self.underlying_checks
1089
+ if not underlying_checks:
1090
+ return
1091
+
1092
+ desc_parts.append(f"{HTML_H4_OPEN}Underlying Security Checks{HTML_H4_CLOSE}")
1093
+ desc_parts.append(f"{HTML_P_OPEN}The following AWS checks contributed to this assessment:{HTML_P_CLOSE}")
1094
+ desc_parts.append(HTML_UL_OPEN)
1095
+
1096
+ for check in underlying_checks:
1097
+ desc_parts.append(f"{HTML_LI_OPEN}{check}{HTML_LI_CLOSE}")
1098
+
1099
+ desc_parts.append(HTML_UL_CLOSE)
1100
+
1101
+ @property
1102
+ def description(self) -> str:
1103
+ """Description of the compliance check using HTML formatting."""
1104
+ desc_parts = [
1105
+ f"<h3>AWS Audit Manager assessment for control {self.control_id}</h3>",
1106
+ HTML_P_OPEN,
1107
+ f"{HTML_STRONG_OPEN}Control:{HTML_STRONG_CLOSE} {self._control_name}{HTML_BR}",
1108
+ f"{HTML_STRONG_OPEN}Framework:{HTML_STRONG_CLOSE} {self.framework_name}{HTML_BR}",
1109
+ f"{HTML_STRONG_OPEN}Assessment:{HTML_STRONG_CLOSE} {self.assessment_name}{HTML_BR}",
1110
+ f"{HTML_STRONG_OPEN}Status:{HTML_STRONG_CLOSE} {self.control_status}{HTML_BR}",
1111
+ f"{HTML_STRONG_OPEN}Evidence Count:{HTML_STRONG_CLOSE} {self.evidence_count}",
1112
+ HTML_P_CLOSE,
1113
+ ]
1114
+
1115
+ if self.control_response:
1116
+ desc_parts.extend(
1117
+ [HTML_P_OPEN, f"{HTML_STRONG_OPEN}Response:{HTML_STRONG_CLOSE} {self.control_response}", HTML_P_CLOSE]
1118
+ )
1119
+
1120
+ # Add compliance result analysis if evidence is available
1121
+ if self.evidence_items:
1122
+ self._add_compliance_assessment_section(desc_parts)
1123
+
1124
+ # Add failed resources section (only if control failed)
1125
+ if self.compliance_result == "FAIL":
1126
+ self._add_failed_resources_section(desc_parts)
1127
+ self._add_underlying_checks_section(desc_parts)
1128
+ self._add_remediation_resources_section(desc_parts)
1129
+
1130
+ # Add remediation guidance if available
1131
+ self._add_remediation_section(desc_parts)
1132
+
1133
+ # Add testing information if available
1134
+ if self.testing_information:
1135
+ desc_parts.extend(
1136
+ [
1137
+ f"{HTML_H4_OPEN}Testing Guidance{HTML_H4_CLOSE}",
1138
+ f"{HTML_P_OPEN}{self.testing_information}{HTML_P_CLOSE}",
1139
+ ]
1140
+ )
1141
+
1142
+ # Add comments from assessors
1143
+ self._add_comments_section(desc_parts)
1144
+
1145
+ return "\n".join(desc_parts)
1146
+
1147
+ @property
1148
+ def framework(self) -> str:
1149
+ """Compliance framework (e.g., NIST800-53R5, CSF)."""
1150
+ framework_mappings = {
1151
+ "NIST SP 800-53 Revision 5": "NIST800-53R5",
1152
+ "NIST SP 800-53 Rev 5": "NIST800-53R5",
1153
+ "NIST 800-53 R5": "NIST800-53R5",
1154
+ "NIST 800-53 Revision 5": "NIST800-53R5",
1155
+ "SOC2": "SOC2",
1156
+ "PCI DSS": "PCI DSS",
1157
+ "HIPAA": "HIPAA",
1158
+ "GDPR": "GDPR",
1159
+ }
1160
+ if not self.framework_name:
1161
+ return "NIST800-53R5"
1162
+ for key, value in framework_mappings.items():
1163
+ if key.lower() in self.framework_name.lower():
1164
+ return value
1165
+ # Return framework name directly for custom frameworks
1166
+ # Check framework_type first (STANDARD vs CUSTOM)
1167
+ if self.framework_type == "CUSTOM":
1168
+ return self.framework_name
1169
+ # For unknown standard frameworks, return the name as-is
1170
+ return self.framework_name
1171
+
1172
+ def _format_control_parts(self, prefix: str, base_num: str, enhancement: Optional[str] = None) -> str:
1173
+ """
1174
+ Format control ID parts into standard RegScale format.
1175
+
1176
+ :param str prefix: Control prefix (e.g., AC, SI)
1177
+ :param str base_num: Base number (e.g., 2, 3)
1178
+ :param Optional[str] enhancement: Enhancement number (e.g., 1, 4)
1179
+ :return: Formatted control ID
1180
+ :rtype: str
1181
+ """
1182
+ # Remove leading zeros
1183
+ base = str(int(base_num))
1184
+ if enhancement:
1185
+ enh = str(int(enhancement))
1186
+ return f"{prefix}-{base}({enh})"
1187
+ return f"{prefix}-{base}"
1188
+
1189
+ def _try_parse_dot_notation(self, control_id: str) -> Optional[str]:
1190
+ """
1191
+ Try to parse dot notation format: AC.2.1 or AC.2.
1192
+
1193
+ :param str control_id: Control ID to parse
1194
+ :return: Normalized control ID or None
1195
+ :rtype: Optional[str]
1196
+ """
1197
+ import re
1198
+
1199
+ dot_pattern = r"^([A-Z]{2,3})\.(\d+)(?:\.(\d+))?$"
1200
+ match = re.match(dot_pattern, control_id)
1201
+ if match:
1202
+ return self._format_control_parts(match.group(1), match.group(2), match.group(3))
1203
+ return None
1204
+
1205
+ def _try_parse_standard_format(self, control_id: str) -> Optional[str]:
1206
+ """
1207
+ Try to parse standard format: AC-2(1), AC-2 (1), AC-2-1, AC-2.1.
1208
+
1209
+ :param str control_id: Control ID to parse
1210
+ :return: Normalized control ID or None
1211
+ :rtype: Optional[str]
1212
+ """
1213
+ import re
1214
+
1215
+ pattern = r"^([A-Z]{2,3})-(\d+)(?:[\s\-\.](\d+)|\s?\((\d+)\))?$"
1216
+ match = re.match(pattern, control_id)
1217
+ if match:
1218
+ enhancement = match.group(3) or match.group(4)
1219
+ return self._format_control_parts(match.group(1), match.group(2), enhancement)
1220
+ return None
1221
+
1222
+ def _try_parse_hyphen_split(self, control_id: str) -> Optional[str]:
1223
+ """
1224
+ Try to parse by splitting on hyphens.
1225
+
1226
+ :param str control_id: Control ID to parse
1227
+ :return: Normalized control ID or None
1228
+ :rtype: Optional[str]
1229
+ """
1230
+ if "-" not in control_id:
1231
+ return None
1232
+
1233
+ parts = control_id.split("-")
1234
+ if len(parts) < 2:
1235
+ return None
1236
+
1237
+ try:
1238
+ enhancement = parts[2] if len(parts) > 2 else None
1239
+ return self._format_control_parts(parts[0], parts[1], enhancement)
1240
+ except (ValueError, IndexError):
1241
+ return None
1242
+
1243
+ def _normalize_control_id(self, control_id: str) -> str:
1244
+ """
1245
+ Normalize control ID to remove leading zeros and standardize format to match RegScale.
1246
+
1247
+ Handles various AWS Audit Manager formats:
1248
+ - AC-2, AC-02
1249
+ - AC-2(1), AC-02(04)
1250
+ - AC-2 (1), AC-02 (04)
1251
+ - AC-2-1, AC-02-04
1252
+ - AC-2.1, AC-02.04, AC.2.1 (dot notation)
1253
+
1254
+ Returns format: AC-2 or AC-2(1) to match RegScale control IDs
1255
+
1256
+ :param str control_id: Raw control ID
1257
+ :return: Normalized control ID in RegScale format
1258
+ :rtype: str
1259
+ """
1260
+ if not control_id:
1261
+ return ""
1262
+
1263
+ control_id = control_id.strip().upper()
1264
+
1265
+ # Try parsing strategies in order
1266
+ result = self._try_parse_dot_notation(control_id)
1267
+ if result:
1268
+ return result
1269
+
1270
+ result = self._try_parse_standard_format(control_id)
1271
+ if result:
1272
+ return result
1273
+
1274
+ result = self._try_parse_hyphen_split(control_id)
1275
+ if result:
1276
+ return result
1277
+
1278
+ logger.warning(f"Could not parse control ID format: '{control_id}'")
1279
+ return control_id
1280
+
1281
+
1282
+ @dataclass
1283
+ class EvidenceCollectionConfig:
1284
+ """Configuration for evidence collection from AWS Audit Manager."""
1285
+
1286
+ collect_evidence: bool = False
1287
+ evidence_control_ids: Optional[List[str]] = None
1288
+ evidence_frequency: int = 30
1289
+ max_evidence_per_control: int = 100
1290
+
1291
+
1292
+ class AWSAuditManagerCompliance(ComplianceIntegration):
1293
+ """Process AWS Audit Manager assessments and create compliance records in RegScale."""
1294
+
1295
+ def __init__(
1296
+ self,
1297
+ plan_id: int,
1298
+ region: str = "us-east-1",
1299
+ framework: str = "NIST800-53R5",
1300
+ assessment_id: Optional[str] = None,
1301
+ create_issues: bool = True,
1302
+ update_control_status: bool = True,
1303
+ create_poams: bool = False,
1304
+ parent_module: str = "securityplans",
1305
+ evidence_config: Optional[EvidenceCollectionConfig] = None,
1306
+ force_refresh: bool = False,
1307
+ use_assessment_evidence_folders: bool = True,
1308
+ use_enhanced_analyzer: bool = False,
1309
+ **kwargs,
1310
+ ):
1311
+ """
1312
+ Initialize AWS Audit Manager compliance integration.
1313
+
1314
+ :param int plan_id: RegScale plan ID
1315
+ :param str region: AWS region
1316
+ :param str framework: Compliance framework
1317
+ :param Optional[str] assessment_id: Specific assessment ID to sync
1318
+ :param bool create_issues: Whether to create issues for failed compliance
1319
+ :param bool update_control_status: Whether to update control implementation status
1320
+ :param bool create_poams: Whether to mark issues as POAMs
1321
+ :param str parent_module: RegScale parent module
1322
+ :param Optional[EvidenceCollectionConfig] evidence_config: Evidence collection configuration
1323
+ :param bool force_refresh: Force refresh of compliance data by bypassing cache
1324
+ :param bool use_enhanced_analyzer: Use enhanced ControlComplianceAnalyzer for evidence analysis
1325
+ :param bool use_assessment_evidence_folders: Use GetEvidenceFoldersByAssessment API for faster
1326
+ evidence collection (default: False, uses per-control API)
1327
+ :param kwargs: Additional parameters including AWS credentials (profile, aws_access_key_id,
1328
+ aws_secret_access_key, aws_session_token)
1329
+ """
1330
+ super().__init__(
1331
+ plan_id=plan_id,
1332
+ framework=framework,
1333
+ create_issues=create_issues,
1334
+ update_control_status=update_control_status,
1335
+ create_poams=create_poams,
1336
+ parent_module=parent_module,
1337
+ **kwargs,
1338
+ )
1339
+
1340
+ self.region = region
1341
+ self.assessment_id = assessment_id
1342
+ self.title = "AWS Audit Manager"
1343
+ self.custom_framework_name = kwargs.get("custom_framework_name")
1344
+ self.use_enhanced_analyzer = use_enhanced_analyzer
1345
+
1346
+ # Evidence collection parameters - support both evidence_config object and individual kwargs
1347
+ if evidence_config:
1348
+ # Use provided evidence_config object
1349
+ self.evidence_config = evidence_config
1350
+ self.collect_evidence = evidence_config.collect_evidence
1351
+ self.evidence_control_ids = evidence_config.evidence_control_ids
1352
+ self.evidence_frequency = evidence_config.evidence_frequency
1353
+ self.max_evidence_per_control = min(evidence_config.max_evidence_per_control, 1000) # AWS API limit
1354
+ else:
1355
+ # Build evidence_config from kwargs (for CLI compatibility)
1356
+ collect_evidence = kwargs.get("collect_evidence", False)
1357
+ evidence_control_ids = kwargs.get("evidence_control_ids")
1358
+ evidence_frequency = kwargs.get("evidence_frequency", 30)
1359
+ max_evidence_per_control = kwargs.get("max_evidence_per_control", 100)
1360
+
1361
+ self.evidence_config = EvidenceCollectionConfig(
1362
+ collect_evidence=collect_evidence,
1363
+ evidence_control_ids=evidence_control_ids,
1364
+ evidence_frequency=evidence_frequency,
1365
+ max_evidence_per_control=max_evidence_per_control,
1366
+ )
1367
+ self.collect_evidence = collect_evidence
1368
+ self.evidence_control_ids = evidence_control_ids
1369
+ self.evidence_frequency = evidence_frequency
1370
+ self.max_evidence_per_control = min(max_evidence_per_control, 1000) # AWS API limit
1371
+
1372
+ # Cache control
1373
+ self.force_refresh = force_refresh
1374
+
1375
+ # Evidence collection method
1376
+ self.use_assessment_evidence_folders = use_assessment_evidence_folders
1377
+
1378
+ # Pre-collected evidence storage (populated before sync when using assessment folders)
1379
+ # Maps control_id (lowercase) -> List[evidence_items]
1380
+ self._evidence_by_control: Dict[str, List[Dict[str, Any]]] = {}
1381
+
1382
+ # Extract AWS credentials from kwargs
1383
+ profile = kwargs.get("profile")
1384
+ aws_access_key_id = kwargs.get("aws_access_key_id")
1385
+ aws_secret_access_key = kwargs.get("aws_secret_access_key")
1386
+ aws_session_token = kwargs.get("aws_session_token")
1387
+
1388
+ # INFO-level logging for credential resolution
1389
+ if aws_access_key_id and aws_secret_access_key:
1390
+ logger.info("Initializing AWS Audit Manager client with explicit credentials")
1391
+ self.session = boto3.Session(
1392
+ region_name=region,
1393
+ aws_access_key_id=aws_access_key_id,
1394
+ aws_secret_access_key=aws_secret_access_key,
1395
+ aws_session_token=aws_session_token,
1396
+ )
1397
+ else:
1398
+ logger.info(f"Initializing AWS Audit Manager client with profile: {profile if profile else 'default'}")
1399
+ self.session = boto3.Session(profile_name=profile, region_name=region)
1400
+
1401
+ try:
1402
+ self.client = self.session.client("auditmanager")
1403
+ logger.info("Successfully created AWS Audit Manager client")
1404
+ except Exception as e:
1405
+ logger.error(f"Failed to create AWS Audit Manager client: {e}")
1406
+ raise
1407
+
1408
+ def get_finding_identifier(self, finding) -> str:
1409
+ """
1410
+ Override parent method to ensure unique issue per control per resource.
1411
+
1412
+ For AWS compliance, we want one issue per failed control per AWS resource.
1413
+ The external_id already includes resource_id, so we use it directly to ensure uniqueness.
1414
+
1415
+ :param finding: IntegrationFinding object
1416
+ :return: Unique identifier for the finding
1417
+ :rtype: str
1418
+ """
1419
+ # Use the full external_id which includes control_id and resource_id
1420
+ # Format: "aws audit manager-{control_id}-{resource_id}"
1421
+ prefix = f"{self.plan_id}:{self.hash_string(finding.external_id)}"
1422
+ return prefix[:450]
1423
+
1424
+ def _is_cache_valid(self) -> bool:
1425
+ """
1426
+ Check if the cache file exists and is within the TTL.
1427
+
1428
+ :return: True if cache is valid, False otherwise
1429
+ :rtype: bool
1430
+ """
1431
+ if not os.path.exists(AUDIT_MANAGER_CACHE_FILE):
1432
+ logger.debug("Cache file does not exist")
1433
+ return False
1434
+
1435
+ file_age = time.time() - os.path.getmtime(AUDIT_MANAGER_CACHE_FILE)
1436
+ is_valid = file_age < CACHE_TTL_SECONDS
1437
+
1438
+ if is_valid:
1439
+ hours_old = file_age / 3600
1440
+ logger.info(f"Using cached Audit Manager data (age: {hours_old:.1f} hours)")
1441
+ else:
1442
+ hours_old = file_age / 3600
1443
+ logger.debug(f"Cache expired (age: {hours_old:.1f} hours, TTL: {CACHE_TTL_SECONDS / 3600} hours)")
1444
+
1445
+ return is_valid
1446
+
1447
+ def _load_cached_data(self) -> List[Dict[str, Any]]:
1448
+ """
1449
+ Load compliance data from cache file.
1450
+
1451
+ :return: List of raw compliance data from cache
1452
+ :rtype: List[Dict[str, Any]]
1453
+ """
1454
+ try:
1455
+ with open(AUDIT_MANAGER_CACHE_FILE, encoding="utf-8") as file:
1456
+ cached_data = json.load(file)
1457
+ logger.info(f"Loaded {len(cached_data)} compliance items from cache")
1458
+ return cached_data
1459
+ except (json.JSONDecodeError, IOError) as e:
1460
+ logger.warning(f"Error reading cache file: {e}. Fetching fresh data.")
1461
+ return []
1462
+
1463
+ def _save_to_cache(self, compliance_data: List[Dict[str, Any]]) -> None:
1464
+ """
1465
+ Save compliance data to cache file.
1466
+
1467
+ :param List[Dict[str, Any]] compliance_data: Data to cache
1468
+ :return: None
1469
+ :rtype: None
1470
+ """
1471
+ try:
1472
+ # Ensure the artifacts directory exists
1473
+ os.makedirs(os.path.dirname(AUDIT_MANAGER_CACHE_FILE), exist_ok=True)
1474
+
1475
+ with open(AUDIT_MANAGER_CACHE_FILE, "w", encoding="utf-8") as file:
1476
+ json.dump(compliance_data, file, indent=2, default=str)
1477
+
1478
+ logger.info(f"Cached {len(compliance_data)} compliance items to {AUDIT_MANAGER_CACHE_FILE}")
1479
+ except IOError as e:
1480
+ logger.warning(f"Error writing to cache file: {e}")
1481
+
1482
+ def _collect_evidence_for_control(
1483
+ self, assessment_id: str, control_set_id: str, control: Dict[str, Any], assessment: Dict[str, Any]
1484
+ ) -> Optional[List[Dict[str, Any]]]:
1485
+ """
1486
+ Collect evidence for a single control if enabled.
1487
+
1488
+ :param str assessment_id: Assessment ID
1489
+ :param str control_set_id: Control set ID
1490
+ :param Dict[str, Any] control: Control data
1491
+ :param Dict[str, Any] assessment: Assessment data
1492
+ :return: List of evidence items or None if not collected
1493
+ :rtype: Optional[List[Dict[str, Any]]]
1494
+ """
1495
+ control_id_raw = control.get("id")
1496
+ control_evidence_count = control.get("evidenceCount", 0)
1497
+
1498
+ # Create a temporary compliance item to get normalized control ID
1499
+ temp_item = AWSAuditManagerComplianceItem(assessment, control)
1500
+ control_id_normalized = temp_item.control_id
1501
+
1502
+ # Check if we should collect evidence for this control
1503
+ if not self._should_collect_control_evidence(control_id_normalized):
1504
+ logger.debug(
1505
+ f"Skipping evidence collection for control {control_id_normalized} "
1506
+ f"(evidenceCount: {control_evidence_count})"
1507
+ )
1508
+ return None
1509
+
1510
+ # Log INFO level for controls with evidence to show progress
1511
+ if control_evidence_count > 0:
1512
+ logger.info(
1513
+ f"Collecting evidence for control {control_id_normalized} "
1514
+ f"({control_evidence_count} evidence items available)..."
1515
+ )
1516
+ else:
1517
+ logger.debug(
1518
+ f"Fetching evidence inline for control {control_id_normalized} (evidenceCount: {control_evidence_count})"
1519
+ )
1520
+
1521
+ # Fetch evidence for this control
1522
+ evidence_items = self._get_control_evidence(
1523
+ assessment_id=assessment_id, control_set_id=control_set_id, control_id=control_id_raw
1524
+ )
1525
+
1526
+ if evidence_items:
1527
+ logger.info(
1528
+ f"Successfully collected {len(evidence_items)} evidence items for control {control_id_normalized}"
1529
+ )
1530
+ else:
1531
+ logger.debug(f"No evidence items retrieved for control {control_id_normalized}")
1532
+
1533
+ return evidence_items
1534
+
1535
+ def _process_assessment_controls(self, assessment: Dict[str, Any]) -> List[Dict[str, Any]]:
1536
+ """
1537
+ Process a single assessment and extract all control data.
1538
+
1539
+ If collect_evidence is True, fetches evidence inline for each control to enable
1540
+ compliance determination based on evidence analysis.
1541
+
1542
+ :param Dict[str, Any] assessment: Assessment data
1543
+ :return: List of control data for this assessment (with optional evidence_items)
1544
+ :rtype: List[Dict[str, Any]]
1545
+ """
1546
+ compliance_data = []
1547
+ control_sets = assessment.get("framework", {}).get("controlSets", [])
1548
+ logger.debug(f"Found {len(control_sets)} control sets in assessment")
1549
+
1550
+ # Extract assessment ID for evidence collection
1551
+ assessment_id = assessment.get("arn", "").split("/")[-1]
1552
+
1553
+ # Calculate total controls for progress tracking
1554
+ total_controls = sum(len(cs.get("controls", [])) for cs in control_sets)
1555
+ logger.info(f"Processing {total_controls} controls across {len(control_sets)} control sets...")
1556
+
1557
+ # Create progress bar for control processing
1558
+ progress = create_progress_object()
1559
+ with progress:
1560
+ task = progress.add_task(
1561
+ f"Processing controls for assessment '{assessment.get('name', 'Unknown')}'", total=total_controls
1562
+ )
1563
+
1564
+ for control_set in control_sets:
1565
+ control_set_id = control_set.get("id")
1566
+ controls = control_set.get("controls", [])
1567
+ logger.debug(f"Found {len(controls)} controls in control set")
1568
+
1569
+ for control in controls:
1570
+ control_data = {"assessment": assessment, "control": control}
1571
+
1572
+ # If evidence collection is enabled, fetch evidence inline for compliance determination
1573
+ if self.collect_evidence:
1574
+ evidence_items = self._collect_evidence_for_control(
1575
+ assessment_id, control_set_id, control, assessment
1576
+ )
1577
+ if evidence_items:
1578
+ control_data["evidence_items"] = evidence_items
1579
+
1580
+ compliance_data.append(control_data)
1581
+ progress.update(task, advance=1)
1582
+
1583
+ logger.info(f"Finished processing {len(compliance_data)} controls for assessment")
1584
+ return compliance_data
1585
+
1586
+ def _should_process_assessment(self, assessment: Dict[str, Any]) -> bool:
1587
+ """
1588
+ Check if assessment should be processed based on framework match.
1589
+
1590
+ For custom frameworks (--framework Custom), matches against the assessment name
1591
+ using the custom_framework_name parameter.
1592
+
1593
+ :param Dict[str, Any] assessment: Assessment data
1594
+ :return: True if assessment should be processed
1595
+ :rtype: bool
1596
+ """
1597
+ if not assessment:
1598
+ return False
1599
+
1600
+ assessment_name = assessment.get("name", "Unknown")
1601
+
1602
+ # Special handling for custom frameworks - match by framework name
1603
+ if self.framework.upper() == "CUSTOM":
1604
+ if not self.custom_framework_name:
1605
+ logger.warning(
1606
+ f"Skipping assessment '{assessment_name}' - framework is set to 'CUSTOM' "
1607
+ "but no custom_framework_name provided. Use --custom-framework-name to specify."
1608
+ )
1609
+ return False
1610
+
1611
+ # Check the framework metadata for custom framework name
1612
+ framework = assessment.get("framework", {})
1613
+ framework_metadata = framework.get("metadata", {})
1614
+ framework_name = framework_metadata.get("name", "")
1615
+
1616
+ # Debug logging to understand the structure
1617
+ logger.debug(f"Assessment '{assessment_name}' framework metadata: {framework_metadata}")
1618
+
1619
+ # Normalize both names for comparison
1620
+ custom_normalized = self.custom_framework_name.lower().replace(" ", "").replace("-", "").replace("_", "")
1621
+ framework_normalized = framework_name.lower().replace(" ", "").replace("-", "").replace("_", "")
1622
+
1623
+ # Match against the framework name (not assessment name)
1624
+ if (
1625
+ custom_normalized == framework_normalized
1626
+ or custom_normalized in framework_normalized
1627
+ or framework_normalized in custom_normalized
1628
+ ):
1629
+ logger.info(
1630
+ f"Processing assessment '{assessment_name}' - uses custom framework '{framework_name}' matching '{self.custom_framework_name}'"
1631
+ )
1632
+ return True
1633
+
1634
+ logger.info(
1635
+ f"Skipping assessment '{assessment_name}' - framework '{framework_name}' does not match custom framework name '{self.custom_framework_name}'"
1636
+ )
1637
+ return False
1638
+
1639
+ # For standard frameworks, match by framework type
1640
+ assessment_framework = self._get_assessment_framework(assessment)
1641
+ if not self._matches_framework(assessment_framework):
1642
+ logger.info(
1643
+ f"Skipping assessment '{assessment_name}' - framework '{assessment_framework}' "
1644
+ f"does not match target framework '{self.framework}'"
1645
+ )
1646
+ return False
1647
+
1648
+ return True
1649
+
1650
+ def _fetch_fresh_compliance_data(self) -> List[Dict[str, Any]]:
1651
+ """
1652
+ Fetch fresh compliance data from AWS Audit Manager.
1653
+
1654
+ :return: List of raw compliance data
1655
+ :rtype: List[Dict[str, Any]]
1656
+ """
1657
+ logger.info("Fetching compliance data from AWS Audit Manager...")
1658
+ compliance_data = []
1659
+
1660
+ assessments = (
1661
+ [self._get_assessment_details(self.assessment_id)] if self.assessment_id else self._list_all_assessments()
1662
+ )
1663
+
1664
+ for assessment in assessments:
1665
+ if not self._should_process_assessment(assessment):
1666
+ continue
1667
+
1668
+ assessment_id = assessment.get("arn", "")
1669
+ logger.info(f"Processing assessment: {assessment.get('name', assessment_id)}")
1670
+ compliance_data.extend(self._process_assessment_controls(assessment))
1671
+
1672
+ logger.info(f"Fetched {len(compliance_data)} compliance items from AWS Audit Manager")
1673
+ return compliance_data
1674
+
1675
+ def fetch_compliance_data(self) -> List[Dict[str, Any]]:
1676
+ """
1677
+ Fetch raw compliance data from AWS Audit Manager.
1678
+
1679
+ Uses cached data if available and not expired (4-hour TTL), unless force_refresh is True.
1680
+ Filters assessments to only include those matching the specified framework.
1681
+
1682
+ :return: List of raw compliance data (assessment + control combinations)
1683
+ :rtype: List[Dict[str, Any]]
1684
+ """
1685
+ # Check if we should use cached data
1686
+ if not self.force_refresh and self._is_cache_valid():
1687
+ cached_data = self._load_cached_data()
1688
+ if cached_data:
1689
+ return self._filter_by_framework(cached_data)
1690
+
1691
+ # Force refresh requested or no valid cache, fetch fresh data from AWS
1692
+ if self.force_refresh:
1693
+ logger.info("Force refresh requested, bypassing cache and fetching fresh data from AWS Audit Manager...")
1694
+
1695
+ try:
1696
+ compliance_data = self._fetch_fresh_compliance_data()
1697
+ self._save_to_cache(compliance_data)
1698
+ return compliance_data
1699
+ except ClientError as e:
1700
+ logger.error(f"Error fetching compliance data from AWS Audit Manager: {e}")
1701
+ return []
1702
+
1703
+ def create_compliance_item(self, raw_data: Dict[str, Any]) -> ComplianceItem:
1704
+ """
1705
+ Create a ComplianceItem from raw compliance data.
1706
+
1707
+ If evidence was pre-collected (via _collect_evidence_before_sync), it will be retrieved
1708
+ from _evidence_by_control storage and included in the compliance item for proper
1709
+ pass/fail determination.
1710
+
1711
+ :param Dict[str, Any] raw_data: Raw compliance data (assessment + control + optional evidence)
1712
+ :return: ComplianceItem instance
1713
+ :rtype: ComplianceItem
1714
+ """
1715
+ assessment = raw_data.get("assessment", {})
1716
+ control = raw_data.get("control", {})
1717
+ evidence_items = raw_data.get("evidence_items", [])
1718
+
1719
+ # If no evidence in raw_data, check if we have pre-collected evidence
1720
+ if not evidence_items and self._evidence_by_control:
1721
+ # Extract control ID from control
1722
+ temp_item = AWSAuditManagerComplianceItem(
1723
+ assessment, control, [], use_enhanced_analyzer=self.use_enhanced_analyzer
1724
+ )
1725
+ control_id = temp_item.control_id.lower()
1726
+
1727
+ # Look up pre-collected evidence
1728
+ evidence_items = self._evidence_by_control.get(control_id, [])
1729
+ if evidence_items:
1730
+ logger.debug(f"Using pre-collected evidence for control {control_id}: {len(evidence_items)} items")
1731
+
1732
+ return AWSAuditManagerComplianceItem(
1733
+ assessment, control, evidence_items, use_enhanced_analyzer=self.use_enhanced_analyzer
1734
+ )
1735
+
1736
+ def _list_all_assessments(self) -> List[Dict[str, Any]]:
1737
+ """
1738
+ List all active assessments.
1739
+
1740
+ :return: List of assessment details
1741
+ :rtype: List[Dict[str, Any]]
1742
+ """
1743
+ assessments = []
1744
+ try:
1745
+ response = self.client.list_assessments()
1746
+ assessment_metadata_list = response.get("assessmentMetadata", [])
1747
+
1748
+ for metadata in assessment_metadata_list:
1749
+ status = metadata.get("status", "")
1750
+ if status in ["ACTIVE", "COMPLETED"]:
1751
+ assessment_id = metadata.get("id", "")
1752
+ assessment = self._get_assessment_details(assessment_id)
1753
+ if assessment:
1754
+ assessments.append(assessment)
1755
+
1756
+ except ClientError as e:
1757
+ logger.error(f"Error listing assessments: {e}")
1758
+
1759
+ return assessments
1760
+
1761
+ def _get_assessment_details(self, assessment_id: str) -> Optional[Dict[str, Any]]:
1762
+ """
1763
+ Get full assessment details including controls and evidence.
1764
+
1765
+ :param str assessment_id: Assessment ID
1766
+ :return: Assessment details or None
1767
+ :rtype: Optional[Dict[str, Any]]
1768
+ """
1769
+ try:
1770
+ response = self.client.get_assessment(assessmentId=assessment_id)
1771
+ assessment = response.get("assessment", {})
1772
+
1773
+ metadata = assessment.get("metadata", {})
1774
+ assessment_data = {
1775
+ "arn": assessment.get("arn", ""),
1776
+ "name": metadata.get("name", ""),
1777
+ "description": metadata.get("description", ""),
1778
+ "complianceType": metadata.get("complianceType", ""),
1779
+ "status": metadata.get("status", ""),
1780
+ "awsAccount": assessment.get("awsAccount", {}),
1781
+ "framework": assessment.get("framework", {}),
1782
+ }
1783
+
1784
+ return assessment_data
1785
+
1786
+ except ClientError as e:
1787
+ if e.response["Error"]["Code"] not in ["ResourceNotFoundException", "AccessDeniedException"]:
1788
+ logger.error(f"Error getting assessment details for {assessment_id}: {e}")
1789
+ return None
1790
+
1791
+ def _get_assessment_framework(self, assessment: Dict[str, Any]) -> str:
1792
+ """
1793
+ Extract framework name from assessment data.
1794
+
1795
+ :param Dict[str, Any] assessment: Assessment data
1796
+ :return: Framework name
1797
+ :rtype: str
1798
+ """
1799
+ # For custom frameworks, we need to handle the case where the assessment
1800
+ # was created from a custom framework. In this case, we should match
1801
+ # based on the assessment name if the framework is CUSTOM
1802
+ if self.framework.upper() == "CUSTOM" and self.custom_framework_name:
1803
+ # Return the assessment name for custom framework matching
1804
+ # This allows matching against the assessment name pattern
1805
+ return assessment.get("name", "")
1806
+
1807
+ framework_name = assessment.get("framework", {}).get("metadata", {}).get("name", "")
1808
+ compliance_type = assessment.get("complianceType", "")
1809
+
1810
+ # Prefer compliance type if available, otherwise use framework name
1811
+ return compliance_type or framework_name
1812
+
1813
+ def _normalize_framework_string(self, framework_string: str) -> str:
1814
+ """
1815
+ Normalize a framework string by removing spaces, hyphens, and underscores.
1816
+
1817
+ :param str framework_string: The string to normalize
1818
+ :return: Normalized string
1819
+ :rtype: str
1820
+ """
1821
+ return framework_string.lower().replace(" ", "").replace("-", "").replace("_", "")
1822
+
1823
+ def _check_custom_framework_match(self, assessment_framework: str) -> bool:
1824
+ """
1825
+ Check if an assessment framework matches a custom framework.
1826
+
1827
+ :param str assessment_framework: Framework name from AWS assessment
1828
+ :return: True if framework matches custom target
1829
+ :rtype: bool
1830
+ """
1831
+ if not self.custom_framework_name:
1832
+ logger.warning(
1833
+ "Framework is set to 'CUSTOM' but no custom_framework_name provided. "
1834
+ "Use --custom-framework-name to specify the custom framework name."
1835
+ )
1836
+ return False
1837
+
1838
+ custom_normalized = self._normalize_framework_string(self.custom_framework_name)
1839
+ actual_normalized = self._normalize_framework_string(assessment_framework)
1840
+
1841
+ # Allow flexible matching for custom frameworks
1842
+ matches = (
1843
+ custom_normalized == actual_normalized
1844
+ or custom_normalized in actual_normalized
1845
+ or actual_normalized in custom_normalized
1846
+ or "customframework" in actual_normalized
1847
+ )
1848
+
1849
+ if matches:
1850
+ logger.debug(f"Custom framework match: '{assessment_framework}' matches '{self.custom_framework_name}'")
1851
+
1852
+ return matches
1853
+
1854
+ def _check_framework_aliases(self, target: str, actual: str) -> bool:
1855
+ """
1856
+ Check if target and actual frameworks match using known aliases.
1857
+
1858
+ :param str target: Normalized target framework
1859
+ :param str actual: Normalized actual framework
1860
+ :return: True if frameworks match via aliases
1861
+ :rtype: bool
1862
+ """
1863
+ framework_aliases = {
1864
+ "nist80053r5": ["nist", "nistsp80053", "nist80053", "80053"],
1865
+ "soc2": ["soc", "soc2typeii", "soc2type2"],
1866
+ "pcidss": ["pci", "pcidss3.2.1", "pcidss3.2"],
1867
+ "hipaa": ["hipaa", "hipaasecurityrule"],
1868
+ "gdpr": ["gdpr", "generaldataprotectionregulation"],
1869
+ }
1870
+
1871
+ for key, aliases in framework_aliases.items():
1872
+ if target.startswith(key) or any(target.startswith(alias) for alias in aliases):
1873
+ if any(alias in actual for alias in aliases):
1874
+ return True
1875
+ return False
1876
+
1877
+ def _matches_framework(self, assessment_framework: str) -> bool:
1878
+ """
1879
+ Check if an assessment framework matches the target framework.
1880
+
1881
+ Handles various naming conventions:
1882
+ - NIST 800-53: "NIST SP 800-53 Revision 5", "NIST800-53R5", "NIST 800-53 R5"
1883
+ - SOC 2: "SOC2", "SOC 2", "SOC 2 Type II"
1884
+ - PCI DSS: "PCI DSS", "PCI DSS 3.2.1"
1885
+ - HIPAA: "HIPAA", "HIPAA Security Rule"
1886
+ - GDPR: "GDPR", "General Data Protection Regulation"
1887
+ - CUSTOM: Matches against custom_framework_name parameter or assessment name patterns
1888
+
1889
+ :param str assessment_framework: Framework name from AWS assessment (or assessment name for custom frameworks)
1890
+ :return: True if framework matches target
1891
+ :rtype: bool
1892
+ """
1893
+ if not assessment_framework:
1894
+ return False
1895
+
1896
+ # Special handling for custom frameworks
1897
+ if self.framework.upper() == "CUSTOM":
1898
+ return self._check_custom_framework_match(assessment_framework)
1899
+
1900
+ # Normalize both for comparison
1901
+ target = self._normalize_framework_string(self.framework)
1902
+ actual = self._normalize_framework_string(assessment_framework)
1903
+
1904
+ # Direct match
1905
+ if target in actual or actual in target:
1906
+ return True
1907
+
1908
+ # Check framework aliases
1909
+ return self._check_framework_aliases(target, actual)
1910
+
1911
+ def _filter_by_framework(self, compliance_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
1912
+ """
1913
+ Filter compliance data to only include items from matching framework.
1914
+
1915
+ :param List[Dict[str, Any]] compliance_data: Raw compliance data
1916
+ :return: Filtered compliance data matching the target framework
1917
+ :rtype: List[Dict[str, Any]]
1918
+ """
1919
+ filtered_data = []
1920
+ frameworks_seen = set()
1921
+
1922
+ for item in compliance_data:
1923
+ assessment = item.get("assessment", {})
1924
+ assessment_framework = self._get_assessment_framework(assessment)
1925
+ frameworks_seen.add(assessment_framework)
1926
+
1927
+ if self._matches_framework(assessment_framework):
1928
+ filtered_data.append(item)
1929
+
1930
+ if filtered_data != compliance_data:
1931
+ logger.info(
1932
+ f"Filtered compliance data by framework: {len(compliance_data)} total items, "
1933
+ f"{len(filtered_data)} matching '{self.framework}'"
1934
+ )
1935
+ logger.debug(f"Frameworks found in cached data: {sorted(frameworks_seen)}")
1936
+
1937
+ return filtered_data
1938
+
1939
+ def _map_resource_type_to_asset_type(self, compliance_item: ComplianceItem) -> str:
1940
+ """
1941
+ Map AWS resource type to RegScale asset type.
1942
+
1943
+ :param ComplianceItem compliance_item: Compliance item with resource information
1944
+ :return: Asset type string
1945
+ :rtype: str
1946
+ """
1947
+ return "AWS Account"
1948
+
1949
+ def _map_severity(self, severity: Optional[str]) -> regscale_models.IssueSeverity:
1950
+ """
1951
+ Map AWS severity to RegScale severity.
1952
+
1953
+ :param Optional[str] severity: Severity string from AWS
1954
+ :return: Mapped RegScale severity enum value
1955
+ :rtype: regscale_models.IssueSeverity
1956
+ """
1957
+ if not severity:
1958
+ return regscale_models.IssueSeverity.Moderate
1959
+
1960
+ severity_mapping = {
1961
+ "CRITICAL": regscale_models.IssueSeverity.Critical,
1962
+ "HIGH": regscale_models.IssueSeverity.High,
1963
+ "MEDIUM": regscale_models.IssueSeverity.Moderate,
1964
+ "LOW": regscale_models.IssueSeverity.Low,
1965
+ }
1966
+
1967
+ return severity_mapping.get(severity.upper(), regscale_models.IssueSeverity.Moderate)
1968
+
1969
+ def _should_collect_control_evidence(self, control_id_normalized: str) -> bool:
1970
+ """
1971
+ Check if evidence should be collected for a control.
1972
+
1973
+ Note: AWS Audit Manager's evidenceCount field in control metadata is not always accurate.
1974
+ We attempt to fetch evidence for all controls (or filtered controls if specified) and
1975
+ let the API determine if evidence exists.
1976
+
1977
+ :param str control_id_normalized: Normalized control ID
1978
+ :return: True if evidence should be collected
1979
+ :rtype: bool
1980
+ """
1981
+ # Filter by control IDs if specified
1982
+ if self.evidence_control_ids:
1983
+ if control_id_normalized not in self.evidence_control_ids:
1984
+ logger.debug(f"Skipping evidence collection for control {control_id_normalized} (not in filter list)")
1985
+ return False
1986
+
1987
+ # Don't skip based on evidenceCount - AWS Audit Manager metadata may be inaccurate
1988
+ # The API call will return empty list if no evidence exists
1989
+ return True
1990
+
1991
+ def _process_control_evidence(
1992
+ self,
1993
+ assessment_id: str,
1994
+ control_set_id: str,
1995
+ control: Dict[str, Any],
1996
+ assessment: Dict[str, Any],
1997
+ all_evidence_items: List[Dict[str, Any]],
1998
+ control_summary: Dict[str, Dict[str, Any]],
1999
+ ) -> None:
2000
+ """
2001
+ Process evidence collection for a single control.
2002
+
2003
+ :param str assessment_id: Assessment ID
2004
+ :param str control_set_id: Control set ID
2005
+ :param Dict[str, Any] control: Control data
2006
+ :param Dict[str, Any] assessment: Assessment data
2007
+ :param List[Dict[str, Any]] all_evidence_items: List to append evidence to
2008
+ :param Dict[str, Dict[str, Any]] control_summary: Summary dict to update
2009
+ :return: None
2010
+ :rtype: None
2011
+ """
2012
+ control_id_raw = control.get("id")
2013
+ control_name = control.get("name", "")
2014
+ control_evidence_count = control.get("evidenceCount", 0)
2015
+
2016
+ # Extract and normalize control ID (e.g., AU-2, AC-3)
2017
+ compliance_item = AWSAuditManagerComplianceItem(assessment, control)
2018
+ control_id_normalized = compliance_item.control_id
2019
+
2020
+ # Check if we should collect evidence for this control
2021
+ if not self._should_collect_control_evidence(control_id_normalized):
2022
+ return
2023
+
2024
+ logger.debug(
2025
+ f"Collecting evidence for control: {control_id_normalized} (evidenceCount: {control_evidence_count})"
2026
+ )
2027
+
2028
+ # Collect evidence for this control
2029
+ evidence_items = self._get_control_evidence(
2030
+ assessment_id=assessment_id, control_set_id=control_set_id, control_id=control_id_raw
2031
+ )
2032
+
2033
+ if evidence_items:
2034
+ # Tag each evidence item with control information for traceability
2035
+ for item in evidence_items:
2036
+ item["_control_id"] = control_id_normalized
2037
+ item["_control_name"] = control_name
2038
+
2039
+ all_evidence_items.extend(evidence_items)
2040
+ control_summary[control_id_normalized] = {
2041
+ "control_name": control_name,
2042
+ "evidence_count": len(evidence_items),
2043
+ }
2044
+ logger.debug(f"Collected {len(evidence_items)} evidence items for control {control_id_normalized}")
2045
+
2046
+ def _collect_assessment_control_evidence(self, assessment: Dict[str, Any]) -> tuple:
2047
+ """
2048
+ Collect evidence for all controls in an assessment.
2049
+
2050
+ :param Dict[str, Any] assessment: Assessment data
2051
+ :return: Tuple of (all_evidence_items, control_summary, controls_processed)
2052
+ :rtype: tuple
2053
+ """
2054
+ assessment_id = assessment.get("arn", "").split("/")[-1]
2055
+ all_evidence_items = []
2056
+ control_summary = {}
2057
+ controls_processed = 0
2058
+
2059
+ # Get control sets from assessment framework
2060
+ control_sets = assessment.get("framework", {}).get("controlSets", [])
2061
+
2062
+ for control_set in control_sets:
2063
+ control_set_id = control_set.get("id")
2064
+ controls = control_set.get("controls", [])
2065
+
2066
+ for control in controls:
2067
+ self._process_control_evidence(
2068
+ assessment_id=assessment_id,
2069
+ control_set_id=control_set_id,
2070
+ control=control,
2071
+ assessment=assessment,
2072
+ all_evidence_items=all_evidence_items,
2073
+ control_summary=control_summary,
2074
+ )
2075
+ controls_processed += 1
2076
+
2077
+ return all_evidence_items, control_summary, controls_processed
2078
+
2079
+ def _get_all_evidence_folders_for_assessment(self, assessment_id: str) -> Dict[str, List[Dict[str, Any]]]:
2080
+ """
2081
+ Get ALL evidence folders for an assessment using GetEvidenceFoldersByAssessment API.
2082
+
2083
+ This is faster than iterating through controls individually because it retrieves
2084
+ all evidence folders in a single paginated operation. Evidence folders are grouped
2085
+ by control ID for easier processing.
2086
+
2087
+ :param str assessment_id: Assessment ID
2088
+ :return: Dict mapping control_id -> list of evidence folders
2089
+ :rtype: Dict[str, List[Dict[str, Any]]]
2090
+ """
2091
+ evidence_folders_by_control = {}
2092
+ next_token = None
2093
+
2094
+ logger.info(f"Fetching all evidence folders for assessment {assessment_id} using assessment-level API")
2095
+
2096
+ try:
2097
+ while True:
2098
+ params = {"assessmentId": assessment_id, "maxResults": 1000}
2099
+ if next_token:
2100
+ params["nextToken"] = next_token
2101
+
2102
+ logger.debug(f"Calling get_evidence_folders_by_assessment (maxResults={params['maxResults']})")
2103
+ response = self.client.get_evidence_folders_by_assessment(**params)
2104
+ evidence_folders = response.get("evidenceFolders", [])
2105
+
2106
+ # Group by control ID
2107
+ for folder in evidence_folders:
2108
+ control_id = folder.get("controlId")
2109
+ if control_id:
2110
+ if control_id not in evidence_folders_by_control:
2111
+ evidence_folders_by_control[control_id] = []
2112
+ evidence_folders_by_control[control_id].append(folder)
2113
+
2114
+ logger.debug(f"Retrieved {len(evidence_folders)} evidence folder(s) in this page")
2115
+
2116
+ next_token = response.get("nextToken")
2117
+ if not next_token:
2118
+ break
2119
+
2120
+ total_folders = sum(len(folders) for folders in evidence_folders_by_control.values())
2121
+ logger.info(
2122
+ f"Found {total_folders} evidence folder(s) across {len(evidence_folders_by_control)} control(s)"
2123
+ )
2124
+ return evidence_folders_by_control
2125
+
2126
+ except ClientError as e:
2127
+ error_code = e.response["Error"]["Code"]
2128
+ error_message = e.response["Error"].get("Message", "")
2129
+ logger.error(
2130
+ f"Error fetching evidence folders by assessment {assessment_id}: {error_code} - {error_message}"
2131
+ )
2132
+ return {}
2133
+
2134
+ def _collect_evidence_assessment_level(self, assessment: Dict[str, Any], assessment_id: str) -> tuple:
2135
+ """
2136
+ Collect evidence using assessment-level API (faster method).
2137
+
2138
+ Uses GetEvidenceFoldersByAssessment to retrieve all evidence folders at once,
2139
+ then processes each control's evidence. This is much faster than iterating
2140
+ through controls individually.
2141
+
2142
+ :param Dict[str, Any] assessment: Assessment data
2143
+ :param str assessment_id: Assessment ID
2144
+ :return: Tuple of (all_evidence_items, control_summary)
2145
+ :rtype: tuple
2146
+ """
2147
+ # Get ALL evidence folders at once
2148
+ evidence_folders_by_control = self._get_all_evidence_folders_for_assessment(assessment_id)
2149
+
2150
+ if not evidence_folders_by_control:
2151
+ logger.warning(f"No evidence folders found for assessment {assessment_id}")
2152
+ return [], {}
2153
+
2154
+ all_evidence_items = []
2155
+ control_summary = {}
2156
+
2157
+ # Create progress bar for evidence collection
2158
+ from rich.progress import (
2159
+ Progress,
2160
+ SpinnerColumn,
2161
+ TextColumn,
2162
+ BarColumn,
2163
+ TaskProgressColumn,
2164
+ TimeRemainingColumn,
2165
+ )
2166
+
2167
+ total_controls = len(evidence_folders_by_control)
2168
+
2169
+ with Progress(
2170
+ SpinnerColumn(),
2171
+ TextColumn("[progress.description]{task.description}"),
2172
+ BarColumn(),
2173
+ TaskProgressColumn(),
2174
+ TimeRemainingColumn(),
2175
+ ) as progress:
2176
+ task = progress.add_task(
2177
+ f"[cyan]Collecting evidence from {total_controls} control(s)...", total=total_controls
2178
+ )
2179
+
2180
+ # Process each control that has evidence folders
2181
+ for control_id_raw, folders in evidence_folders_by_control.items():
2182
+ # Get control info from first folder
2183
+ control_set_id = folders[0].get("controlSetId")
2184
+ control_name = folders[0].get("controlName", control_id_raw)
2185
+
2186
+ # Normalize control ID by creating a temporary compliance item
2187
+ # We need to build control data from the folder metadata
2188
+ control_data = {"id": control_id_raw, "name": control_name}
2189
+ temp_item = AWSAuditManagerComplianceItem(assessment, control_data)
2190
+ control_id_normalized = temp_item.control_id
2191
+
2192
+ # Filter by evidence_control_ids if specified
2193
+ if self.evidence_control_ids and control_id_normalized not in self.evidence_control_ids:
2194
+ logger.debug(f"Skipping control {control_id_normalized} (not in filter list)")
2195
+ progress.advance(task)
2196
+ continue
2197
+
2198
+ logger.debug(
2199
+ f"Collecting evidence for control {control_id_normalized} ({len(folders)} evidence folder(s))"
2200
+ )
2201
+
2202
+ # Collect evidence from these folders
2203
+ evidence_items = self._process_evidence_folders(
2204
+ assessment_id, control_set_id, control_id_raw, folders, control_name=control_id_normalized
2205
+ )
2206
+
2207
+ if evidence_items:
2208
+ # Tag each evidence item with control information
2209
+ for item in evidence_items:
2210
+ item["_control_id"] = control_id_normalized
2211
+ item["_control_name"] = control_name
2212
+
2213
+ all_evidence_items.extend(evidence_items)
2214
+ control_summary[control_id_normalized] = {
2215
+ "control_name": control_name,
2216
+ "evidence_count": len(evidence_items),
2217
+ }
2218
+ logger.debug(f"Collected {len(evidence_items)} evidence items for control {control_id_normalized}")
2219
+
2220
+ progress.advance(task)
2221
+
2222
+ return all_evidence_items, control_summary
2223
+
2224
+ def _process_cached_evidence(self) -> tuple:
2225
+ """
2226
+ Process pre-collected cached evidence.
2227
+
2228
+ :return: Tuple of (all_evidence_items, control_summary, controls_processed)
2229
+ :rtype: tuple
2230
+ """
2231
+ all_evidence_items = []
2232
+ control_summary = {}
2233
+
2234
+ # Aggregate all evidence items from the cache
2235
+ for control_id, evidence_items in self._evidence_by_control.items():
2236
+ all_evidence_items.extend(evidence_items)
2237
+
2238
+ # Extract control name from the first evidence item if available
2239
+ control_name = "Unknown Control"
2240
+ if evidence_items and len(evidence_items) > 0:
2241
+ first_item = evidence_items[0]
2242
+ if isinstance(first_item, dict):
2243
+ control_name = first_item.get("_control_name", first_item.get("controlName", control_id))
2244
+
2245
+ # Create control summary in the expected format
2246
+ control_summary[control_id] = {
2247
+ "control_name": control_name,
2248
+ "evidence_count": len(evidence_items),
2249
+ }
2250
+
2251
+ controls_processed = len(control_summary)
2252
+ logger.info(f"Reusing cached evidence: {len(all_evidence_items)} items from {controls_processed} controls")
2253
+
2254
+ return all_evidence_items, control_summary, controls_processed
2255
+
2256
+ def collect_assessment_evidence(self, assessments: List[Dict[str, Any]]) -> None:
2257
+ """
2258
+ Collect evidence artifacts from AWS Audit Manager assessments.
2259
+
2260
+ Collects evidence from yesterday's evidence folders (UTC timezone) to provide
2261
+ daily compliance evidence snapshots. If no evidence exists for yesterday for a
2262
+ control, that control is skipped.
2263
+
2264
+ Supports two collection methods:
2265
+ 1. Assessment-level: GetEvidenceFoldersByAssessment (faster, single API call)
2266
+ 2. Control-level: GetEvidenceFoldersByAssessmentControl (current, per-control iteration)
2267
+
2268
+ Aggregates all evidence across all controls in each assessment and creates
2269
+ a single consolidated JSONL file per assessment stored in the artifacts directory.
2270
+ Creates one RegScale Evidence record per assessment with the consolidated file attached.
2271
+
2272
+ :param List[Dict[str, Any]] assessments: List of assessment data
2273
+ :return: None
2274
+ :rtype: None
2275
+ """
2276
+ if not self.collect_evidence:
2277
+ logger.debug("Evidence collection disabled, skipping")
2278
+ return
2279
+
2280
+ collection_method = "assessment-level" if self.use_assessment_evidence_folders else "control-level"
2281
+
2282
+ # Check if evidence was already collected during pre-sync
2283
+ if self._evidence_by_control:
2284
+ logger.info(f"Using pre-collected evidence from {len(self._evidence_by_control)} controls")
2285
+ else:
2286
+ logger.info(f"Starting evidence collection from AWS Audit Manager using {collection_method} API...")
2287
+
2288
+ evidence_records_created = 0
2289
+
2290
+ for assessment in assessments:
2291
+ assessment_name = assessment.get("name", "Unknown Assessment")
2292
+ assessment_id = assessment.get("arn", "").split("/")[-1]
2293
+
2294
+ logger.info(f"Processing evidence for assessment: {assessment_name}")
2295
+
2296
+ # Check if evidence was already collected during pre-sync
2297
+ if self._evidence_by_control:
2298
+ # Reuse pre-collected evidence instead of fetching again from AWS
2299
+ all_evidence_items, control_summary, controls_processed = self._process_cached_evidence()
2300
+ elif self.use_assessment_evidence_folders:
2301
+ # NEW: Fast method - get all evidence folders at once
2302
+ all_evidence_items, control_summary = self._collect_evidence_assessment_level(assessment, assessment_id)
2303
+ controls_processed = len(control_summary)
2304
+ else:
2305
+ # EXISTING: Per-control iteration method (backward compatible)
2306
+ all_evidence_items, control_summary, controls_processed = self._collect_assessment_control_evidence(
2307
+ assessment
2308
+ )
2309
+
2310
+ # Create consolidated evidence record if we collected any evidence
2311
+ if all_evidence_items:
2312
+ evidence_record = self._create_consolidated_evidence_record(
2313
+ assessment=assessment,
2314
+ assessment_name=assessment_name,
2315
+ all_evidence_items=all_evidence_items,
2316
+ control_summary=control_summary,
2317
+ controls_processed=controls_processed,
2318
+ )
2319
+
2320
+ if evidence_record:
2321
+ evidence_records_created += 1
2322
+ else:
2323
+ logger.info(f"No evidence collected for assessment: {assessment_name}")
2324
+
2325
+ logger.info(f"Evidence collection complete: {evidence_records_created} consolidated evidence record(s) created")
2326
+
2327
+ def _get_evidence_folders(self, assessment_id: str, control_set_id: str, control_id: str) -> List[Dict[str, Any]]:
2328
+ """
2329
+ Get all evidence folders for a specific control.
2330
+
2331
+ :param str assessment_id: Assessment ID
2332
+ :param str control_set_id: Control set ID
2333
+ :param str control_id: Control ID (AWS internal ID)
2334
+ :return: List of evidence folders
2335
+ :rtype: List[Dict[str, Any]]
2336
+ """
2337
+ logger.debug(
2338
+ f"Getting evidence folders for control: assessmentId={assessment_id}, "
2339
+ f"controlSetId={control_set_id}, controlId={control_id}"
2340
+ )
2341
+
2342
+ try:
2343
+ folders_response = self.client.get_evidence_folders_by_assessment_control(
2344
+ assessmentId=assessment_id, controlSetId=control_set_id, controlId=control_id
2345
+ )
2346
+
2347
+ evidence_folders = folders_response.get("evidenceFolders", [])
2348
+ logger.debug(f"Found {len(evidence_folders)} evidence folder(s) for control {control_id}")
2349
+
2350
+ return evidence_folders
2351
+
2352
+ except ClientError as e:
2353
+ error_code = e.response["Error"]["Code"]
2354
+ error_message = e.response["Error"].get("Message", "")
2355
+ logger.error(f"Error fetching evidence folders for control {control_id}: {error_code} - {error_message}")
2356
+ raise
2357
+
2358
+ def _parse_evidence_timestamp(self, time_str: Any) -> Optional[datetime]:
2359
+ """
2360
+ Parse evidence timestamp from various formats.
2361
+
2362
+ :param Any time_str: Timestamp string or datetime object
2363
+ :return: Parsed datetime object or None if unparseable
2364
+ :rtype: Optional[datetime]
2365
+ """
2366
+ if not time_str:
2367
+ return None
2368
+
2369
+ try:
2370
+ if isinstance(time_str, datetime):
2371
+ # If it's already a datetime object, ensure it's naive UTC
2372
+ if time_str.tzinfo:
2373
+ # Convert to UTC and make naive
2374
+ return time_str.astimezone(timezone.utc).replace(tzinfo=None)
2375
+ return time_str
2376
+
2377
+ if not isinstance(time_str, str):
2378
+ return None
2379
+
2380
+ # Handle ISO format with T separator
2381
+ if "T" in time_str:
2382
+ # Parse ISO format and convert to naive UTC
2383
+ dt = datetime.fromisoformat(time_str.replace("Z", "+00:00"))
2384
+ # If timezone-aware, convert to UTC and make naive
2385
+ if dt.tzinfo:
2386
+ dt = dt.astimezone(timezone.utc).replace(tzinfo=None)
2387
+ return dt
2388
+
2389
+ # Handle format with space separator
2390
+ parts = time_str.split("-")
2391
+ if len(parts) > 3: # Has timezone offset
2392
+ # Remove timezone suffix and parse as naive datetime
2393
+ time_without_tz = time_str.rsplit("-", 1)[0] if "+" not in time_str else time_str.rsplit("+", 1)[0]
2394
+ date_format = "%Y-%m-%d %H:%M:%S.%f" if "." in time_without_tz else "%Y-%m-%d %H:%M:%S"
2395
+ # Parse as naive datetime (already in UTC after removing timezone)
2396
+ return datetime.strptime(time_without_tz.strip(), date_format)
2397
+
2398
+ # Simple date format
2399
+ date_format = "%Y-%m-%d %H:%M:%S.%f" if "." in time_str else "%Y-%m-%d %H:%M:%S"
2400
+ return datetime.strptime(time_str, date_format)
2401
+
2402
+ except (ValueError, TypeError) as e:
2403
+ logger.debug(f"Error parsing timestamp '{time_str}': {e}")
2404
+ return None
2405
+
2406
+ def _filter_evidence_by_date(
2407
+ self, evidence_items: List[Dict[str, Any]], yesterday_start: datetime, yesterday_end: datetime
2408
+ ) -> List[Dict[str, Any]]:
2409
+ """
2410
+ Filter evidence items to only include those from yesterday's date range.
2411
+
2412
+ :param List[Dict[str, Any]] evidence_items: Evidence items to filter
2413
+ :param datetime yesterday_start: Start of yesterday's date range
2414
+ :param datetime yesterday_end: End of yesterday's date range
2415
+ :return: Filtered evidence items
2416
+ :rtype: List[Dict[str, Any]]
2417
+ """
2418
+ filtered_evidence = []
2419
+
2420
+ for item in evidence_items:
2421
+ time_str = item.get("time")
2422
+ evidence_time = self._parse_evidence_timestamp(time_str)
2423
+
2424
+ if not evidence_time:
2425
+ logger.debug(f"Skipping evidence item without parseable timestamp: {item.get('id', 'unknown')}")
2426
+ continue
2427
+
2428
+ if yesterday_start <= evidence_time < yesterday_end:
2429
+ filtered_evidence.append(item)
2430
+ else:
2431
+ logger.debug(
2432
+ f"Excluding evidence item {item.get('id', 'unknown')} with timestamp {time_str} "
2433
+ f"(outside yesterday's range)"
2434
+ )
2435
+
2436
+ return filtered_evidence
2437
+
2438
+ def _collect_evidence_from_folder(
2439
+ self,
2440
+ assessment_id: str,
2441
+ control_set_id: str,
2442
+ evidence_folder_id: str,
2443
+ evidence_items: List[Dict[str, Any]],
2444
+ ) -> None:
2445
+ """
2446
+ Collect evidence from a single evidence folder with pagination.
2447
+
2448
+ Filters evidence items to only include those from yesterday's date to prevent
2449
+ including evidence from the current day.
2450
+
2451
+ :param str assessment_id: Assessment ID
2452
+ :param str control_set_id: Control set ID
2453
+ :param str evidence_folder_id: Evidence folder ID
2454
+ :param List[Dict[str, Any]] evidence_items: List to append evidence to
2455
+ :return: None
2456
+ :rtype: None
2457
+ """
2458
+ next_token = None
2459
+ max_results = 50 # API maximum per request
2460
+
2461
+ # Calculate yesterday's date range in UTC
2462
+ now_utc = datetime.now(timezone.utc).replace(tzinfo=None)
2463
+ yesterday_start = (now_utc - timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)
2464
+ yesterday_end = yesterday_start + timedelta(days=1)
2465
+
2466
+ logger.debug(
2467
+ f"Filtering evidence items for date range: {yesterday_start.isoformat()} to {yesterday_end.isoformat()} UTC"
2468
+ )
2469
+
2470
+ while len(evidence_items) < self.max_evidence_per_control:
2471
+ # Build request parameters
2472
+ params = {
2473
+ "assessmentId": assessment_id,
2474
+ "controlSetId": control_set_id,
2475
+ "evidenceFolderId": evidence_folder_id,
2476
+ "maxResults": min(max_results, self.max_evidence_per_control - len(evidence_items)),
2477
+ }
2478
+
2479
+ if next_token:
2480
+ params["nextToken"] = next_token
2481
+
2482
+ logger.debug(
2483
+ f"Calling get_evidence_by_evidence_folder: "
2484
+ f"evidenceFolderId={evidence_folder_id}, maxResults={params['maxResults']}"
2485
+ )
2486
+
2487
+ # Make API call
2488
+ response = self.client.get_evidence_by_evidence_folder(**params)
2489
+
2490
+ # Get evidence items from the folder
2491
+ # Note: We don't filter by date here since the folder is already from the correct date
2492
+ page_evidence = response.get("evidence", [])
2493
+
2494
+ # Add all evidence items from the folder (folder date filtering already applied)
2495
+ evidence_items.extend(page_evidence)
2496
+
2497
+ logger.debug(
2498
+ f"Retrieved {len(page_evidence)} evidence item(s) from folder {evidence_folder_id} "
2499
+ f"(total so far: {len(evidence_items)})"
2500
+ )
2501
+
2502
+ # Check for more pages
2503
+ next_token = response.get("nextToken")
2504
+ if not next_token or not page_evidence:
2505
+ break
2506
+
2507
+ def _process_evidence_folders(
2508
+ self,
2509
+ assessment_id: str,
2510
+ control_set_id: str,
2511
+ control_id: str,
2512
+ evidence_folders: List[Dict[str, Any]],
2513
+ control_name: Optional[str] = None,
2514
+ ) -> List[Dict[str, Any]]:
2515
+ """
2516
+ Process yesterday's evidence folders for a control and collect evidence items.
2517
+
2518
+ This method filters evidence folders to only process folders from yesterday (UTC timezone)
2519
+ to avoid collecting duplicate or stale evidence. If no evidence folders exist for yesterday,
2520
+ an info message is logged and an empty list is returned.
2521
+
2522
+ :param str assessment_id: Assessment ID
2523
+ :param str control_set_id: Control set ID
2524
+ :param str control_id: Control ID (AWS internal ID)
2525
+ :param List[Dict[str, Any]] evidence_folders: List of evidence folders to filter
2526
+ :param Optional[str] control_name: Human-readable control name (e.g., AC-1, AU-2)
2527
+ :return: List of evidence items from yesterday's folders
2528
+ :rtype: List[Dict[str, Any]]
2529
+ """
2530
+ evidence_items = []
2531
+ display_control = control_name if control_name else control_id
2532
+
2533
+ # Calculate yesterday's date in UTC (AWS Audit Manager uses UTC timestamps)
2534
+ yesterday = (datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=1)).strftime("%Y-%m-%d")
2535
+
2536
+ logger.debug(f"Filtering evidence folders for date: {yesterday} (UTC) for control {display_control}")
2537
+
2538
+ # Helper function to match folder date with yesterday's date
2539
+ def matches_yesterday(folder: Dict[str, Any]) -> bool:
2540
+ folder_date = folder.get("date")
2541
+ if not folder_date:
2542
+ return False
2543
+
2544
+ # AWS returns datetime objects - convert to date string for comparison
2545
+ if isinstance(folder_date, datetime):
2546
+ folder_date_str = folder_date.strftime("%Y-%m-%d")
2547
+ else:
2548
+ # Handle string dates (defensive programming)
2549
+ folder_date_str = str(folder_date)
2550
+
2551
+ return folder_date_str.startswith(yesterday)
2552
+
2553
+ # Filter folders to only yesterday's date
2554
+ yesterdays_folders = [f for f in evidence_folders if matches_yesterday(f)]
2555
+
2556
+ logger.info(
2557
+ f"Found {len(yesterdays_folders)} evidence folder(s) for {yesterday} "
2558
+ f"out of {len(evidence_folders)} total folder(s) for control {display_control}"
2559
+ )
2560
+
2561
+ if not yesterdays_folders:
2562
+ logger.info(f"No evidence folders found for yesterday ({yesterday}) for control {display_control}")
2563
+ return evidence_items
2564
+
2565
+ # Process yesterday's folders
2566
+ for folder in yesterdays_folders:
2567
+ if len(evidence_items) >= self.max_evidence_per_control:
2568
+ logger.debug(
2569
+ f"Reached max evidence limit ({self.max_evidence_per_control}), "
2570
+ f"stopping evidence collection for control {display_control}"
2571
+ )
2572
+ break
2573
+
2574
+ evidence_folder_id = folder.get("id")
2575
+ folder_date = folder.get("date")
2576
+ folder_evidence_count = folder.get("evidenceResourcesIncludedCount", 0)
2577
+
2578
+ logger.info(
2579
+ f"Processing evidence folder {evidence_folder_id} for control {display_control} "
2580
+ f"(date: {folder_date}, evidence count: {folder_evidence_count})"
2581
+ )
2582
+
2583
+ self._collect_evidence_from_folder(assessment_id, control_set_id, evidence_folder_id, evidence_items)
2584
+
2585
+ return evidence_items
2586
+
2587
+ def _get_control_evidence(self, assessment_id: str, control_set_id: str, control_id: str) -> List[Dict[str, Any]]:
2588
+ """
2589
+ Get evidence items for a specific control from AWS Audit Manager.
2590
+
2591
+ AWS Audit Manager organizes evidence in daily evidence folders within each control.
2592
+ This method retrieves all evidence folders for the control, filters for yesterday's
2593
+ folders (UTC timezone), then collects evidence from those folders up to
2594
+ max_evidence_per_control items.
2595
+
2596
+ :param str assessment_id: Assessment ID
2597
+ :param str control_set_id: Control set ID
2598
+ :param str control_id: Control ID (AWS internal ID)
2599
+ :return: List of evidence items from yesterday's evidence folders
2600
+ :rtype: List[Dict[str, Any]]
2601
+ """
2602
+ evidence_items = []
2603
+
2604
+ try:
2605
+ # Step 1: Get all evidence folders for this control
2606
+ evidence_folders = self._get_evidence_folders(assessment_id, control_set_id, control_id)
2607
+
2608
+ if not evidence_folders:
2609
+ logger.debug(f"No evidence folders found for control {control_id}")
2610
+ return evidence_items
2611
+
2612
+ # Step 2: Filter for yesterday's folders and collect evidence
2613
+ evidence_items = self._process_evidence_folders(assessment_id, control_set_id, control_id, evidence_folders)
2614
+
2615
+ logger.debug(
2616
+ f"Retrieved {len(evidence_items)} evidence item(s) for control {control_id} "
2617
+ f"from {len(evidence_folders)} evidence folder(s)"
2618
+ )
2619
+
2620
+ except ClientError as e:
2621
+ error_code = e.response["Error"]["Code"]
2622
+ error_message = e.response["Error"].get("Message", "")
2623
+ if error_code in ["ResourceNotFoundException", "AccessDeniedException"]:
2624
+ logger.warning(f"Cannot access evidence for control {control_id}: {error_code} - {error_message}")
2625
+ else:
2626
+ logger.error(f"Error retrieving evidence for control {control_id}: {error_code} - {error_message}")
2627
+
2628
+ return evidence_items
2629
+
2630
+ def _create_consolidated_evidence_record(
2631
+ self,
2632
+ assessment: Dict[str, Any],
2633
+ assessment_name: str,
2634
+ all_evidence_items: List[Dict[str, Any]],
2635
+ control_summary: Dict[str, Dict[str, Any]],
2636
+ controls_processed: int,
2637
+ ) -> Optional[Any]:
2638
+ """
2639
+ Create a RegScale Evidence record with consolidated evidence from all controls.
2640
+
2641
+ Saves a consolidated JSONL file to the artifacts directory and attaches it to
2642
+ a single Evidence record per assessment.
2643
+
2644
+ :param Dict[str, Any] assessment: Full assessment data from AWS Audit Manager
2645
+ :param str assessment_name: Assessment name
2646
+ :param List[Dict[str, Any]] all_evidence_items: All evidence items across controls
2647
+ :param Dict[str, Dict[str, Any]] control_summary: Summary of evidence per control
2648
+ :param int controls_processed: Number of controls processed
2649
+ :return: Created Evidence record or None
2650
+ :rtype: Optional[Any]
2651
+ """
2652
+ from datetime import datetime, timedelta
2653
+
2654
+ from regscale.models.regscale_models.evidence import Evidence
2655
+
2656
+ # Build evidence title and description
2657
+ scan_date = get_current_datetime(dt_format="%Y-%m-%d")
2658
+ title = f"AWS Audit Manager Evidence - {assessment_name} - {scan_date}"
2659
+
2660
+ # Analyze evidence and build description
2661
+ description, safe_assessment_name, file_name = self._build_evidence_description(
2662
+ assessment=assessment,
2663
+ assessment_name=assessment_name,
2664
+ all_evidence_items=all_evidence_items,
2665
+ control_summary=control_summary,
2666
+ controls_processed=controls_processed,
2667
+ scan_date=scan_date,
2668
+ )
2669
+
2670
+ # Calculate due date
2671
+ due_date = (datetime.now() + timedelta(days=self.evidence_frequency)).isoformat()
2672
+
2673
+ try:
2674
+ # Create Evidence record
2675
+ evidence = Evidence(
2676
+ title=title,
2677
+ description=description,
2678
+ status="Collected",
2679
+ updateFrequency=self.evidence_frequency,
2680
+ dueDate=due_date,
2681
+ )
2682
+
2683
+ created_evidence = evidence.create()
2684
+ if not created_evidence or not created_evidence.id:
2685
+ logger.error("Failed to create evidence record")
2686
+ return None
2687
+
2688
+ logger.info(f"Created evidence record {created_evidence.id}: {title}")
2689
+
2690
+ # Save and upload consolidated evidence file
2691
+ self._upload_consolidated_evidence(
2692
+ created_evidence_id=created_evidence.id,
2693
+ safe_assessment_name=safe_assessment_name,
2694
+ scan_date=scan_date,
2695
+ file_name=file_name,
2696
+ all_evidence_items=all_evidence_items,
2697
+ )
2698
+
2699
+ # Link evidence to SSP
2700
+ ssp_linked = self._link_evidence_to_ssp(created_evidence.id)
2701
+
2702
+ # Link evidence to control implementations
2703
+ controls_linked = self._link_evidence_to_control_implementations(created_evidence.id, control_summary)
2704
+
2705
+ # Log summary of evidence mapping per policy
2706
+ if ssp_linked:
2707
+ logger.info(
2708
+ f"Successfully mapped 1 evidence to {controls_linked} controls "
2709
+ f"(Evidence ID: {created_evidence.id}, SSP ID: {self.plan_id})"
2710
+ )
2711
+ else:
2712
+ logger.warning(
2713
+ f"Evidence record (ID: {created_evidence.id}) created but could not be linked to SSP. "
2714
+ f"Linked to {controls_linked} control(s)"
2715
+ )
2716
+
2717
+ return created_evidence
2718
+
2719
+ except Exception as ex:
2720
+ logger.error(
2721
+ f"Failed to create consolidated evidence for assessment {assessment_name}: {ex}", exc_info=True
2722
+ )
2723
+ return None
2724
+
2725
+ def _get_compliance_check_from_resources(self, resources_included: List[Dict[str, Any]]) -> Optional[str]:
2726
+ """
2727
+ Determine compliance check status from resource-level checks.
2728
+
2729
+ :param List[Dict[str, Any]] resources_included: List of resources with complianceCheck fields
2730
+ :return: Aggregated compliance check status or None
2731
+ :rtype: Optional[str]
2732
+ """
2733
+ if not resources_included:
2734
+ return None
2735
+
2736
+ resource_checks = [r.get("complianceCheck") for r in resources_included]
2737
+ if "FAILED" in resource_checks:
2738
+ return "FAILED"
2739
+ if any(check == "COMPLIANT" for check in resource_checks):
2740
+ return "COMPLIANT"
2741
+ if any(check == "NOT_APPLICABLE" for check in resource_checks):
2742
+ return "NOT_APPLICABLE"
2743
+ return None
2744
+
2745
+ def _track_failed_control(
2746
+ self, control_id: Optional[str], failed_controls: set, failed_by_control: Dict[str, int]
2747
+ ) -> None:
2748
+ """
2749
+ Track a failed control for reporting.
2750
+
2751
+ :param Optional[str] control_id: Control ID to track
2752
+ :param set failed_controls: Set of failed control IDs
2753
+ :param Dict[str, int] failed_by_control: Dictionary tracking failure count per control
2754
+ :return: None
2755
+ :rtype: None
2756
+ """
2757
+ if control_id:
2758
+ failed_controls.add(control_id)
2759
+ failed_by_control[control_id] = failed_by_control.get(control_id, 0) + 1
2760
+
2761
+ def _count_compliance_status(
2762
+ self,
2763
+ compliance_check: Optional[str],
2764
+ evidence: Dict[str, Any],
2765
+ compliant_count: int,
2766
+ failed_count: int,
2767
+ not_applicable_count: int,
2768
+ inconclusive_count: int,
2769
+ failed_controls: set,
2770
+ failed_by_control: Dict[str, int],
2771
+ ) -> tuple:
2772
+ """
2773
+ Count compliance status and update tracking collections.
2774
+
2775
+ :param Optional[str] compliance_check: Compliance check status
2776
+ :param Dict[str, Any] evidence: Evidence item with control_id
2777
+ :param int compliant_count: Current compliant count
2778
+ :param int failed_count: Current failed count
2779
+ :param int not_applicable_count: Current not applicable count
2780
+ :param int inconclusive_count: Current inconclusive count
2781
+ :param set failed_controls: Set of failed control IDs
2782
+ :param Dict[str, int] failed_by_control: Dictionary tracking failure count per control
2783
+ :return: Tuple of updated counts
2784
+ :rtype: tuple
2785
+ """
2786
+ if compliance_check == "FAILED":
2787
+ failed_count += 1
2788
+ control_id = evidence.get("_control_id")
2789
+ self._track_failed_control(control_id, failed_controls, failed_by_control)
2790
+ elif compliance_check == "COMPLIANT":
2791
+ compliant_count += 1
2792
+ elif compliance_check == "NOT_APPLICABLE":
2793
+ not_applicable_count += 1
2794
+ else:
2795
+ inconclusive_count += 1
2796
+
2797
+ return compliant_count, failed_count, not_applicable_count, inconclusive_count
2798
+
2799
+ def _analyze_compliance_results(self, evidence_items: List[Dict[str, Any]]) -> Dict[str, Any]:
2800
+ """
2801
+ Analyze compliance results from evidence items to determine pass/fail statistics.
2802
+
2803
+ Checks both root-level and resource-level complianceCheck fields (same logic
2804
+ as _aggregate_evidence_compliance).
2805
+
2806
+ :param List[Dict[str, Any]] evidence_items: Evidence items to analyze
2807
+ :return: Dictionary with compliance statistics
2808
+ :rtype: Dict[str, Any]
2809
+ """
2810
+ compliant_count = 0
2811
+ failed_count = 0
2812
+ inconclusive_count = 0
2813
+ not_applicable_count = 0
2814
+ failed_controls = set()
2815
+ failed_by_control = {}
2816
+
2817
+ for evidence in evidence_items:
2818
+ # Check root-level complianceCheck first
2819
+ compliance_check = evidence.get("complianceCheck")
2820
+
2821
+ # If no root-level check, look in resourcesIncluded
2822
+ if compliance_check is None:
2823
+ resources_included = evidence.get("resourcesIncluded", [])
2824
+ compliance_check = self._get_compliance_check_from_resources(resources_included)
2825
+
2826
+ # Count compliance results
2827
+ compliant_count, failed_count, not_applicable_count, inconclusive_count = self._count_compliance_status(
2828
+ compliance_check,
2829
+ evidence,
2830
+ compliant_count,
2831
+ failed_count,
2832
+ not_applicable_count,
2833
+ inconclusive_count,
2834
+ failed_controls,
2835
+ failed_by_control,
2836
+ )
2837
+
2838
+ total_with_checks = compliant_count + failed_count
2839
+
2840
+ return {
2841
+ "compliant_count": compliant_count,
2842
+ "failed_count": failed_count,
2843
+ "inconclusive_count": inconclusive_count,
2844
+ "not_applicable_count": not_applicable_count,
2845
+ "total_with_checks": total_with_checks,
2846
+ "failed_controls": failed_controls,
2847
+ "failed_by_control": failed_by_control,
2848
+ }
2849
+
2850
+ def _extract_assessment_metadata(self, assessment: Dict[str, Any]) -> Dict[str, Any]:
2851
+ """
2852
+ Extract assessment metadata for description building.
2853
+
2854
+ :param Dict[str, Any] assessment: Assessment data
2855
+ :return: Dictionary with extracted metadata
2856
+ :rtype: Dict[str, Any]
2857
+ """
2858
+ assessment_arn = assessment.get("arn", "N/A")
2859
+ assessment_id = assessment_arn.split("/")[-1] if assessment_arn != "N/A" else "N/A"
2860
+ assessment_status = assessment.get("status", "Unknown")
2861
+
2862
+ framework = assessment.get("framework", {})
2863
+ framework_name = framework.get("metadata", {}).get("name", "N/A")
2864
+ framework_type = framework.get("type", "N/A")
2865
+
2866
+ aws_account = assessment.get("awsAccount", {})
2867
+ account_id = aws_account.get("id", "N/A")
2868
+ account_name = aws_account.get("name", "N/A")
2869
+ account_display = f"{account_name} ({account_id})" if account_name != "N/A" else account_id
2870
+
2871
+ metadata = assessment.get("metadata", {})
2872
+ assessment_description = metadata.get("description", "")
2873
+ creation_time = metadata.get("creationTime")
2874
+ last_updated = metadata.get("lastUpdated")
2875
+
2876
+ return {
2877
+ "assessment_id": assessment_id,
2878
+ "assessment_status": assessment_status,
2879
+ "framework_name": framework_name,
2880
+ "framework_type": framework_type,
2881
+ "account_display": account_display,
2882
+ "assessment_description": assessment_description,
2883
+ "creation_time": creation_time,
2884
+ "last_updated": last_updated,
2885
+ }
2886
+
2887
+ def _add_timestamp_to_description(self, description_parts: list, label: str, timestamp: Any) -> None:
2888
+ """
2889
+ Add formatted timestamp to description if valid.
2890
+
2891
+ :param list description_parts: List to append timestamp HTML to
2892
+ :param str label: Label for the timestamp (e.g., 'Created', 'Last Updated')
2893
+ :param Any timestamp: Timestamp value (int, float, or datetime)
2894
+ :return: None
2895
+ :rtype: None
2896
+ """
2897
+ from datetime import datetime
2898
+
2899
+ if not timestamp:
2900
+ return
2901
+
2902
+ try:
2903
+ if isinstance(timestamp, (int, float)):
2904
+ dt_obj = datetime.fromtimestamp(timestamp)
2905
+ else:
2906
+ dt_obj = timestamp
2907
+ description_parts.append(
2908
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}{label}:{HTML_STRONG_CLOSE} "
2909
+ f"{dt_obj.strftime('%Y-%m-%d %H:%M:%S')}{HTML_LI_CLOSE}"
2910
+ )
2911
+ except Exception:
2912
+ pass
2913
+
2914
+ def _add_assessment_details_section(
2915
+ self, description_parts: list, assessment_name: str, metadata: Dict[str, Any]
2916
+ ) -> None:
2917
+ """
2918
+ Add assessment details section to description.
2919
+
2920
+ :param list description_parts: List to append HTML sections to
2921
+ :param str assessment_name: Assessment name
2922
+ :param Dict[str, Any] metadata: Extracted metadata dictionary
2923
+ :return: None
2924
+ :rtype: None
2925
+ """
2926
+ description_parts.extend(
2927
+ [
2928
+ "<h1>AWS Audit Manager Evidence</h1>",
2929
+ f"{HTML_P_OPEN}{HTML_STRONG_OPEN}Assessment:{HTML_STRONG_CLOSE} {assessment_name}{HTML_P_CLOSE}",
2930
+ ]
2931
+ )
2932
+
2933
+ if metadata["assessment_description"]:
2934
+ description_parts.append(f"{HTML_P_OPEN}{metadata['assessment_description']}{HTML_P_CLOSE}")
2935
+
2936
+ description_parts.extend(
2937
+ [
2938
+ "<h2>Assessment Details</h2>",
2939
+ HTML_UL_OPEN,
2940
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}Assessment ID:{HTML_STRONG_CLOSE} "
2941
+ f"<code>{metadata['assessment_id']}</code>{HTML_LI_CLOSE}",
2942
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}Status:{HTML_STRONG_CLOSE} "
2943
+ f"{metadata['assessment_status']}{HTML_LI_CLOSE}",
2944
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}AWS Account:{HTML_STRONG_CLOSE} "
2945
+ f"{metadata['account_display']}{HTML_LI_CLOSE}",
2946
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}Framework:{HTML_STRONG_CLOSE} "
2947
+ f"{metadata['framework_name']} ({metadata['framework_type']}){HTML_LI_CLOSE}",
2948
+ ]
2949
+ )
2950
+
2951
+ self._add_timestamp_to_description(description_parts, "Created", metadata["creation_time"])
2952
+ self._add_timestamp_to_description(description_parts, "Last Updated", metadata["last_updated"])
2953
+ description_parts.append(HTML_UL_CLOSE)
2954
+
2955
+ def _add_compliance_results_section(
2956
+ self,
2957
+ description_parts: list,
2958
+ compliance_stats: Dict[str, Any],
2959
+ control_summary: Dict[str, Dict[str, Any]],
2960
+ total_evidence_count: int,
2961
+ ) -> None:
2962
+ """
2963
+ Add compliance results section to description.
2964
+
2965
+ :param list description_parts: List to append HTML sections to
2966
+ :param Dict[str, Any] compliance_stats: Compliance statistics
2967
+ :param Dict[str, Dict[str, Any]] control_summary: Control summary
2968
+ :param int total_evidence_count: Total evidence count
2969
+ :return: None
2970
+ :rtype: None
2971
+ """
2972
+ description_parts.extend(
2973
+ [
2974
+ "<h2>Compliance Results</h2>",
2975
+ HTML_UL_OPEN,
2976
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}Evidence with Compliance Checks:{HTML_STRONG_CLOSE} "
2977
+ f"{compliance_stats['total_with_checks']:,} of {total_evidence_count:,}{HTML_LI_CLOSE}",
2978
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}Compliant:{HTML_STRONG_CLOSE} "
2979
+ f"{compliance_stats['compliant_count']:,}{HTML_LI_CLOSE}",
2980
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}Failed:{HTML_STRONG_CLOSE} "
2981
+ f"<span style='color: red;'>{compliance_stats['failed_count']:,}</span>{HTML_LI_CLOSE}",
2982
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}Not Applicable:{HTML_STRONG_CLOSE} "
2983
+ f"{compliance_stats['not_applicable_count']:,}{HTML_LI_CLOSE}",
2984
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}Inconclusive:{HTML_STRONG_CLOSE} "
2985
+ f"{compliance_stats['inconclusive_count']:,}{HTML_LI_CLOSE}",
2986
+ HTML_UL_CLOSE,
2987
+ ]
2988
+ )
2989
+
2990
+ if compliance_stats["failed_controls"]:
2991
+ description_parts.extend(
2992
+ [
2993
+ "<h3>Failed Controls</h3>",
2994
+ f"{HTML_P_OPEN}<span style='color: red;'>{HTML_STRONG_OPEN}The following controls have "
2995
+ f"failed compliance checks:{HTML_STRONG_CLOSE}</span>{HTML_P_CLOSE}",
2996
+ HTML_UL_OPEN,
2997
+ ]
2998
+ )
2999
+ for control_id in sorted(compliance_stats["failed_controls"]):
3000
+ control_name = control_summary.get(control_id, {}).get("control_name", control_id)
3001
+ failed_evidence_count = compliance_stats["failed_by_control"].get(control_id, 0)
3002
+ description_parts.append(
3003
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}{control_id}:{HTML_STRONG_CLOSE} {control_name} "
3004
+ f"({failed_evidence_count} failed evidence item(s)){HTML_LI_CLOSE}"
3005
+ )
3006
+ description_parts.append(HTML_UL_CLOSE)
3007
+
3008
+ def _add_evidence_summary_sections(
3009
+ self,
3010
+ description_parts: list,
3011
+ total_evidence_count: int,
3012
+ control_summary: Dict[str, Dict[str, Any]],
3013
+ controls_processed: int,
3014
+ analysis: Dict[str, Any],
3015
+ ) -> None:
3016
+ """
3017
+ Add evidence summary and related sections to description.
3018
+
3019
+ :param list description_parts: List to append HTML sections to
3020
+ :param int total_evidence_count: Total evidence count
3021
+ :param Dict[str, Dict[str, Any]] control_summary: Control summary
3022
+ :param int controls_processed: Number of controls processed
3023
+ :param Dict[str, Any] analysis: Evidence analysis results
3024
+ :return: None
3025
+ :rtype: None
3026
+ """
3027
+ description_parts.extend(
3028
+ [
3029
+ "<h2>Evidence Summary</h2>",
3030
+ HTML_UL_OPEN,
3031
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}Total Evidence Items:{HTML_STRONG_CLOSE} "
3032
+ f"{total_evidence_count:,}{HTML_LI_CLOSE}",
3033
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}Controls with Evidence:{HTML_STRONG_CLOSE} "
3034
+ f"{len(control_summary)}{HTML_LI_CLOSE}",
3035
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}Controls Processed:{HTML_STRONG_CLOSE} "
3036
+ f"{controls_processed}{HTML_LI_CLOSE}",
3037
+ HTML_UL_CLOSE,
3038
+ "<h2>Controls Summary</h2>",
3039
+ HTML_UL_OPEN,
3040
+ ]
3041
+ )
3042
+
3043
+ for control_id in sorted(control_summary.keys()):
3044
+ control_info = control_summary[control_id]
3045
+ description_parts.append(
3046
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}{control_id}:{HTML_STRONG_CLOSE} "
3047
+ f"{control_info['evidence_count']:,} items - <em>{control_info['control_name']}</em>{HTML_LI_CLOSE}"
3048
+ )
3049
+
3050
+ description_parts.append(HTML_UL_CLOSE)
3051
+ description_parts.append("<h2>Data Sources</h2>")
3052
+ description_parts.append(HTML_UL_OPEN)
3053
+ for source in sorted(analysis["data_sources"]):
3054
+ description_parts.append(f"{HTML_LI_OPEN}{source}{HTML_LI_CLOSE}")
3055
+ description_parts.append(HTML_UL_CLOSE)
3056
+
3057
+ if analysis["event_names"]:
3058
+ description_parts.append("<h2>Event Types (Sample)</h2>")
3059
+ description_parts.append(HTML_UL_OPEN)
3060
+ for event_name in sorted(list(analysis["event_names"])[:10]):
3061
+ description_parts.append(f"{HTML_LI_OPEN}<code>{event_name}</code>{HTML_LI_CLOSE}")
3062
+ description_parts.append(HTML_UL_CLOSE)
3063
+
3064
+ if analysis["date_range_start"] and analysis["date_range_end"]:
3065
+ description_parts.extend(
3066
+ [
3067
+ "<h2>Evidence Date Range</h2>",
3068
+ HTML_UL_OPEN,
3069
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}From:{HTML_STRONG_CLOSE} "
3070
+ f"{analysis['date_range_start'].strftime('%Y-%m-%d %H:%M:%S')}{HTML_LI_CLOSE}",
3071
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}To:{HTML_STRONG_CLOSE} "
3072
+ f"{analysis['date_range_end'].strftime('%Y-%m-%d %H:%M:%S')}{HTML_LI_CLOSE}",
3073
+ HTML_UL_CLOSE,
3074
+ ]
3075
+ )
3076
+
3077
+ def _build_evidence_description(
3078
+ self,
3079
+ assessment: Dict[str, Any],
3080
+ assessment_name: str,
3081
+ all_evidence_items: List[Dict[str, Any]],
3082
+ control_summary: Dict[str, Dict[str, Any]],
3083
+ controls_processed: int,
3084
+ scan_date: str,
3085
+ ) -> tuple:
3086
+ """
3087
+ Build evidence description with analysis of evidence items using HTML formatting.
3088
+
3089
+ :param Dict[str, Any] assessment: Full assessment data from AWS Audit Manager
3090
+ :param str assessment_name: Assessment name
3091
+ :param List[Dict[str, Any]] all_evidence_items: All evidence items
3092
+ :param Dict[str, Dict[str, Any]] control_summary: Control summary
3093
+ :param int controls_processed: Number of controls processed
3094
+ :param str scan_date: Scan date
3095
+ :return: Tuple of (description, safe_assessment_name, file_name)
3096
+ :rtype: tuple
3097
+ """
3098
+ # Extract assessment metadata
3099
+ metadata = self._extract_assessment_metadata(assessment)
3100
+
3101
+ # Analyze evidence items
3102
+ total_evidence_count = len(all_evidence_items)
3103
+ analysis = self._analyze_evidence_items(all_evidence_items)
3104
+ compliance_stats = self._analyze_compliance_results(all_evidence_items)
3105
+
3106
+ # Build description parts
3107
+ description_parts = []
3108
+
3109
+ self._add_assessment_details_section(description_parts, assessment_name, metadata)
3110
+ self._add_compliance_results_section(description_parts, compliance_stats, control_summary, total_evidence_count)
3111
+ self._add_evidence_summary_sections(
3112
+ description_parts, total_evidence_count, control_summary, controls_processed, analysis
3113
+ )
3114
+
3115
+ # Generate safe filename from assessment name
3116
+ safe_assessment_name = assessment_name.replace(" ", "_").replace("/", "_")[:50]
3117
+ file_name = f"audit_manager_evidence_{safe_assessment_name}_{scan_date}.jsonl.gz"
3118
+
3119
+ description_parts.extend(
3120
+ [
3121
+ "<h2>Attached Files</h2>",
3122
+ HTML_UL_OPEN,
3123
+ f"{HTML_LI_OPEN}{HTML_STRONG_OPEN}{file_name}{HTML_STRONG_CLOSE} (gzipped JSONL format){HTML_LI_CLOSE}",
3124
+ HTML_UL_CLOSE,
3125
+ ]
3126
+ )
3127
+
3128
+ return "\n".join(description_parts), safe_assessment_name, file_name
3129
+
3130
+ def _convert_evidence_timestamp(self, evidence_time: Any) -> Optional[Any]:
3131
+ """
3132
+ Convert evidence timestamp to datetime if needed.
3133
+
3134
+ :param Any evidence_time: Evidence time value (datetime, int, float, or other)
3135
+ :return: Datetime object or None if invalid
3136
+ :rtype: Optional[Any]
3137
+ """
3138
+ from datetime import datetime
3139
+
3140
+ # Handle both datetime objects and timestamp integers
3141
+ if isinstance(evidence_time, (int, float)):
3142
+ return datetime.fromtimestamp(evidence_time / 1000)
3143
+ if isinstance(evidence_time, datetime):
3144
+ return evidence_time
3145
+ return None
3146
+
3147
+ def _update_date_range(self, evidence_time: Any, date_range_start: Any, date_range_end: Any) -> tuple:
3148
+ """
3149
+ Update date range with new evidence timestamp.
3150
+
3151
+ :param Any evidence_time: Datetime object
3152
+ :param Any date_range_start: Current start datetime or None
3153
+ :param Any date_range_end: Current end datetime or None
3154
+ :return: Tuple of (updated_start, updated_end)
3155
+ :rtype: tuple
3156
+ """
3157
+ updated_start = date_range_start
3158
+ updated_end = date_range_end
3159
+
3160
+ if not date_range_start or evidence_time < date_range_start:
3161
+ updated_start = evidence_time
3162
+ if not date_range_end or evidence_time > date_range_end:
3163
+ updated_end = evidence_time
3164
+
3165
+ return updated_start, updated_end
3166
+
3167
+ def _process_evidence_item(
3168
+ self, item: Dict[str, Any], data_sources: set, event_names: set, date_range_start: Any, date_range_end: Any
3169
+ ) -> tuple:
3170
+ """
3171
+ Process a single evidence item to extract analysis data.
3172
+
3173
+ :param Dict[str, Any] item: Evidence item
3174
+ :param set data_sources: Set to update with data sources
3175
+ :param set event_names: Set to update with event names
3176
+ :param Any date_range_start: Current start datetime
3177
+ :param Any date_range_end: Current end datetime
3178
+ :return: Tuple of (date_range_start, date_range_end)
3179
+ :rtype: tuple
3180
+ """
3181
+ # Extract data source
3182
+ if "dataSource" in item:
3183
+ data_sources.add(item["dataSource"])
3184
+
3185
+ # Extract event name
3186
+ if "eventName" in item:
3187
+ event_names.add(item["eventName"])
3188
+
3189
+ # Extract and process timestamp
3190
+ if "time" in item:
3191
+ evidence_time = self._convert_evidence_timestamp(item["time"])
3192
+ if evidence_time:
3193
+ date_range_start, date_range_end = self._update_date_range(
3194
+ evidence_time, date_range_start, date_range_end
3195
+ )
3196
+
3197
+ return date_range_start, date_range_end
3198
+
3199
+ def _analyze_evidence_items(self, evidence_items: List[Dict[str, Any]]) -> Dict[str, Any]:
3200
+ """
3201
+ Analyze evidence items to extract data sources, event names, and date ranges.
3202
+
3203
+ :param List[Dict[str, Any]] evidence_items: Evidence items to analyze
3204
+ :return: Dictionary with analysis results
3205
+ :rtype: Dict[str, Any]
3206
+ """
3207
+ data_sources = set()
3208
+ event_names = set()
3209
+ date_range_start = None
3210
+ date_range_end = None
3211
+
3212
+ for item in evidence_items:
3213
+ date_range_start, date_range_end = self._process_evidence_item(
3214
+ item, data_sources, event_names, date_range_start, date_range_end
3215
+ )
3216
+
3217
+ return {
3218
+ "data_sources": data_sources,
3219
+ "event_names": event_names,
3220
+ "date_range_start": date_range_start,
3221
+ "date_range_end": date_range_end,
3222
+ }
3223
+
3224
+ def _compress_evidence_data(self, evidence_items: List[Dict[str, Any]]) -> tuple:
3225
+ """
3226
+ Compress evidence data to gzipped JSONL format.
3227
+
3228
+ :param List[Dict[str, Any]] evidence_items: Evidence items to compress
3229
+ :return: Tuple of (compressed_data, uncompressed_size_mb, compressed_size_mb, compression_ratio)
3230
+ :rtype: tuple
3231
+ """
3232
+ import gzip
3233
+ from io import BytesIO
3234
+
3235
+ jsonl_content = "\n".join([json.dumps(item, default=str) for item in evidence_items])
3236
+
3237
+ # Compress the JSONL content in memory
3238
+ compressed_buffer = BytesIO()
3239
+ with gzip.open(compressed_buffer, "wt", encoding="utf-8", compresslevel=9) as gz_file:
3240
+ gz_file.write(jsonl_content)
3241
+
3242
+ compressed_data = compressed_buffer.getvalue()
3243
+ compressed_size_mb = len(compressed_data) / (1024 * 1024)
3244
+ uncompressed_size_mb = len(jsonl_content.encode("utf-8")) / (1024 * 1024)
3245
+ compression_ratio = (1 - (len(compressed_data) / len(jsonl_content.encode("utf-8")))) * 100
3246
+
3247
+ return compressed_data, uncompressed_size_mb, compressed_size_mb, compression_ratio
3248
+
3249
+ def _upload_consolidated_evidence(
3250
+ self,
3251
+ created_evidence_id: int,
3252
+ safe_assessment_name: str,
3253
+ scan_date: str,
3254
+ file_name: str,
3255
+ all_evidence_items: List[Dict[str, Any]],
3256
+ ) -> None:
3257
+ """
3258
+ Save and upload consolidated evidence file as gzipped JSONL.
3259
+
3260
+ :param int created_evidence_id: Evidence record ID
3261
+ :param str safe_assessment_name: Safe assessment name for filename
3262
+ :param str scan_date: Scan date
3263
+ :param str file_name: File name for upload (will be modified to add .gz)
3264
+ :param List[Dict[str, Any]] all_evidence_items: All evidence items
3265
+ """
3266
+ from regscale.core.app.api import Api
3267
+ from regscale.models.regscale_models.file import File
3268
+
3269
+ # Save consolidated JSONL file to artifacts directory (already gzipped)
3270
+ artifacts_file_path = self._save_consolidated_evidence_file(
3271
+ assessment_name=safe_assessment_name, scan_date=scan_date, evidence_items=all_evidence_items
3272
+ )
3273
+
3274
+ if artifacts_file_path:
3275
+ logger.info(f"Saved consolidated evidence to: {artifacts_file_path}")
3276
+
3277
+ # Compress evidence data for upload to RegScale
3278
+ compressed_data, uncompressed_size_mb, compressed_size_mb, compression_ratio = self._compress_evidence_data(
3279
+ all_evidence_items
3280
+ )
3281
+
3282
+ logger.info(
3283
+ "Compressed evidence: %.2f MB -> %.2f MB (%.1f%% reduction)",
3284
+ uncompressed_size_mb,
3285
+ compressed_size_mb,
3286
+ compression_ratio,
3287
+ )
3288
+
3289
+ # Upload with .gz extension
3290
+ api = Api()
3291
+ gzipped_file_name = file_name if file_name.endswith(".gz") else f"{file_name}.gz"
3292
+
3293
+ success = File.upload_file_to_regscale(
3294
+ file_name=gzipped_file_name,
3295
+ parent_id=created_evidence_id,
3296
+ parent_module="evidence",
3297
+ api=api,
3298
+ file_data=compressed_data,
3299
+ tags=f"aws,audit-manager,{safe_assessment_name.lower()}",
3300
+ )
3301
+
3302
+ if success:
3303
+ logger.info(f"Uploaded compressed evidence file for evidence {created_evidence_id}")
3304
+ else:
3305
+ logger.warning(f"Failed to upload compressed evidence file for evidence {created_evidence_id}")
3306
+
3307
+ def _save_consolidated_evidence_file(
3308
+ self, assessment_name: str, scan_date: str, evidence_items: List[Dict[str, Any]]
3309
+ ) -> Optional[str]:
3310
+ """
3311
+ Save consolidated evidence items to gzipped JSONL file in artifacts directory.
3312
+
3313
+ :param str assessment_name: Safe assessment name for filename
3314
+ :param str scan_date: Scan date string
3315
+ :param List[Dict[str, Any]] evidence_items: All evidence items
3316
+ :return: File path if successful, None otherwise
3317
+ :rtype: Optional[str]
3318
+ """
3319
+ import gzip
3320
+
3321
+ try:
3322
+ # Ensure artifacts directory exists
3323
+ artifacts_dir = os.path.join("artifacts", "aws", "audit_manager_evidence")
3324
+ os.makedirs(artifacts_dir, exist_ok=True)
3325
+
3326
+ # Create file path with .gz extension
3327
+ file_name = f"audit_manager_evidence_{assessment_name}_{scan_date}.jsonl.gz"
3328
+ file_path = os.path.join(artifacts_dir, file_name)
3329
+
3330
+ # Write compressed JSONL file
3331
+ with gzip.open(file_path, "wt", encoding="utf-8", compresslevel=9) as f:
3332
+ for item in evidence_items:
3333
+ f.write(json.dumps(item, default=str) + "\n")
3334
+
3335
+ # Get file size for logging
3336
+ file_size_mb = os.path.getsize(file_path) / (1024 * 1024)
3337
+ logger.info(f"Saved {len(evidence_items)} evidence items to {file_path} ({file_size_mb:.2f} MB compressed)")
3338
+ return file_path
3339
+
3340
+ except IOError as ex:
3341
+ logger.error(f"Failed to save consolidated evidence file: {ex}")
3342
+ return None
3343
+
3344
+ def _link_evidence_to_ssp(self, evidence_id: int) -> bool:
3345
+ """
3346
+ Link evidence to Security Plan.
3347
+
3348
+ :param int evidence_id: Evidence record ID
3349
+ :return: True if successfully linked, False otherwise
3350
+ :rtype: bool
3351
+ """
3352
+ from regscale.models.regscale_models.evidence_mapping import EvidenceMapping
3353
+
3354
+ mapping = EvidenceMapping(evidenceID=evidence_id, mappedID=self.plan_id, mappingType="securityplans")
3355
+
3356
+ try:
3357
+ created_mapping = mapping.create()
3358
+ if created_mapping and created_mapping.id:
3359
+ logger.info(f"Linked evidence {evidence_id} to SSP {self.plan_id}")
3360
+ return True
3361
+ else:
3362
+ logger.warning(f"Failed to link evidence to SSP: mapping.create() returned {created_mapping}")
3363
+ return False
3364
+ except Exception as ex:
3365
+ logger.warning(f"Failed to link evidence to SSP: {ex}")
3366
+ return False
3367
+
3368
+ def _link_evidence_to_control_implementations(
3369
+ self, evidence_id: int, control_summary: Dict[str, Dict[str, Any]]
3370
+ ) -> int:
3371
+ """
3372
+ Link evidence to control implementations in the security plan.
3373
+
3374
+ For each control in the control_summary, looks up the corresponding control implementation
3375
+ in the security plan and creates an evidence mapping to link the evidence to that
3376
+ implementation. This ensures evidence is properly associated with both the security plan
3377
+ and the specific control implementations.
3378
+
3379
+ Note: Uses mappingType="controls" to properly link evidence to control implementations,
3380
+ matching the RegScale evidence mapping model expectations.
3381
+
3382
+ :param int evidence_id: Evidence record ID
3383
+ :param Dict[str, Dict[str, Any]] control_summary: Summary of controls with evidence
3384
+ :return: Number of controls successfully linked
3385
+ :rtype: int
3386
+ """
3387
+ if not control_summary:
3388
+ logger.debug("No control summary provided, skipping control implementation evidence mapping")
3389
+ return 0
3390
+
3391
+ implementations = self._prepare_control_implementations()
3392
+ linked_count, failed_control_ids = self._link_controls_to_evidence(
3393
+ evidence_id, control_summary, implementations
3394
+ )
3395
+ self._log_linking_results(evidence_id, linked_count, failed_control_ids, len(control_summary))
3396
+ return linked_count
3397
+
3398
+ def _prepare_control_implementations(self):
3399
+ """
3400
+ Get and cache control implementations for the security plan.
3401
+
3402
+ :return: List of control implementations
3403
+ :rtype: list
3404
+ """
3405
+ implementations = self._get_control_implementations()
3406
+ if implementations:
3407
+ self._build_control_lookup_cache(implementations)
3408
+ logger.debug(f"Found {len(implementations)} control implementations in SSP {self.plan_id}")
3409
+ else:
3410
+ logger.warning(
3411
+ f"No control implementations found in SSP {self.plan_id} - evidence will not be linked to controls"
3412
+ )
3413
+ return implementations
3414
+
3415
+ def _link_controls_to_evidence(
3416
+ self, evidence_id: int, control_summary: Dict[str, Dict[str, Any]], implementations
3417
+ ) -> tuple:
3418
+ """
3419
+ Link all controls in summary to evidence.
3420
+
3421
+ :param int evidence_id: Evidence record ID
3422
+ :param Dict[str, Dict[str, Any]] control_summary: Summary of controls
3423
+ :param implementations: List of control implementations
3424
+ :return: Tuple of (linked_count, failed_control_ids)
3425
+ :rtype: tuple
3426
+ """
3427
+ logger.info(
3428
+ f"Linking evidence {evidence_id} to {len(control_summary)} control implementation(s) in SSP {self.plan_id}..."
3429
+ )
3430
+ control_ids_sample = list(control_summary.keys())[:5]
3431
+ logger.debug(f"Sample control IDs to be linked: {control_ids_sample}")
3432
+
3433
+ linked_count = 0
3434
+ failed_control_ids = []
3435
+
3436
+ for control_id in control_summary.keys():
3437
+ success = self._link_single_control_to_evidence(evidence_id, control_id, implementations)
3438
+ if success:
3439
+ linked_count += 1
3440
+ else:
3441
+ failed_control_ids.append(control_id)
3442
+
3443
+ return linked_count, failed_control_ids
3444
+
3445
+ def _link_single_control_to_evidence(self, evidence_id: int, control_id: str, implementations) -> bool:
3446
+ """
3447
+ Link a single control to evidence.
3448
+
3449
+ :param int evidence_id: Evidence record ID
3450
+ :param str control_id: Control ID to link
3451
+ :param implementations: List of control implementations
3452
+ :return: True if successfully linked, False otherwise
3453
+ :rtype: bool
3454
+ """
3455
+ from regscale.models.regscale_models.evidence_mapping import EvidenceMapping
3456
+
3457
+ try:
3458
+ control_impl, _ = self._find_matching_implementation(control_id, implementations)
3459
+
3460
+ if not control_impl or not control_impl.id:
3461
+ logger.info(
3462
+ f"Control implementation not found in SSP for control {control_id}, skipping evidence mapping"
3463
+ )
3464
+ return False
3465
+
3466
+ mapping = EvidenceMapping(evidenceID=evidence_id, mappedID=control_impl.id, mappingType="controls")
3467
+ created_mapping = mapping.create()
3468
+
3469
+ if created_mapping and created_mapping.id:
3470
+ logger.info(
3471
+ f"Successfully linked evidence {evidence_id} to control implementation {control_impl.id} "
3472
+ f"(control {control_id}, mapping ID: {created_mapping.id})"
3473
+ )
3474
+ return True
3475
+
3476
+ logger.warning(
3477
+ f"Failed to create evidence mapping for control {control_id}: "
3478
+ f"mapping.create() returned {created_mapping}"
3479
+ )
3480
+ return False
3481
+
3482
+ except Exception as ex:
3483
+ logger.warning(f"Failed to link evidence to control implementation {control_id}: {ex}")
3484
+ return False
3485
+
3486
+ def _log_linking_results(
3487
+ self, evidence_id: int, linked_count: int, failed_control_ids: list, total_controls: int
3488
+ ) -> None:
3489
+ """
3490
+ Log the results of evidence linking.
3491
+
3492
+ :param int evidence_id: Evidence record ID
3493
+ :param int linked_count: Number of controls successfully linked
3494
+ :param list failed_control_ids: List of failed control IDs
3495
+ :param int total_controls: Total number of controls attempted
3496
+ :rtype: None
3497
+ """
3498
+ failed_count = len(failed_control_ids)
3499
+
3500
+ if linked_count > 0:
3501
+ logger.info(
3502
+ f"Successfully linked evidence {evidence_id} to {linked_count} of {total_controls} "
3503
+ f"control implementation(s) in SSP {self.plan_id}"
3504
+ )
3505
+ if failed_count > 0:
3506
+ self._log_failed_controls(failed_control_ids)
3507
+ elif failed_count > 0:
3508
+ logger.warning(
3509
+ f"Failed to link evidence {evidence_id} to any control implementations. "
3510
+ f"{failed_count} control(s) not found in SSP {self.plan_id}: {', '.join(failed_control_ids[:10])}"
3511
+ f"{f' (and {len(failed_control_ids) - 10} more)' if len(failed_control_ids) > 10 else ''}"
3512
+ )
3513
+
3514
+ def _log_failed_controls(self, failed_control_ids: list) -> None:
3515
+ """
3516
+ Log failed control IDs.
3517
+
3518
+ :param list failed_control_ids: List of failed control IDs
3519
+ :rtype: None
3520
+ """
3521
+ failed_count = len(failed_control_ids)
3522
+ failed_sample = ", ".join(failed_control_ids[:10])
3523
+ remaining_msg = f" (and {failed_count - 10} more)" if failed_count > 10 else ""
3524
+ logger.warning(
3525
+ f"{failed_count} control(s) could not be linked (not found in SSP): {failed_sample}{remaining_msg}"
3526
+ )
3527
+
3528
+ def _handle_permission_error(self, permission_name: str, error: ClientError) -> bool:
3529
+ """
3530
+ Handle permission test error and log appropriately.
3531
+
3532
+ :param str permission_name: Permission being tested
3533
+ :param ClientError error: AWS ClientError exception
3534
+ :return: False (permission denied/failed)
3535
+ :rtype: bool
3536
+ """
3537
+ if error.response["Error"]["Code"] in ["AccessDeniedException", "UnauthorizedException"]:
3538
+ logger.error(f"✗ {permission_name} - DENIED: {error.response['Error']['Message']}")
3539
+ else:
3540
+ logger.warning(f"? {permission_name} - Error: {error}")
3541
+ return False
3542
+
3543
+ def _test_list_assessments_permission(self, permissions: Dict[str, bool]) -> None:
3544
+ """
3545
+ Test auditmanager:ListAssessments permission.
3546
+
3547
+ :param Dict[str, bool] permissions: Dictionary to update with test result
3548
+ :return: None
3549
+ :rtype: None
3550
+ """
3551
+ try:
3552
+ self.client.list_assessments(maxResults=1)
3553
+ permissions[IAM_PERMISSION_LIST_ASSESSMENTS] = True
3554
+ logger.info(f"✓ {IAM_PERMISSION_LIST_ASSESSMENTS} - OK")
3555
+ except ClientError as e:
3556
+ permissions[IAM_PERMISSION_LIST_ASSESSMENTS] = self._handle_permission_error(
3557
+ IAM_PERMISSION_LIST_ASSESSMENTS, e
3558
+ )
3559
+
3560
+ def _test_get_assessment_permission(self, permissions: Dict[str, bool]) -> None:
3561
+ """
3562
+ Test auditmanager:GetAssessment permission.
3563
+
3564
+ :param Dict[str, bool] permissions: Dictionary to update with test result
3565
+ :return: None
3566
+ :rtype: None
3567
+ """
3568
+ if not self.assessment_id:
3569
+ logger.info(f"⊘ {IAM_PERMISSION_GET_ASSESSMENT} - Skipped (no assessment_id provided)")
3570
+ return
3571
+
3572
+ try:
3573
+ self.client.get_assessment(assessmentId=self.assessment_id)
3574
+ permissions[IAM_PERMISSION_GET_ASSESSMENT] = True
3575
+ logger.info(f"✓ {IAM_PERMISSION_GET_ASSESSMENT} - OK")
3576
+ except ClientError as e:
3577
+ permissions[IAM_PERMISSION_GET_ASSESSMENT] = self._handle_permission_error(IAM_PERMISSION_GET_ASSESSMENT, e)
3578
+
3579
+ def _get_first_control_from_assessment(self) -> Optional[tuple]:
3580
+ """
3581
+ Get first control from assessment for permission testing.
3582
+
3583
+ :return: Tuple of (control_set_id, control_id) or None
3584
+ :rtype: Optional[tuple]
3585
+ """
3586
+ try:
3587
+ response = self.client.get_assessment(assessmentId=self.assessment_id)
3588
+ assessment = response.get("assessment", {})
3589
+ control_sets = assessment.get("framework", {}).get("controlSets", [])
3590
+
3591
+ if control_sets and control_sets[0].get("controls"):
3592
+ control_set_id = control_sets[0].get("id")
3593
+ control_id = control_sets[0].get("controls", [])[0].get("id")
3594
+ return control_set_id, control_id
3595
+ except ClientError as e:
3596
+ logger.warning(f"Could not retrieve assessment for permission testing: {e}")
3597
+ return None
3598
+
3599
+ def _test_evidence_folders_permission(
3600
+ self, permissions: Dict[str, bool], control_set_id: str, control_id: str
3601
+ ) -> None:
3602
+ """
3603
+ Test auditmanager:GetEvidenceFoldersByAssessmentControl permission.
3604
+
3605
+ :param Dict[str, bool] permissions: Dictionary to update with test result
3606
+ :param str control_set_id: Control set ID for testing
3607
+ :param str control_id: Control ID for testing
3608
+ :return: None
3609
+ :rtype: None
3610
+ """
3611
+ try:
3612
+ self.client.get_evidence_folders_by_assessment_control(
3613
+ assessmentId=self.assessment_id, controlSetId=control_set_id, controlId=control_id
3614
+ )
3615
+ permissions[IAM_PERMISSION_GET_EVIDENCE_FOLDERS] = True
3616
+ logger.info(f"✓ {IAM_PERMISSION_GET_EVIDENCE_FOLDERS} - OK")
3617
+ except ClientError as e:
3618
+ permissions[IAM_PERMISSION_GET_EVIDENCE_FOLDERS] = self._handle_permission_error(
3619
+ IAM_PERMISSION_GET_EVIDENCE_FOLDERS, e
3620
+ )
3621
+
3622
+ def _test_evidence_permissions(self, permissions: Dict[str, bool]) -> None:
3623
+ """
3624
+ Test evidence-related permissions (requires assessment with controls).
3625
+
3626
+ :param Dict[str, bool] permissions: Dictionary to update with test results
3627
+ :return: None
3628
+ :rtype: None
3629
+ """
3630
+ if not self.assessment_id:
3631
+ logger.info(f"⊘ {IAM_PERMISSION_GET_EVIDENCE_FOLDERS} - Skipped (no assessment_id provided)")
3632
+ logger.info("⊘ auditmanager:GetEvidenceByEvidenceFolder - Skipped (no assessment_id provided)")
3633
+ return
3634
+
3635
+ control_info = self._get_first_control_from_assessment()
3636
+ if control_info:
3637
+ control_set_id, control_id = control_info
3638
+ self._test_evidence_folders_permission(permissions, control_set_id, control_id)
3639
+ logger.info("⊘ auditmanager:GetEvidenceByEvidenceFolder - Cannot test (requires evidence folder ID)")
3640
+ else:
3641
+ logger.info(f"⊘ {IAM_PERMISSION_GET_EVIDENCE_FOLDERS} - Skipped (no controls in assessment)")
3642
+ logger.info("⊘ auditmanager:GetEvidenceByEvidenceFolder - Skipped (no controls in assessment)")
3643
+
3644
+ def _log_permission_test_summary(self, permissions: Dict[str, bool]) -> None:
3645
+ """
3646
+ Log summary of permission test results.
3647
+
3648
+ :param Dict[str, bool] permissions: Dictionary of permission test results
3649
+ :return: None
3650
+ :rtype: None
3651
+ """
3652
+ passed = sum(1 for v in permissions.values() if v)
3653
+ total = len(permissions)
3654
+ logger.info(f"\nPermission Test Summary: {passed}/{total} permissions verified")
3655
+
3656
+ if passed < total:
3657
+ logger.warning(
3658
+ "\nSome permissions are missing. Evidence collection may fail. "
3659
+ "Please ensure your IAM role/user has the required AWS Audit Manager permissions."
3660
+ )
3661
+ else:
3662
+ logger.info("\nAll tested permissions are OK!")
3663
+
3664
+ def test_iam_permissions(self) -> Dict[str, bool]:
3665
+ """
3666
+ Test IAM permissions required for AWS Audit Manager evidence collection.
3667
+
3668
+ Tests the following permissions:
3669
+ - auditmanager:ListAssessments
3670
+ - auditmanager:GetAssessment
3671
+ - auditmanager:GetEvidenceFoldersByAssessmentControl
3672
+ - auditmanager:GetEvidenceByEvidenceFolder
3673
+
3674
+ :return: Dictionary mapping permission names to test results (True=success, False=denied)
3675
+ :rtype: Dict[str, bool]
3676
+ """
3677
+ logger.info("Testing IAM permissions for AWS Audit Manager...")
3678
+ permissions = {}
3679
+
3680
+ self._test_list_assessments_permission(permissions)
3681
+ self._test_get_assessment_permission(permissions)
3682
+ self._test_evidence_permissions(permissions)
3683
+
3684
+ self._log_permission_test_summary(permissions)
3685
+
3686
+ return permissions
3687
+
3688
+ def _fetch_assessments_for_evidence(self) -> List[Dict[str, Any]]:
3689
+ """
3690
+ Fetch assessments for evidence collection.
3691
+
3692
+ :return: List of assessments
3693
+ :rtype: List[Dict[str, Any]]
3694
+ """
3695
+ if self.assessment_id:
3696
+ assessments = [self._get_assessment_details(self.assessment_id)]
3697
+ logger.debug(f"Using specific assessment ID: {self.assessment_id}")
3698
+ else:
3699
+ assessments = self._list_all_assessments()
3700
+ logger.debug(f"Listed {len(assessments)} total assessments")
3701
+ return assessments
3702
+
3703
+ def _log_assessment_details(self, assessments: List[Dict[str, Any]]) -> None:
3704
+ """
3705
+ Log assessment details before filtering.
3706
+
3707
+ :param List[Dict[str, Any]] assessments: List of assessments to log
3708
+ :return: None
3709
+ :rtype: None
3710
+ """
3711
+ for assessment in assessments:
3712
+ if not assessment:
3713
+ continue
3714
+ framework_info = self._get_assessment_framework(assessment)
3715
+ logger.debug(
3716
+ f"Assessment '{assessment.get('name', 'Unknown')}' - "
3717
+ f"Framework info: '{framework_info}', "
3718
+ f"ComplianceType: '{assessment.get('complianceType', '')}'"
3719
+ )
3720
+
3721
+ def _filter_assessments_by_framework(self, assessments: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
3722
+ """
3723
+ Filter assessments by framework.
3724
+
3725
+ IMPORTANT: When assessment_id is explicitly provided, skip framework filtering
3726
+ since the user has already specified exactly which assessment to use.
3727
+
3728
+ :param List[Dict[str, Any]] assessments: Assessments to filter
3729
+ :return: Filtered assessments
3730
+ :rtype: List[Dict[str, Any]]
3731
+ """
3732
+ # If assessment_id is provided, skip framework filtering
3733
+ if self.assessment_id:
3734
+ logger.info(f"Using specific assessment ID '{self.assessment_id}' - bypassing framework filter")
3735
+ return assessments
3736
+
3737
+ filtered_assessments = []
3738
+ for assessment in assessments:
3739
+ if not assessment:
3740
+ continue
3741
+
3742
+ framework_info = self._get_assessment_framework(assessment)
3743
+ if self._matches_framework(framework_info):
3744
+ filtered_assessments.append(assessment)
3745
+ logger.debug(f"✓ Assessment '{assessment.get('name')}' passed framework filter")
3746
+ else:
3747
+ logger.debug(
3748
+ f"✗ Assessment '{assessment.get('name')}' filtered out - "
3749
+ f"framework '{framework_info}' does not match '{self.framework}' "
3750
+ f"(custom name: '{self.custom_framework_name or 'N/A'}')"
3751
+ )
3752
+ return filtered_assessments
3753
+
3754
+ def _process_evidence_collection(self, assessments: List[Dict[str, Any]]) -> None:
3755
+ """
3756
+ Process evidence collection for filtered assessments.
3757
+
3758
+ :param List[Dict[str, Any]] assessments: Filtered assessments
3759
+ :return: None
3760
+ :rtype: None
3761
+ """
3762
+ if assessments:
3763
+ # Log assessment details for debugging
3764
+ if self.assessment_id:
3765
+ assessment = assessments[0]
3766
+ logger.info(
3767
+ f"Processing specific assessment: "
3768
+ f"Name: '{assessment.get('name', 'N/A')}', "
3769
+ f"Framework: '{assessment.get('framework', {}).get('metadata', {}).get('name', 'N/A')}', "
3770
+ f"Compliance Type: '{assessment.get('complianceType', 'N/A')}'"
3771
+ )
3772
+ else:
3773
+ logger.info(f"Found {len(assessments)} assessment(s) matching framework filter for evidence collection")
3774
+
3775
+ self.collect_assessment_evidence(assessments)
3776
+ else:
3777
+ # More detailed error message when assessment_id is provided
3778
+ if self.assessment_id:
3779
+ logger.error(
3780
+ f"Failed to find assessment with ID '{self.assessment_id}'. "
3781
+ f"Please verify the assessment ID exists and you have permission to access it."
3782
+ )
3783
+ else:
3784
+ logger.warning(
3785
+ f"No assessments found for evidence collection. "
3786
+ f"Framework: '{self.framework}', Custom name: '{self.custom_framework_name or 'N/A'}'"
3787
+ )
3788
+
3789
+ def _collect_evidence_before_sync(self, assessments: List[Dict[str, Any]]) -> None:
3790
+ """
3791
+ Collect evidence before compliance sync to enable evidence-based compliance determination.
3792
+
3793
+ This method collects evidence and stores it by control ID so that when compliance items
3794
+ are created, they have evidence available for proper pass/fail status determination.
3795
+
3796
+ :param List[Dict[str, Any]] assessments: Filtered assessments to collect evidence for
3797
+ :return: None
3798
+ :rtype: None
3799
+ """
3800
+ logger.info("Pre-collecting evidence for compliance items...")
3801
+
3802
+ for assessment in assessments:
3803
+ assessment_id = assessment.get("arn", "").split("/")[-1]
3804
+ assessment_name = assessment.get("name", assessment_id)
3805
+
3806
+ logger.info(f"Pre-collecting evidence for assessment: {assessment_name}")
3807
+
3808
+ # Use assessment-level evidence collection (fast method)
3809
+ all_evidence_items, control_summary = self._collect_evidence_assessment_level(assessment, assessment_id)
3810
+
3811
+ # Store evidence by control ID for use during compliance item creation
3812
+ for control_id, control_data in control_summary.items():
3813
+ # Normalize control ID to lowercase for consistent lookup
3814
+ control_id_lower = control_id.lower()
3815
+
3816
+ # Get evidence items for this control
3817
+ evidence_for_control = [
3818
+ item for item in all_evidence_items if item.get("_control_id", "").lower() == control_id_lower
3819
+ ]
3820
+
3821
+ # Store in the evidence map
3822
+ if evidence_for_control:
3823
+ self._evidence_by_control[control_id_lower] = evidence_for_control
3824
+ logger.debug(
3825
+ f"Stored {len(evidence_for_control)} evidence items for control {control_id} "
3826
+ f"({control_data.get('passed', 0)}P/{control_data.get('failed', 0)}F)"
3827
+ )
3828
+
3829
+ logger.info(
3830
+ f"Pre-collected evidence for {len(self._evidence_by_control)} controls "
3831
+ f"({sum(len(items) for items in self._evidence_by_control.values())} total evidence items)"
3832
+ )
3833
+
3834
+ def sync_compliance(self) -> None:
3835
+ """
3836
+ Sync compliance data from AWS Audit Manager to RegScale.
3837
+
3838
+ Extends the base sync_compliance method to add evidence collection.
3839
+
3840
+ CRITICAL: When using assessment-evidence-folders, evidence must be collected BEFORE
3841
+ compliance sync so that compliance items have proper pass/fail status for issue creation.
3842
+
3843
+ :return: None
3844
+ :rtype: None
3845
+ """
3846
+ # If evidence collection is enabled with assessment folders, collect evidence FIRST
3847
+ # so compliance items have proper pass/fail status from the start
3848
+ if self.collect_evidence and self.use_assessment_evidence_folders:
3849
+ logger.info("Collecting evidence before compliance sync to enable proper pass/fail determination...")
3850
+ try:
3851
+ # Fetch assessments
3852
+ assessments = self._fetch_assessments_for_evidence()
3853
+
3854
+ # Log assessment details
3855
+ self._log_assessment_details(assessments)
3856
+
3857
+ # Filter by framework
3858
+ filtered_assessments = self._filter_assessments_by_framework(assessments)
3859
+
3860
+ # Collect evidence and store by control ID for later use
3861
+ self._collect_evidence_before_sync(filtered_assessments)
3862
+
3863
+ except Exception as e:
3864
+ logger.error(f"Error during pre-sync evidence collection: {e}", exc_info=True)
3865
+ # Continue with sync even if evidence collection fails
3866
+
3867
+ # Call the base class sync_compliance to handle control assessments and issue creation
3868
+ # Compliance items will now have evidence available for proper pass/fail determination
3869
+ super().sync_compliance()
3870
+
3871
+ # After sync, create Evidence records and link to controls if evidence collection is enabled
3872
+ if self.collect_evidence:
3873
+ logger.info("Evidence collection enabled, creating Evidence records and linking to controls...")
3874
+ try:
3875
+ # Fetch assessments
3876
+ assessments = self._fetch_assessments_for_evidence()
3877
+
3878
+ # Log assessment details
3879
+ self._log_assessment_details(assessments)
3880
+
3881
+ # Filter by framework
3882
+ filtered_assessments = self._filter_assessments_by_framework(assessments)
3883
+
3884
+ # Process evidence collection (old inline method)
3885
+ self._process_evidence_collection(filtered_assessments)
3886
+
3887
+ except Exception as e:
3888
+ logger.error(f"Error during evidence collection: {e}", exc_info=True)
3889
+
3890
+ def create_finding_from_compliance_item(self, compliance_item):
3891
+ """
3892
+ Override to set identification field for AWS Audit Manager findings.
3893
+
3894
+ AWS Audit Manager findings represent assessment/audit results rather than
3895
+ vulnerability assessments, so we use "Assessment/Audit (Internal)" as the
3896
+ identification type.
3897
+
3898
+ :param ComplianceItem compliance_item: The compliance item to create a finding from
3899
+ :return: IntegrationFinding with proper identification set
3900
+ """
3901
+ # Call parent implementation to create the base finding
3902
+ finding = super().create_finding_from_compliance_item(compliance_item)
3903
+
3904
+ if finding:
3905
+ # Set the identification to Assessment/Audit (Internal) instead of default Vulnerability Assessment
3906
+ finding.identification = "Assessment/Audit (Internal)"
3907
+
3908
+ return finding