regscale-cli 6.21.2.0__py3-none-any.whl → 6.28.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. regscale/_version.py +1 -1
  2. regscale/airflow/hierarchy.py +2 -2
  3. regscale/core/app/api.py +5 -2
  4. regscale/core/app/application.py +36 -6
  5. regscale/core/app/internal/control_editor.py +73 -21
  6. regscale/core/app/internal/evidence.py +727 -204
  7. regscale/core/app/internal/login.py +4 -2
  8. regscale/core/app/internal/model_editor.py +219 -64
  9. regscale/core/app/utils/app_utils.py +86 -12
  10. regscale/core/app/utils/catalog_utils/common.py +1 -1
  11. regscale/core/login.py +21 -4
  12. regscale/core/utils/async_graphql_client.py +363 -0
  13. regscale/core/utils/date.py +77 -1
  14. regscale/dev/cli.py +26 -0
  15. regscale/dev/code_gen.py +109 -24
  16. regscale/dev/version.py +72 -0
  17. regscale/integrations/commercial/__init__.py +30 -2
  18. regscale/integrations/commercial/aws/audit_manager_compliance.py +3908 -0
  19. regscale/integrations/commercial/aws/cli.py +3107 -54
  20. regscale/integrations/commercial/aws/cloudtrail_control_mappings.py +333 -0
  21. regscale/integrations/commercial/aws/cloudtrail_evidence.py +501 -0
  22. regscale/integrations/commercial/aws/cloudwatch_control_mappings.py +357 -0
  23. regscale/integrations/commercial/aws/cloudwatch_evidence.py +490 -0
  24. regscale/integrations/commercial/{amazon → aws}/common.py +71 -19
  25. regscale/integrations/commercial/aws/config_compliance.py +914 -0
  26. regscale/integrations/commercial/aws/conformance_pack_mappings.py +198 -0
  27. regscale/integrations/commercial/aws/control_compliance_analyzer.py +439 -0
  28. regscale/integrations/commercial/aws/evidence_generator.py +283 -0
  29. regscale/integrations/commercial/aws/guardduty_control_mappings.py +340 -0
  30. regscale/integrations/commercial/aws/guardduty_evidence.py +1053 -0
  31. regscale/integrations/commercial/aws/iam_control_mappings.py +368 -0
  32. regscale/integrations/commercial/aws/iam_evidence.py +574 -0
  33. regscale/integrations/commercial/aws/inventory/__init__.py +338 -22
  34. regscale/integrations/commercial/aws/inventory/base.py +107 -5
  35. regscale/integrations/commercial/aws/inventory/resources/analytics.py +390 -0
  36. regscale/integrations/commercial/aws/inventory/resources/applications.py +234 -0
  37. regscale/integrations/commercial/aws/inventory/resources/audit_manager.py +513 -0
  38. regscale/integrations/commercial/aws/inventory/resources/cloudtrail.py +315 -0
  39. regscale/integrations/commercial/aws/inventory/resources/cloudtrail_logs_metadata.py +476 -0
  40. regscale/integrations/commercial/aws/inventory/resources/cloudwatch.py +191 -0
  41. regscale/integrations/commercial/aws/inventory/resources/compute.py +328 -9
  42. regscale/integrations/commercial/aws/inventory/resources/config.py +464 -0
  43. regscale/integrations/commercial/aws/inventory/resources/containers.py +74 -9
  44. regscale/integrations/commercial/aws/inventory/resources/database.py +481 -31
  45. regscale/integrations/commercial/aws/inventory/resources/developer_tools.py +253 -0
  46. regscale/integrations/commercial/aws/inventory/resources/guardduty.py +286 -0
  47. regscale/integrations/commercial/aws/inventory/resources/iam.py +470 -0
  48. regscale/integrations/commercial/aws/inventory/resources/inspector.py +476 -0
  49. regscale/integrations/commercial/aws/inventory/resources/integration.py +175 -61
  50. regscale/integrations/commercial/aws/inventory/resources/kms.py +447 -0
  51. regscale/integrations/commercial/aws/inventory/resources/machine_learning.py +358 -0
  52. regscale/integrations/commercial/aws/inventory/resources/networking.py +390 -67
  53. regscale/integrations/commercial/aws/inventory/resources/s3.py +394 -0
  54. regscale/integrations/commercial/aws/inventory/resources/security.py +268 -72
  55. regscale/integrations/commercial/aws/inventory/resources/securityhub.py +473 -0
  56. regscale/integrations/commercial/aws/inventory/resources/storage.py +288 -29
  57. regscale/integrations/commercial/aws/inventory/resources/systems_manager.py +657 -0
  58. regscale/integrations/commercial/aws/inventory/resources/vpc.py +655 -0
  59. regscale/integrations/commercial/aws/kms_control_mappings.py +288 -0
  60. regscale/integrations/commercial/aws/kms_evidence.py +879 -0
  61. regscale/integrations/commercial/aws/ocsf/__init__.py +7 -0
  62. regscale/integrations/commercial/aws/ocsf/constants.py +115 -0
  63. regscale/integrations/commercial/aws/ocsf/mapper.py +435 -0
  64. regscale/integrations/commercial/aws/org_control_mappings.py +286 -0
  65. regscale/integrations/commercial/aws/org_evidence.py +666 -0
  66. regscale/integrations/commercial/aws/s3_control_mappings.py +356 -0
  67. regscale/integrations/commercial/aws/s3_evidence.py +632 -0
  68. regscale/integrations/commercial/aws/scanner.py +1072 -205
  69. regscale/integrations/commercial/aws/security_hub.py +319 -0
  70. regscale/integrations/commercial/aws/session_manager.py +282 -0
  71. regscale/integrations/commercial/aws/ssm_control_mappings.py +291 -0
  72. regscale/integrations/commercial/aws/ssm_evidence.py +492 -0
  73. regscale/integrations/commercial/jira.py +489 -153
  74. regscale/integrations/commercial/microsoft_defender/defender.py +326 -5
  75. regscale/integrations/commercial/microsoft_defender/defender_api.py +348 -14
  76. regscale/integrations/commercial/microsoft_defender/defender_constants.py +157 -0
  77. regscale/integrations/commercial/qualys/__init__.py +167 -68
  78. regscale/integrations/commercial/qualys/scanner.py +305 -39
  79. regscale/integrations/commercial/sarif/sairf_importer.py +432 -0
  80. regscale/integrations/commercial/sarif/sarif_converter.py +67 -0
  81. regscale/integrations/commercial/sicura/api.py +79 -42
  82. regscale/integrations/commercial/sicura/commands.py +8 -2
  83. regscale/integrations/commercial/sicura/scanner.py +83 -44
  84. regscale/integrations/commercial/stigv2/ckl_parser.py +5 -5
  85. regscale/integrations/commercial/synqly/assets.py +133 -16
  86. regscale/integrations/commercial/synqly/edr.py +2 -8
  87. regscale/integrations/commercial/synqly/query_builder.py +536 -0
  88. regscale/integrations/commercial/synqly/ticketing.py +27 -0
  89. regscale/integrations/commercial/synqly/vulnerabilities.py +165 -28
  90. regscale/integrations/commercial/tenablev2/cis_parsers.py +453 -0
  91. regscale/integrations/commercial/tenablev2/cis_scanner.py +447 -0
  92. regscale/integrations/commercial/tenablev2/commands.py +146 -5
  93. regscale/integrations/commercial/tenablev2/scanner.py +1 -3
  94. regscale/integrations/commercial/tenablev2/stig_parsers.py +113 -57
  95. regscale/integrations/commercial/wizv2/WizDataMixin.py +1 -1
  96. regscale/integrations/commercial/wizv2/click.py +191 -76
  97. regscale/integrations/commercial/wizv2/compliance/__init__.py +15 -0
  98. regscale/integrations/commercial/wizv2/{policy_compliance_helpers.py → compliance/helpers.py} +78 -60
  99. regscale/integrations/commercial/wizv2/compliance_report.py +1592 -0
  100. regscale/integrations/commercial/wizv2/core/__init__.py +133 -0
  101. regscale/integrations/commercial/wizv2/{async_client.py → core/client.py} +7 -3
  102. regscale/integrations/commercial/wizv2/{constants.py → core/constants.py} +92 -89
  103. regscale/integrations/commercial/wizv2/core/file_operations.py +237 -0
  104. regscale/integrations/commercial/wizv2/fetchers/__init__.py +11 -0
  105. regscale/integrations/commercial/wizv2/{data_fetcher.py → fetchers/policy_assessment.py} +66 -9
  106. regscale/integrations/commercial/wizv2/file_cleanup.py +104 -0
  107. regscale/integrations/commercial/wizv2/issue.py +776 -28
  108. regscale/integrations/commercial/wizv2/models/__init__.py +0 -0
  109. regscale/integrations/commercial/wizv2/parsers/__init__.py +34 -0
  110. regscale/integrations/commercial/wizv2/{parsers.py → parsers/main.py} +1 -1
  111. regscale/integrations/commercial/wizv2/processors/__init__.py +11 -0
  112. regscale/integrations/commercial/wizv2/{finding_processor.py → processors/finding.py} +1 -1
  113. regscale/integrations/commercial/wizv2/reports.py +243 -0
  114. regscale/integrations/commercial/wizv2/sbom.py +1 -1
  115. regscale/integrations/commercial/wizv2/scanner.py +1031 -441
  116. regscale/integrations/commercial/wizv2/utils/__init__.py +48 -0
  117. regscale/integrations/commercial/wizv2/{utils.py → utils/main.py} +116 -61
  118. regscale/integrations/commercial/wizv2/variables.py +89 -3
  119. regscale/integrations/compliance_integration.py +1036 -151
  120. regscale/integrations/control_matcher.py +432 -0
  121. regscale/integrations/due_date_handler.py +333 -0
  122. regscale/integrations/milestone_manager.py +291 -0
  123. regscale/integrations/public/__init__.py +14 -0
  124. regscale/integrations/public/cci_importer.py +834 -0
  125. regscale/integrations/public/csam/__init__.py +0 -0
  126. regscale/integrations/public/csam/csam.py +938 -0
  127. regscale/integrations/public/csam/csam_agency_defined.py +179 -0
  128. regscale/integrations/public/csam/csam_common.py +154 -0
  129. regscale/integrations/public/csam/csam_controls.py +432 -0
  130. regscale/integrations/public/csam/csam_poam.py +124 -0
  131. regscale/integrations/public/fedramp/click.py +77 -6
  132. regscale/integrations/public/fedramp/docx_parser.py +10 -1
  133. regscale/integrations/public/fedramp/fedramp_cis_crm.py +675 -289
  134. regscale/integrations/public/fedramp/fedramp_five.py +1 -1
  135. regscale/integrations/public/fedramp/poam/scanner.py +75 -7
  136. regscale/integrations/public/fedramp/poam_export_v5.py +888 -0
  137. regscale/integrations/scanner_integration.py +1961 -430
  138. regscale/models/integration_models/CCI_List.xml +1 -0
  139. regscale/models/integration_models/aqua.py +2 -2
  140. regscale/models/integration_models/cisa_kev_data.json +805 -11
  141. regscale/models/integration_models/flat_file_importer/__init__.py +5 -8
  142. regscale/models/integration_models/nexpose.py +36 -10
  143. regscale/models/integration_models/qualys.py +3 -4
  144. regscale/models/integration_models/synqly_models/capabilities.json +1 -1
  145. regscale/models/integration_models/synqly_models/connectors/vulnerabilities.py +87 -18
  146. regscale/models/integration_models/synqly_models/filter_parser.py +332 -0
  147. regscale/models/integration_models/synqly_models/ocsf_mapper.py +124 -25
  148. regscale/models/integration_models/synqly_models/synqly_model.py +89 -16
  149. regscale/models/locking.py +12 -8
  150. regscale/models/platform.py +4 -2
  151. regscale/models/regscale_models/__init__.py +7 -0
  152. regscale/models/regscale_models/assessment.py +2 -1
  153. regscale/models/regscale_models/catalog.py +1 -1
  154. regscale/models/regscale_models/compliance_settings.py +251 -1
  155. regscale/models/regscale_models/component.py +1 -0
  156. regscale/models/regscale_models/control_implementation.py +236 -41
  157. regscale/models/regscale_models/control_objective.py +74 -5
  158. regscale/models/regscale_models/file.py +2 -0
  159. regscale/models/regscale_models/form_field_value.py +5 -3
  160. regscale/models/regscale_models/inheritance.py +44 -0
  161. regscale/models/regscale_models/issue.py +301 -102
  162. regscale/models/regscale_models/milestone.py +33 -14
  163. regscale/models/regscale_models/organization.py +3 -0
  164. regscale/models/regscale_models/regscale_model.py +310 -73
  165. regscale/models/regscale_models/security_plan.py +4 -2
  166. regscale/models/regscale_models/vulnerability.py +3 -3
  167. regscale/regscale.py +25 -4
  168. regscale/templates/__init__.py +0 -0
  169. regscale/utils/threading/threadhandler.py +20 -15
  170. regscale/validation/record.py +23 -1
  171. {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/METADATA +17 -33
  172. {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/RECORD +310 -111
  173. tests/core/__init__.py +0 -0
  174. tests/core/utils/__init__.py +0 -0
  175. tests/core/utils/test_async_graphql_client.py +472 -0
  176. tests/fixtures/test_fixture.py +13 -8
  177. tests/regscale/core/test_login.py +171 -4
  178. tests/regscale/integrations/commercial/__init__.py +0 -0
  179. tests/regscale/integrations/commercial/aws/__init__.py +0 -0
  180. tests/regscale/integrations/commercial/aws/test_audit_manager_compliance.py +1304 -0
  181. tests/regscale/integrations/commercial/aws/test_audit_manager_evidence_aggregation.py +341 -0
  182. tests/regscale/integrations/commercial/aws/test_aws_analytics_collector.py +260 -0
  183. tests/regscale/integrations/commercial/aws/test_aws_applications_collector.py +242 -0
  184. tests/regscale/integrations/commercial/aws/test_aws_audit_manager_collector.py +1155 -0
  185. tests/regscale/integrations/commercial/aws/test_aws_cloudtrail_collector.py +534 -0
  186. tests/regscale/integrations/commercial/aws/test_aws_config_collector.py +400 -0
  187. tests/regscale/integrations/commercial/aws/test_aws_developer_tools_collector.py +203 -0
  188. tests/regscale/integrations/commercial/aws/test_aws_guardduty_collector.py +315 -0
  189. tests/regscale/integrations/commercial/aws/test_aws_iam_collector.py +458 -0
  190. tests/regscale/integrations/commercial/aws/test_aws_inspector_collector.py +353 -0
  191. tests/regscale/integrations/commercial/aws/test_aws_inventory_integration.py +530 -0
  192. tests/regscale/integrations/commercial/aws/test_aws_kms_collector.py +919 -0
  193. tests/regscale/integrations/commercial/aws/test_aws_machine_learning_collector.py +237 -0
  194. tests/regscale/integrations/commercial/aws/test_aws_s3_collector.py +722 -0
  195. tests/regscale/integrations/commercial/aws/test_aws_scanner_integration.py +722 -0
  196. tests/regscale/integrations/commercial/aws/test_aws_securityhub_collector.py +792 -0
  197. tests/regscale/integrations/commercial/aws/test_aws_systems_manager_collector.py +918 -0
  198. tests/regscale/integrations/commercial/aws/test_aws_vpc_collector.py +996 -0
  199. tests/regscale/integrations/commercial/aws/test_cli_evidence.py +431 -0
  200. tests/regscale/integrations/commercial/aws/test_cloudtrail_control_mappings.py +452 -0
  201. tests/regscale/integrations/commercial/aws/test_cloudtrail_evidence.py +788 -0
  202. tests/regscale/integrations/commercial/aws/test_config_compliance.py +298 -0
  203. tests/regscale/integrations/commercial/aws/test_conformance_pack_mappings.py +200 -0
  204. tests/regscale/integrations/commercial/aws/test_control_compliance_analyzer.py +375 -0
  205. tests/regscale/integrations/commercial/aws/test_datetime_parsing.py +223 -0
  206. tests/regscale/integrations/commercial/aws/test_evidence_generator.py +386 -0
  207. tests/regscale/integrations/commercial/aws/test_guardduty_control_mappings.py +564 -0
  208. tests/regscale/integrations/commercial/aws/test_guardduty_evidence.py +1041 -0
  209. tests/regscale/integrations/commercial/aws/test_iam_control_mappings.py +718 -0
  210. tests/regscale/integrations/commercial/aws/test_iam_evidence.py +1375 -0
  211. tests/regscale/integrations/commercial/aws/test_kms_control_mappings.py +656 -0
  212. tests/regscale/integrations/commercial/aws/test_kms_evidence.py +1163 -0
  213. tests/regscale/integrations/commercial/aws/test_ocsf_mapper.py +370 -0
  214. tests/regscale/integrations/commercial/aws/test_org_control_mappings.py +546 -0
  215. tests/regscale/integrations/commercial/aws/test_org_evidence.py +1240 -0
  216. tests/regscale/integrations/commercial/aws/test_s3_control_mappings.py +672 -0
  217. tests/regscale/integrations/commercial/aws/test_s3_evidence.py +987 -0
  218. tests/regscale/integrations/commercial/aws/test_scanner_evidence.py +373 -0
  219. tests/regscale/integrations/commercial/aws/test_security_hub_config_filtering.py +539 -0
  220. tests/regscale/integrations/commercial/aws/test_session_manager.py +516 -0
  221. tests/regscale/integrations/commercial/aws/test_ssm_control_mappings.py +588 -0
  222. tests/regscale/integrations/commercial/aws/test_ssm_evidence.py +735 -0
  223. tests/regscale/integrations/commercial/conftest.py +28 -0
  224. tests/regscale/integrations/commercial/microsoft_defender/__init__.py +1 -0
  225. tests/regscale/integrations/commercial/microsoft_defender/test_defender.py +1517 -0
  226. tests/regscale/integrations/commercial/microsoft_defender/test_defender_api.py +1748 -0
  227. tests/regscale/integrations/commercial/microsoft_defender/test_defender_constants.py +327 -0
  228. tests/regscale/integrations/commercial/microsoft_defender/test_defender_scanner.py +487 -0
  229. tests/regscale/integrations/commercial/test_aws.py +3742 -0
  230. tests/regscale/integrations/commercial/test_burp.py +48 -0
  231. tests/regscale/integrations/commercial/test_crowdstrike.py +49 -0
  232. tests/regscale/integrations/commercial/test_dependabot.py +341 -0
  233. tests/regscale/integrations/commercial/test_gcp.py +1543 -0
  234. tests/regscale/integrations/commercial/test_gitlab.py +549 -0
  235. tests/regscale/integrations/commercial/test_ip_mac_address_length.py +84 -0
  236. tests/regscale/integrations/commercial/test_jira.py +2204 -0
  237. tests/regscale/integrations/commercial/test_npm_audit.py +42 -0
  238. tests/regscale/integrations/commercial/test_okta.py +1228 -0
  239. tests/regscale/integrations/commercial/test_sarif_converter.py +251 -0
  240. tests/regscale/integrations/commercial/test_sicura.py +349 -0
  241. tests/regscale/integrations/commercial/test_snow.py +423 -0
  242. tests/regscale/integrations/commercial/test_sonarcloud.py +394 -0
  243. tests/regscale/integrations/commercial/test_sqlserver.py +186 -0
  244. tests/regscale/integrations/commercial/test_stig.py +33 -0
  245. tests/regscale/integrations/commercial/test_stig_mapper.py +153 -0
  246. tests/regscale/integrations/commercial/test_stigv2.py +406 -0
  247. tests/regscale/integrations/commercial/test_wiz.py +1365 -0
  248. tests/regscale/integrations/commercial/test_wiz_inventory.py +256 -0
  249. tests/regscale/integrations/commercial/wizv2/__init__.py +339 -0
  250. tests/regscale/integrations/commercial/wizv2/compliance/__init__.py +1 -0
  251. tests/regscale/integrations/commercial/wizv2/compliance/test_helpers.py +903 -0
  252. tests/regscale/integrations/commercial/wizv2/core/__init__.py +1 -0
  253. tests/regscale/integrations/commercial/wizv2/core/test_auth.py +701 -0
  254. tests/regscale/integrations/commercial/wizv2/core/test_client.py +1037 -0
  255. tests/regscale/integrations/commercial/wizv2/core/test_file_operations.py +989 -0
  256. tests/regscale/integrations/commercial/wizv2/fetchers/__init__.py +1 -0
  257. tests/regscale/integrations/commercial/wizv2/fetchers/test_policy_assessment.py +805 -0
  258. tests/regscale/integrations/commercial/wizv2/parsers/__init__.py +1 -0
  259. tests/regscale/integrations/commercial/wizv2/parsers/test_main.py +1153 -0
  260. tests/regscale/integrations/commercial/wizv2/processors/__init__.py +1 -0
  261. tests/regscale/integrations/commercial/wizv2/processors/test_finding.py +671 -0
  262. tests/regscale/integrations/commercial/wizv2/test_WizDataMixin.py +537 -0
  263. tests/regscale/integrations/commercial/wizv2/test_click_comprehensive.py +851 -0
  264. tests/regscale/integrations/commercial/wizv2/test_compliance_report_comprehensive.py +910 -0
  265. tests/regscale/integrations/commercial/wizv2/test_compliance_report_normalization.py +138 -0
  266. tests/regscale/integrations/commercial/wizv2/test_file_cleanup.py +283 -0
  267. tests/regscale/integrations/commercial/wizv2/test_file_operations.py +260 -0
  268. tests/regscale/integrations/commercial/wizv2/test_issue.py +343 -0
  269. tests/regscale/integrations/commercial/wizv2/test_issue_comprehensive.py +1203 -0
  270. tests/regscale/integrations/commercial/wizv2/test_reports.py +497 -0
  271. tests/regscale/integrations/commercial/wizv2/test_sbom.py +643 -0
  272. tests/regscale/integrations/commercial/wizv2/test_scanner_comprehensive.py +805 -0
  273. tests/regscale/integrations/commercial/wizv2/test_wiz_click_client_id.py +165 -0
  274. tests/regscale/integrations/commercial/wizv2/test_wiz_compliance_report.py +1394 -0
  275. tests/regscale/integrations/commercial/wizv2/test_wiz_compliance_unit.py +341 -0
  276. tests/regscale/integrations/commercial/wizv2/test_wiz_control_normalization.py +138 -0
  277. tests/regscale/integrations/commercial/wizv2/test_wiz_findings_comprehensive.py +364 -0
  278. tests/regscale/integrations/commercial/wizv2/test_wiz_inventory_comprehensive.py +644 -0
  279. tests/regscale/integrations/commercial/wizv2/test_wiz_status_mapping.py +149 -0
  280. tests/regscale/integrations/commercial/wizv2/test_wizv2.py +1218 -0
  281. tests/regscale/integrations/commercial/wizv2/test_wizv2_utils.py +519 -0
  282. tests/regscale/integrations/commercial/wizv2/utils/__init__.py +1 -0
  283. tests/regscale/integrations/commercial/wizv2/utils/test_main.py +1523 -0
  284. tests/regscale/integrations/public/__init__.py +0 -0
  285. tests/regscale/integrations/public/fedramp/__init__.py +1 -0
  286. tests/regscale/integrations/public/fedramp/test_gen_asset_list.py +150 -0
  287. tests/regscale/integrations/public/fedramp/test_poam_export_v5.py +1293 -0
  288. tests/regscale/integrations/public/test_alienvault.py +220 -0
  289. tests/regscale/integrations/public/test_cci.py +1053 -0
  290. tests/regscale/integrations/public/test_cisa.py +1021 -0
  291. tests/regscale/integrations/public/test_emass.py +518 -0
  292. tests/regscale/integrations/public/test_fedramp.py +1152 -0
  293. tests/regscale/integrations/public/test_fedramp_cis_crm.py +3661 -0
  294. tests/regscale/integrations/public/test_file_uploads.py +506 -0
  295. tests/regscale/integrations/public/test_oscal.py +453 -0
  296. tests/regscale/integrations/test_compliance_status_mapping.py +406 -0
  297. tests/regscale/integrations/test_control_matcher.py +1421 -0
  298. tests/regscale/integrations/test_control_matching.py +155 -0
  299. tests/regscale/integrations/test_milestone_manager.py +408 -0
  300. tests/regscale/models/test_control_implementation.py +118 -3
  301. tests/regscale/models/test_form_field_value_integration.py +304 -0
  302. tests/regscale/models/test_issue.py +378 -1
  303. tests/regscale/models/test_module_integration.py +582 -0
  304. tests/regscale/models/test_tenable_integrations.py +811 -105
  305. regscale/integrations/commercial/wizv2/policy_compliance.py +0 -3057
  306. regscale/integrations/public/fedramp/mappings/fedramp_r4_parts.json +0 -7388
  307. regscale/integrations/public/fedramp/mappings/fedramp_r5_parts.json +0 -9605
  308. regscale/integrations/public/fedramp/parts_mapper.py +0 -107
  309. /regscale/integrations/commercial/{amazon → sarif}/__init__.py +0 -0
  310. /regscale/integrations/commercial/wizv2/{wiz_auth.py → core/auth.py} +0 -0
  311. {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/LICENSE +0 -0
  312. {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/WHEEL +0 -0
  313. {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/entry_points.txt +0 -0
  314. {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,834 @@
1
+ #!/usr/bin/env python
2
+ """RegScale CLI command to normalize CCI data from XML files."""
3
+ import datetime
4
+ import logging
5
+ import xml.etree.ElementTree as ET
6
+ from typing import Dict, List, Optional, Tuple
7
+
8
+ import click
9
+ from rich.progress import Progress, TaskID
10
+
11
+ from regscale.core.app.application import Application
12
+ from regscale.core.app.utils.app_utils import create_progress_object, error_and_exit
13
+ from regscale.models.regscale_models import Catalog, SecurityControl, CCI, ControlObjective
14
+
15
+ logger = logging.getLogger("regscale")
16
+
17
+ # RegScale date format constant
18
+ REGSCALE_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
19
+
20
+
21
+ class CCIImporter:
22
+ """Imports CCI data from XML files and maps to security controls."""
23
+
24
+ def __init__(self, xml_data: ET.Element, version: str = "5", verbose: bool = False):
25
+ """
26
+ Initialize the CCI importer.
27
+
28
+ :param ET.Element xml_data: The root element of the XML data
29
+ :param str version: NIST version to use for filtering
30
+ :param bool verbose: Whether to output verbose information
31
+ """
32
+ self.xml_data = xml_data
33
+ self.normalized_cci: Dict[str, List[Dict]] = {}
34
+ self.cci_grouped_by_index: Dict[str, str] = {}
35
+ self.verbose = verbose
36
+ self.reference_version = version
37
+ self._user_context: Optional[Tuple[Optional[str], int]] = None
38
+
39
+ @staticmethod
40
+ def _parse_control_id(ref_index: str) -> str:
41
+ """
42
+ Extract the main control_id from a reference index (e.g., 'AC-1 a 1 (b)' -> 'AC-1').
43
+
44
+ :param str ref_index: Reference index string to parse
45
+ :return: Main control ID
46
+ :rtype: str
47
+ """
48
+ parts = ref_index.strip().split()
49
+ return parts[0] if parts else ""
50
+
51
+ @staticmethod
52
+ def format_index(index: str) -> str:
53
+ """
54
+ Format index according to ControlObjective matching requirements.
55
+
56
+ Examples:
57
+ 'AC-1 a 1' -> 'AC-1(a)(1)'
58
+ 'IA-13 (03) (a)' -> 'IA-13(03)(a)'
59
+ 'AC-1 a 1 (a)' -> 'AC-1(a)(1)(a)'
60
+
61
+ :param str index: Raw index string from XML
62
+ :return: Formatted index string
63
+ :rtype: str
64
+ """
65
+ import re
66
+
67
+ index = index.strip()
68
+
69
+ # Pattern: match either (text) or non-whitespace text
70
+ pattern = r"\([^)]+\)|[^\s()]+"
71
+ parts = re.findall(pattern, index)
72
+
73
+ if len(parts) <= 1:
74
+ return parts[0] if parts else index
75
+
76
+ # First part is the base control (e.g., 'AC-1', 'IA-13')
77
+ result = parts[0]
78
+
79
+ # Process remaining parts
80
+ for part in parts[1:]:
81
+ if part.startswith("("):
82
+ # Already has parentheses, just append
83
+ result += part
84
+ else:
85
+ # Need to add parentheses
86
+ result += f"({part})"
87
+
88
+ return result
89
+
90
+ @staticmethod
91
+ def parse_objective_id(objective_id: str) -> Tuple[Optional[str], Optional[str]]:
92
+ """
93
+ Parse an objective otherId to extract control base and part.
94
+
95
+ Supports both NIST 800-53 Revision 4 and 5 formats:
96
+
97
+ Revision 5 Examples:
98
+ "ac-1_smt.a" -> ("AC-1", "a")
99
+ "ac-2.3_smt.a" -> ("AC-2(3)", "a")
100
+ "au-10.1_smt.a" -> ("AU-10(1)", "a")
101
+ "ac-2.4_smt" -> ("AC-2(4)", None)
102
+
103
+ Revision 4 Examples:
104
+ "ac-1_smt.a.1" -> ("AC-1", "a")
105
+ "ac-1_smt.b.2" -> ("AC-1", "b")
106
+ "ac-2.3_smt.d" -> ("AC-2(3)", "d")
107
+
108
+ :param str objective_id: Objective otherId value
109
+ :return: Tuple of (control_base, part_letter or None)
110
+ :rtype: Tuple[Optional[str], Optional[str]]
111
+ """
112
+ import re
113
+
114
+ # Pattern 1: xx-nn[.nn]_smt.x[.nn] (with part letter, optional subpart for rev 4)
115
+ # Matches: ac-1_smt.a, ac-1_smt.a.1, ac-2.3_smt.d
116
+ match = re.match(r"^([a-z]+)-(\d+)(?:\.(\d+))?_smt\.([a-z]+)(?:\.\d+)?$", objective_id.lower())
117
+
118
+ if match:
119
+ family = match.group(1).upper()
120
+ control_num = match.group(2)
121
+ enhancement = match.group(3)
122
+ part = match.group(4)
123
+ else:
124
+ # Pattern 2: xx-nn[.nn]_smt[.x][.nn] (without clear part letter, or just enhancement)
125
+ # Matches: ac-2.4_smt, ac-1_smt
126
+ match = re.match(r"^([a-z]+)-(\d+)(?:\.(\d+))?_smt(?:\.[a-z]+)?(?:\.\d+)?$", objective_id.lower())
127
+ if not match:
128
+ return None, None
129
+
130
+ family = match.group(1).upper()
131
+ control_num = match.group(2)
132
+ enhancement = match.group(3)
133
+ part = None
134
+
135
+ if enhancement:
136
+ # Enhancement like AC-2(3)
137
+ control_base = f"{family}-{control_num}({enhancement})"
138
+ else:
139
+ # Base control like AC-1
140
+ control_base = f"{family}-{control_num}"
141
+
142
+ return control_base, part
143
+
144
+ @staticmethod
145
+ def find_matching_ccis(control_base: str, part: Optional[str], cci_map: Dict[str, str]) -> List[str]:
146
+ """
147
+ Find all CCI IDs that match the control base and part.
148
+
149
+ Examples:
150
+ control_base="AC-1", part="a" matches:
151
+ - AC-1(a)(1)(a)
152
+ - AC-1(a)(1)(b)
153
+ - AC-1(a)(2)
154
+ - AC-1(a)
155
+
156
+ :param str control_base: Control identifier (e.g., "AC-1", "AC-2(3)")
157
+ :param Optional[str] part: Part letter (e.g., "a", "b") or None for enhancements
158
+ :param Dict[str, str] cci_map: Map of formatted index to comma-separated CCI IDs
159
+ :return: List of CCI ID strings (comma-separated)
160
+ :rtype: List[str]
161
+ """
162
+ matching_ccis = []
163
+
164
+ for index, cci_ids in cci_map.items():
165
+ # Check if index starts with control_base
166
+ if index.startswith(control_base):
167
+ # Extract the part after control_base
168
+ remainder = index[len(control_base) :]
169
+
170
+ # For enhancements without parts, only match exact control (no remainder)
171
+ # And Check if the remainder starts with (part)
172
+ if (part is None and remainder == "") or remainder.startswith(f"({part})") or remainder == f"({part})":
173
+ matching_ccis.append(cci_ids)
174
+
175
+ return matching_ccis
176
+
177
+ def find_matching_ccis_by_name(self, control_base: str, name: str, cci_map: Dict[str, str]) -> List[str]:
178
+ """
179
+ Find CCIs by matching control base and objective name field.
180
+ Fallback method when otherId matching fails.
181
+
182
+ Supports both NIST 800-53 Revision 4 and 5 label formats:
183
+
184
+ Revision 5 Examples:
185
+ control_base="AC-2(4)", name="AC-2(4)" -> matches AC-2(4)
186
+ control_base="AC-1", name="a" -> matches AC-1(a), AC-1(a)(1), etc.
187
+
188
+ Revision 4 Examples:
189
+ control_base="AC-1", name="a.1." -> matches AC-1(a), AC-1(a)(1), etc.
190
+ control_base="AC-1", name="b.2." -> matches AC-1(b), AC-1(b)(2), etc.
191
+
192
+ :param str control_base: Control identifier
193
+ :param str name: Objective name field
194
+ :param Dict[str, str] cci_map: Map of formatted index to comma-separated CCI IDs
195
+ :return: List of CCI ID strings (comma-separated)
196
+ :rtype: List[str]
197
+ """
198
+ import re
199
+
200
+ matching_ccis = []
201
+
202
+ # Remove trailing period and whitespace from name for matching
203
+ clean_name = name.strip().rstrip(".").strip()
204
+
205
+ # If name exactly matches control_base, match that control
206
+ if clean_name == control_base:
207
+ if control_base in cci_map:
208
+ matching_ccis.append(cci_map[control_base])
209
+ return matching_ccis
210
+
211
+ # Try to extract part letter from different formats
212
+ part = None
213
+
214
+ # Check if name is a single letter (Revision 5 format: "a", "b")
215
+ if len(clean_name) == 1 and clean_name.isalpha():
216
+ part = clean_name.lower()
217
+ elif match := re.match(r"^([a-z])\.\d+$", clean_name.lower()):
218
+ part = match.group(1)
219
+
220
+ # If we extracted a part letter, try matching
221
+ if part:
222
+ matching_ccis = self._extract_part_letter(cci_map, control_base, part, matching_ccis)
223
+
224
+ return matching_ccis
225
+
226
+ @staticmethod
227
+ def _extract_part_letter(
228
+ cci_map: Dict[str, str], control_base: str, part: str, matching_ccis: List[str]
229
+ ) -> List[str]:
230
+ """
231
+ Extract the part letter from the name.
232
+
233
+ :param Dict[str, str] cci_map: The map of CCI IDs to their indices.
234
+ :param str control_base: The control base to match against.
235
+ :param str part: The part letter to match against.
236
+ :param List[str] matching_ccis: The list of matching CCI IDs.
237
+ :rtype: List[str]
238
+ """
239
+ for index, cci_ids in cci_map.items():
240
+ if index.startswith(control_base):
241
+ remainder = index[len(control_base) :]
242
+ if remainder.startswith(f"({part})") or remainder == f"({part})":
243
+ matching_ccis.append(cci_ids)
244
+ return matching_ccis
245
+
246
+ @staticmethod
247
+ def ccis_already_present(current_other_id: str, new_cci_ids: str) -> bool:
248
+ """
249
+ Check if any of the new CCI IDs are already present in current otherId.
250
+ Prevents duplicate CCI mappings.
251
+
252
+ :param str current_other_id: Current otherId value
253
+ :param str new_cci_ids: Comma-separated string of CCI IDs to add
254
+ :return: True if any CCIs are already present
255
+ :rtype: bool
256
+ """
257
+ if not current_other_id:
258
+ return False
259
+
260
+ # Extract individual CCI IDs from both strings
261
+ existing_ccis: set[str] = {cci.strip() for cci in current_other_id.split(",") if cci.strip().startswith("CCI-")}
262
+ new_ccis: set[str] = {cci.strip() for cci in new_cci_ids.split(",") if cci.strip().startswith("CCI-")}
263
+
264
+ # Check if there's any overlap
265
+ return bool(existing_ccis & new_ccis)
266
+
267
+ @staticmethod
268
+ def _extract_cci_data(cci_item: ET.Element) -> Tuple[Optional[str], str]:
269
+ """
270
+ Extract CCI ID and definition from CCI item.
271
+
272
+ :param ET.Element cci_item: XML element containing CCI data
273
+ :return: Tuple of (cci_id, definition)
274
+ :rtype: Tuple[Optional[str], str]
275
+ """
276
+ cci_id = cci_item.get("id")
277
+ definition_elem = cci_item.find(".//{http://iase.disa.mil/cci}definition")
278
+ definition = definition_elem.text if definition_elem is not None and definition_elem.text else ""
279
+ return cci_id, definition
280
+
281
+ def _process_references(self, references: List[ET.Element], cci_id: str, definition: str) -> None:
282
+ """
283
+ Process reference elements and add to normalized CCI data.
284
+
285
+ :param List[ET.Element] references: List of reference XML elements
286
+ :param str cci_id: CCI identifier
287
+ :param str definition: CCI definition text
288
+ :rtype: None
289
+ """
290
+ for ref in references:
291
+ if not self._is_valid_reference(ref):
292
+ continue
293
+
294
+ ref_index = ref.get("index")
295
+ if ref_index:
296
+ main_control = self._parse_control_id(ref_index)
297
+ self._add_cci_to_control(main_control, cci_id, definition)
298
+
299
+ def _is_valid_reference(self, ref: ET.Element) -> bool:
300
+ """
301
+ Check if reference matches the target version.
302
+
303
+ :param ET.Element ref: Reference XML element
304
+ :return: True if reference version matches target version
305
+ :rtype: bool
306
+ """
307
+ ref_version = ref.get("version")
308
+ return ref_version is not None and ref_version == self.reference_version
309
+
310
+ def _add_cci_to_control(self, main_control: str, cci_id: str, definition: str) -> None:
311
+ """
312
+ Add CCI data to the normalized structure for a control.
313
+
314
+ :param str main_control: Control identifier
315
+ :param str cci_id: CCI identifier
316
+ :param str definition: CCI definition
317
+ :rtype: None
318
+ """
319
+ if main_control not in self.normalized_cci:
320
+ self.normalized_cci[main_control] = []
321
+ self.normalized_cci[main_control].append({"cci_id": cci_id, "definition": definition})
322
+
323
+ def parse_cci(self) -> None:
324
+ """
325
+ Parse CCI items from XML and create both mapping structures.
326
+
327
+ Creates:
328
+ - normalized_cci: Dict[control_id, List[Dict]] - for SecurityControl mapping
329
+ - cci_grouped_by_index: Dict[formatted_index, str] - for ControlObjective mapping
330
+
331
+ :rtype: None
332
+ """
333
+ if self.verbose:
334
+ logger.info("Parsing CCI items from XML...")
335
+
336
+ # Track all CCI items with formatted indices for objective mapping
337
+ from collections import defaultdict
338
+
339
+ temp_grouped: Dict[str, List[str]] = defaultdict(list)
340
+
341
+ for cci_item in self.xml_data.findall(".//{http://iase.disa.mil/cci}cci_item"):
342
+ cci_id, definition = self._extract_cci_data(cci_item)
343
+ if not cci_id:
344
+ continue
345
+
346
+ references = cci_item.findall(".//{http://iase.disa.mil/cci}reference")
347
+
348
+ for ref in references:
349
+ if not self._is_valid_reference(ref):
350
+ continue
351
+
352
+ ref_index = ref.get("index")
353
+ if ref_index:
354
+ # Existing: simple control ID for SecurityControl mapping
355
+ main_control = self._parse_control_id(ref_index)
356
+ self._add_cci_to_control(main_control, cci_id, definition)
357
+
358
+ # NEW: formatted index for ControlObjective mapping
359
+ formatted_index = self.format_index(ref_index)
360
+ temp_grouped[formatted_index].append(cci_id)
361
+
362
+ # Convert to comma-separated format
363
+ self.cci_grouped_by_index = {index: ", ".join(cci_list) for index, cci_list in temp_grouped.items()}
364
+
365
+ if self.verbose:
366
+ logger.info(f"Created {len(self.normalized_cci)} control mappings")
367
+ logger.info(f"Created {len(self.cci_grouped_by_index)} formatted index mappings")
368
+
369
+ @staticmethod
370
+ def _get_catalog(catalog_id: int) -> Catalog:
371
+ """
372
+ Get the catalog with specified ID.
373
+
374
+ :param int catalog_id: ID of the catalog to retrieve
375
+ :return: Catalog instance
376
+ :rtype: Catalog
377
+ :raises SystemExit: If catalog not found
378
+ """
379
+ try:
380
+ catalog = Catalog.get(id=catalog_id)
381
+ if catalog is None:
382
+ error_and_exit(f"Catalog with id {catalog_id} not found. Please ensure the catalog exists.")
383
+ return catalog
384
+ except Exception:
385
+ error_and_exit(f"Catalog with id {catalog_id} not found. Please ensure the catalog exists.")
386
+
387
+ def _get_user_context(self) -> Tuple[Optional[str], int]:
388
+ """
389
+ Get user ID and tenant ID from application config.
390
+
391
+ :return: Tuple of (user_id, tenant_id)
392
+ :rtype: Tuple[Optional[str], int]
393
+ """
394
+ if self._user_context is None:
395
+ app = Application()
396
+ user_id = app.config.get("userId")
397
+ tenant_id = app.config.get("tenantId", 1)
398
+
399
+ try:
400
+ user_id = str(user_id) if user_id else None
401
+ except (TypeError, ValueError):
402
+ user_id = None
403
+ if self.verbose:
404
+ logger.warning("userId in config is not set or invalid; created_by will be None.")
405
+
406
+ # Convert tenant_id to int if it's a string
407
+ try:
408
+ tenant_id = int(tenant_id)
409
+ except (TypeError, ValueError):
410
+ tenant_id = 1
411
+ if self.verbose:
412
+ logger.warning("tenantId in config is not valid; using default value 1.")
413
+
414
+ self._user_context = (user_id, tenant_id)
415
+
416
+ return self._user_context
417
+
418
+ @staticmethod
419
+ def _find_existing_cci(control_id: int, cci_id: str) -> Optional[CCI]:
420
+ """
421
+ Find existing CCI by ID within a control.
422
+
423
+ :param int control_id: Security control ID
424
+ :param str cci_id: CCI identifier to search for
425
+ :return: Existing CCI instance or None
426
+ :rtype: Optional[CCI]
427
+ """
428
+ try:
429
+ existing_ccis: List[CCI] = CCI.get_all_by_parent(parent_id=control_id)
430
+ for existing in existing_ccis:
431
+ if existing.uuid == cci_id:
432
+ return existing
433
+ except Exception:
434
+ pass
435
+ return None
436
+
437
+ @staticmethod
438
+ def _create_cci_data(
439
+ cci_id: str, definition: str, user_id: Optional[str], tenant_id: int, current_time: str
440
+ ) -> Dict:
441
+ """
442
+ Create common CCI data structure.
443
+
444
+ :param str cci_id: CCI identifier
445
+ :param str definition: CCI definition
446
+ :param Optional[str] user_id: User ID
447
+ :param int tenant_id: Tenant ID
448
+ :param str current_time: Current timestamp string
449
+ :return: Dictionary with common CCI attributes
450
+ :rtype: Dict
451
+ """
452
+ return {
453
+ "name": cci_id,
454
+ "description": definition,
455
+ "controlType": "policy",
456
+ "publishDate": current_time,
457
+ "dateLastUpdated": current_time,
458
+ "lastUpdatedById": user_id,
459
+ "isPublic": True,
460
+ "tenantsId": tenant_id,
461
+ }
462
+
463
+ @staticmethod
464
+ def _update_existing_cci(existing_cci: CCI, cci_data: Dict) -> None:
465
+ """
466
+ Update an existing CCI with new data.
467
+
468
+ :param CCI existing_cci: CCI instance to update
469
+ :param Dict cci_data: Dictionary with CCI attributes
470
+ :rtype: None
471
+ """
472
+ for key, value in cci_data.items():
473
+ setattr(existing_cci, key, value)
474
+ existing_cci.create_or_update()
475
+
476
+ @staticmethod
477
+ def _create_new_cci(cci_id: str, cci_data: Dict, control_id: int, user_id: Optional[str], current_time: str) -> CCI:
478
+ """
479
+ Create a new CCI instance.
480
+
481
+ :param str cci_id: CCI identifier
482
+ :param Dict cci_data: Dictionary with common CCI attributes
483
+ :param int control_id: Security control ID
484
+ :param Optional[str] user_id: User ID
485
+ :param str current_time: Current timestamp string
486
+ :return: Created CCI instance
487
+ :rtype: CCI
488
+ """
489
+ new_cci = CCI(
490
+ uuid=cci_id,
491
+ securityControlId=control_id,
492
+ createdById=user_id,
493
+ dateCreated=current_time,
494
+ **cci_data,
495
+ )
496
+ new_cci.create()
497
+ return new_cci
498
+
499
+ def _process_cci_for_control(
500
+ self, control_id: int, cci_list: List[Dict], user_id: Optional[str], tenant_id: int
501
+ ) -> Tuple[int, int]:
502
+ """
503
+ Process all CCI items for a specific control.
504
+
505
+ :param int control_id: Security control ID
506
+ :param List[Dict] cci_list: List of CCI data dictionaries
507
+ :param Optional[str] user_id: User ID
508
+ :param int tenant_id: Tenant ID
509
+ :return: Tuple of (created_count, updated_count)
510
+ :rtype: Tuple[int, int]
511
+ """
512
+ created_count = 0
513
+ updated_count = 0
514
+ current_time = datetime.datetime.now().strftime(REGSCALE_DATE_FORMAT)
515
+
516
+ for cci in cci_list:
517
+ cci_id = cci["cci_id"]
518
+ definition = cci["definition"]
519
+
520
+ existing_cci = self._find_existing_cci(control_id, cci_id)
521
+ cci_data = self._create_cci_data(cci_id, definition, user_id, tenant_id, current_time)
522
+
523
+ if existing_cci:
524
+ self._update_existing_cci(existing_cci, cci_data)
525
+ updated_count += 1
526
+ else:
527
+ self._create_new_cci(cci_id, cci_data, control_id, user_id, current_time)
528
+ created_count += 1
529
+
530
+ return created_count, updated_count
531
+
532
+ def map_to_security_controls(self, catalog_id: int = 1) -> Dict[str, int]:
533
+ """
534
+ Map normalized CCI data to security controls in the database.
535
+
536
+ :param int catalog_id: ID of the catalog containing security controls (default: 1)
537
+ :return: Dictionary with operation statistics
538
+ :rtype: Dict[str, int]
539
+ """
540
+ if self.verbose:
541
+ logger.info("Mapping CCI data to security controls...")
542
+
543
+ catalog = self._get_catalog(catalog_id)
544
+ security_controls: List[SecurityControl] = SecurityControl.get_all_by_parent(parent_id=catalog.id)
545
+ control_map = {sc.controlId: sc.id for sc in security_controls}
546
+
547
+ user_id, tenant_id = self._get_user_context()
548
+
549
+ created_count = 0
550
+ updated_count = 0
551
+ skipped_count = 0
552
+
553
+ with create_progress_object() as progress:
554
+ logger.info(f"Parsing and mapping {len(self.normalized_cci)} normalized CCI entries...")
555
+ main_task = progress.add_task("Parsing and mapping CCIs...", total=len(self.normalized_cci))
556
+ for main_control, cci_list in self.normalized_cci.items():
557
+ if main_control in control_map:
558
+ control_id = control_map[main_control]
559
+ control_created, control_updated = self._process_cci_for_control(
560
+ control_id, cci_list, user_id, tenant_id
561
+ )
562
+ created_count += control_created
563
+ updated_count += control_updated
564
+ else:
565
+ skipped_count += len(cci_list)
566
+ if self.verbose:
567
+ logger.warning(f"Warning: Control not found for key: {main_control}")
568
+ progress.update(main_task, advance=1)
569
+
570
+ return {
571
+ "created": created_count,
572
+ "updated": updated_count,
573
+ "skipped": skipped_count,
574
+ "total_processed": len(self.normalized_cci),
575
+ }
576
+
577
+ def map_to_control_objectives(self, catalog_id: int = 1) -> Dict[str, int]:
578
+ """
579
+ Map grouped CCI data to control objectives in the database.
580
+ Updates the otherId field of existing ControlObjective records.
581
+
582
+ :param int catalog_id: ID of the catalog containing control objectives (default: 1)
583
+ :return: Dictionary with operation statistics
584
+ :rtype: Dict[str, int]
585
+ """
586
+ if self.verbose:
587
+ logger.info("Mapping CCI data to control objectives...")
588
+
589
+ # Fetch all objectives for the catalog
590
+ objectives: List[ControlObjective] = ControlObjective.get_by_catalog(catalog_id=catalog_id)
591
+
592
+ if self.verbose:
593
+ logger.info(f"Found {len(objectives)} objectives in catalog {catalog_id}")
594
+
595
+ objectives_updated = 0
596
+ objectives_skipped = 0
597
+ objectives_not_found = 0
598
+
599
+ with create_progress_object() as progress:
600
+ logger.info(f"Processing {len(objectives)} objectives...")
601
+ task = progress.add_task("Mapping CCIs to objectives...", total=len(objectives))
602
+
603
+ for obj in objectives:
604
+ objective_id = obj.otherId
605
+
606
+ # Skip objectives without proper otherId
607
+ if not objective_id or "_smt" not in objective_id:
608
+ objectives_not_found += 1
609
+ progress.update(task, advance=1)
610
+ continue
611
+
612
+ # Extract just the objective ID part (before any CCIs)
613
+ # Format: "ac-1_smt.a" or "ac-1_smt.a, CCI-000001, CCI-000002"
614
+ objective_id_parts = objective_id.split(",")
615
+ base_objective_id = objective_id_parts[0].strip()
616
+
617
+ # Parse the objective ID
618
+ control_base, part = self.parse_objective_id(base_objective_id)
619
+
620
+ if not control_base:
621
+ objectives_not_found += 1
622
+ progress.update(task, advance=1)
623
+ continue
624
+
625
+ # Find matching CCIs by otherId
626
+ matching_ccis = self.find_matching_ccis(control_base, part, self.cci_grouped_by_index)
627
+
628
+ # Fallback: try matching by name
629
+ if not matching_ccis and obj.name:
630
+ matching_ccis = self.find_matching_ccis_by_name(control_base, obj.name, self.cci_grouped_by_index)
631
+
632
+ if matching_ccis:
633
+ skipped_count, updated_count = self._handle_matching_ccis(
634
+ control_objective=obj,
635
+ matching_ccis=matching_ccis,
636
+ base_objective_id=base_objective_id,
637
+ )
638
+ objectives_skipped += skipped_count
639
+ objectives_updated += updated_count
640
+ else:
641
+ objectives_not_found += 1
642
+
643
+ progress.update(task, advance=1)
644
+
645
+ return {
646
+ "updated": objectives_updated,
647
+ "skipped": objectives_skipped,
648
+ "not_found": objectives_not_found,
649
+ "total_processed": len(objectives),
650
+ }
651
+
652
+ def _handle_matching_ccis(
653
+ self,
654
+ control_objective: ControlObjective,
655
+ matching_ccis: List[str],
656
+ base_objective_id: str,
657
+ ) -> Tuple[int, int]:
658
+ """
659
+ Handle matching CCIs.
660
+
661
+ :param ControlObjective control_objective: ControlObjective instance
662
+ :param List[str] matching_ccis: List of matching CCI IDs
663
+ :param str base_objective_id: Base objective ID
664
+ :return: Tuple of (number of objectives skipped, number of objectives updated)
665
+ :rtype: Tuple[int, int]
666
+ """
667
+ # Combine all CCI IDs
668
+ all_cci_ids = ", ".join(matching_ccis)
669
+
670
+ # Check for duplicates
671
+ if self.ccis_already_present(control_objective.otherId, all_cci_ids):
672
+ if self.verbose:
673
+ logger.info(f"Skipping {base_objective_id} - CCIs already present")
674
+ return 1, 0
675
+
676
+ # Update the otherId field
677
+ control_objective.otherId = f"{control_objective.otherId}, {all_cci_ids}"
678
+ control_objective.save()
679
+
680
+ if self.verbose:
681
+ logger.info(f"Updated {base_objective_id} with {all_cci_ids}")
682
+ return 0, 1
683
+
684
+ def get_normalized_cci(self) -> Dict[str, List[Dict]]:
685
+ """
686
+ Get the normalized CCI data.
687
+
688
+ :return: Dictionary of normalized CCI data
689
+ :rtype: Dict[str, List[Dict]]
690
+ """
691
+ return self.normalized_cci
692
+
693
+
694
+ def _load_xml_file(xml_file: str) -> ET.Element:
695
+ """
696
+ Load and parse XML file.
697
+
698
+ :param str xml_file: Path to XML file
699
+ :return: Root element of parsed XML
700
+ :rtype: ET.Element
701
+ :raises click.ClickException: If XML parsing fails
702
+ """
703
+ try:
704
+ logger.info(f"Loading XML file: {xml_file}")
705
+ tree = ET.parse(xml_file)
706
+ return tree.getroot()
707
+ except ET.ParseError as e:
708
+ error_and_exit(f"Failed to parse XML file: {e}")
709
+
710
+
711
+ def _display_verbose_output(normalized_data: Dict[str, List[Dict]]) -> None:
712
+ """
713
+ Display detailed normalized CCI data.
714
+
715
+ :param Dict[str, List[Dict]] normalized_data: Dictionary of normalized CCI data
716
+ :rtype: None
717
+ """
718
+ logger.info("\nNormalized CCI Data:")
719
+ for key, value in normalized_data.items():
720
+ logger.info(f" {key}: {len(value)} CCI items")
721
+ for cci in value:
722
+ definition_preview = cci["definition"][:100] + "..." if len(cci["definition"]) > 100 else cci["definition"]
723
+ logger.info(f" - {cci['cci_id']}: {definition_preview}")
724
+
725
+
726
+ def _display_results(stats: Dict[str, int]) -> None:
727
+ """
728
+ Display database operation results.
729
+
730
+ :param Dict[str, int] stats: Dictionary with operation statistics
731
+ :rtype: None
732
+ """
733
+ logger.info(
734
+ f"[green]\nDatabase operations completed:"
735
+ f"[green]\n - Created: {stats['created']}"
736
+ f"[green]\n - Updated: {stats['updated']}"
737
+ f"[green]\n - Skipped: {stats['skipped']}"
738
+ f"[green]\n - Total processed: {stats['total_processed']}",
739
+ )
740
+
741
+
742
+ def _display_objective_results(stats: Dict[str, int]) -> None:
743
+ """
744
+ Display control objective mapping results.
745
+
746
+ :param Dict[str, int] stats: Dictionary with operation statistics
747
+ :rtype: None
748
+ """
749
+ logger.info(
750
+ f"[green]\nControl objective operations completed:"
751
+ f"[green]\n - Updated: {stats['updated']}"
752
+ f"[green]\n - Skipped: {stats['skipped']}"
753
+ f"[green]\n - Not found: {stats['not_found']}"
754
+ f"[green]\n - Total processed: {stats['total_processed']}",
755
+ )
756
+
757
+
758
+ def _process_cci_import(
759
+ importer: CCIImporter, dry_run: bool, verbose: bool, catalog_id: int, disable_objectives: bool = False
760
+ ) -> None:
761
+ """
762
+ Process CCI import with optional database operations.
763
+
764
+ :param CCIImporter importer: CCIImporter instance
765
+ :param bool dry_run: Whether to skip database operations
766
+ :param bool verbose: Whether to display verbose output
767
+ :param int catalog_id: ID of the catalog containing security controls
768
+ :param bool disable_objectives: Whether to disable mapping to control objectives
769
+ :rtype: None
770
+ """
771
+ importer.parse_cci()
772
+ normalized_data = importer.get_normalized_cci()
773
+
774
+ logger.info(f"[green]Successfully parsed {len(normalized_data)} normalized CCI entries[/green]")
775
+
776
+ if verbose:
777
+ _display_verbose_output(normalized_data)
778
+
779
+ if not dry_run:
780
+ # Map to SecurityControl (existing functionality)
781
+ stats = importer.map_to_security_controls(catalog_id)
782
+ _display_results(stats)
783
+
784
+ # Map to ControlObjective (new functionality)
785
+ if not disable_objectives:
786
+ logger.info("\n[cyan]Mapping CCIs to control objectives...[/cyan]")
787
+ obj_stats = importer.map_to_control_objectives(catalog_id)
788
+ _display_objective_results(obj_stats)
789
+ else:
790
+ logger.info("\n[yellow]DRY RUN MODE: No database changes were made[/yellow]")
791
+
792
+
793
+ @click.command(name="cci_importer")
794
+ @click.option(
795
+ "--xml_file", "-f", type=click.Path(exists=True), default=None, required=False, help="Path to the CCI XML file."
796
+ )
797
+ @click.option("--dry-run", "-d", is_flag=True, help="Parse and display normalized data without saving to database")
798
+ @click.option("--verbose", "-v", is_flag=True, help="Display detailed output including all normalized CCI data")
799
+ @click.option(
800
+ "--nist-version", "-n", type=click.Choice(["4", "5"]), default="5", help="NIST 800-53 Revision version (default: 5)"
801
+ )
802
+ @click.option(
803
+ "--catalog-id", "-c", type=click.INT, default=1, help="ID of the catalog containing security controls (default: 1)"
804
+ )
805
+ @click.option(
806
+ "--disable-objectives",
807
+ "-o",
808
+ is_flag=True,
809
+ help="Disable mapping CCIs to control objectives (updates otherId field)",
810
+ )
811
+ def cci_importer(
812
+ xml_file: str, dry_run: bool, verbose: bool, nist_version: str, catalog_id: int, disable_objectives: bool
813
+ ) -> None:
814
+ """Import CCI data from XML files and map to security controls and/or objectives.
815
+
816
+ By default, maps CCIs to SecurityControl entities. Use --disable-objectives flag
817
+ to also update ControlObjective.otherId fields with CCI mappings.
818
+
819
+ If no XML file is specified, defaults to packaged CCI_List.xml.
820
+ """
821
+
822
+ try:
823
+ if not xml_file:
824
+ import importlib.resources as pkg_resources
825
+ from regscale.models import integration_models
826
+
827
+ files = pkg_resources.files(integration_models)
828
+ cci_path = files / "CCI_List.xml"
829
+ xml_file = str(cci_path)
830
+ root = _load_xml_file(xml_file)
831
+ importer = CCIImporter(root, version=nist_version, verbose=verbose)
832
+ _process_cci_import(importer, dry_run, verbose, catalog_id, disable_objectives)
833
+ except Exception as e:
834
+ error_and_exit(f"Unexpected error: {e}")