regscale-cli 6.21.2.0__py3-none-any.whl → 6.28.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. regscale/_version.py +1 -1
  2. regscale/airflow/hierarchy.py +2 -2
  3. regscale/core/app/api.py +5 -2
  4. regscale/core/app/application.py +36 -6
  5. regscale/core/app/internal/control_editor.py +73 -21
  6. regscale/core/app/internal/evidence.py +727 -204
  7. regscale/core/app/internal/login.py +4 -2
  8. regscale/core/app/internal/model_editor.py +219 -64
  9. regscale/core/app/utils/app_utils.py +86 -12
  10. regscale/core/app/utils/catalog_utils/common.py +1 -1
  11. regscale/core/login.py +21 -4
  12. regscale/core/utils/async_graphql_client.py +363 -0
  13. regscale/core/utils/date.py +77 -1
  14. regscale/dev/cli.py +26 -0
  15. regscale/dev/code_gen.py +109 -24
  16. regscale/dev/version.py +72 -0
  17. regscale/integrations/commercial/__init__.py +30 -2
  18. regscale/integrations/commercial/aws/audit_manager_compliance.py +3908 -0
  19. regscale/integrations/commercial/aws/cli.py +3107 -54
  20. regscale/integrations/commercial/aws/cloudtrail_control_mappings.py +333 -0
  21. regscale/integrations/commercial/aws/cloudtrail_evidence.py +501 -0
  22. regscale/integrations/commercial/aws/cloudwatch_control_mappings.py +357 -0
  23. regscale/integrations/commercial/aws/cloudwatch_evidence.py +490 -0
  24. regscale/integrations/commercial/{amazon → aws}/common.py +71 -19
  25. regscale/integrations/commercial/aws/config_compliance.py +914 -0
  26. regscale/integrations/commercial/aws/conformance_pack_mappings.py +198 -0
  27. regscale/integrations/commercial/aws/control_compliance_analyzer.py +439 -0
  28. regscale/integrations/commercial/aws/evidence_generator.py +283 -0
  29. regscale/integrations/commercial/aws/guardduty_control_mappings.py +340 -0
  30. regscale/integrations/commercial/aws/guardduty_evidence.py +1053 -0
  31. regscale/integrations/commercial/aws/iam_control_mappings.py +368 -0
  32. regscale/integrations/commercial/aws/iam_evidence.py +574 -0
  33. regscale/integrations/commercial/aws/inventory/__init__.py +338 -22
  34. regscale/integrations/commercial/aws/inventory/base.py +107 -5
  35. regscale/integrations/commercial/aws/inventory/resources/analytics.py +390 -0
  36. regscale/integrations/commercial/aws/inventory/resources/applications.py +234 -0
  37. regscale/integrations/commercial/aws/inventory/resources/audit_manager.py +513 -0
  38. regscale/integrations/commercial/aws/inventory/resources/cloudtrail.py +315 -0
  39. regscale/integrations/commercial/aws/inventory/resources/cloudtrail_logs_metadata.py +476 -0
  40. regscale/integrations/commercial/aws/inventory/resources/cloudwatch.py +191 -0
  41. regscale/integrations/commercial/aws/inventory/resources/compute.py +328 -9
  42. regscale/integrations/commercial/aws/inventory/resources/config.py +464 -0
  43. regscale/integrations/commercial/aws/inventory/resources/containers.py +74 -9
  44. regscale/integrations/commercial/aws/inventory/resources/database.py +481 -31
  45. regscale/integrations/commercial/aws/inventory/resources/developer_tools.py +253 -0
  46. regscale/integrations/commercial/aws/inventory/resources/guardduty.py +286 -0
  47. regscale/integrations/commercial/aws/inventory/resources/iam.py +470 -0
  48. regscale/integrations/commercial/aws/inventory/resources/inspector.py +476 -0
  49. regscale/integrations/commercial/aws/inventory/resources/integration.py +175 -61
  50. regscale/integrations/commercial/aws/inventory/resources/kms.py +447 -0
  51. regscale/integrations/commercial/aws/inventory/resources/machine_learning.py +358 -0
  52. regscale/integrations/commercial/aws/inventory/resources/networking.py +390 -67
  53. regscale/integrations/commercial/aws/inventory/resources/s3.py +394 -0
  54. regscale/integrations/commercial/aws/inventory/resources/security.py +268 -72
  55. regscale/integrations/commercial/aws/inventory/resources/securityhub.py +473 -0
  56. regscale/integrations/commercial/aws/inventory/resources/storage.py +288 -29
  57. regscale/integrations/commercial/aws/inventory/resources/systems_manager.py +657 -0
  58. regscale/integrations/commercial/aws/inventory/resources/vpc.py +655 -0
  59. regscale/integrations/commercial/aws/kms_control_mappings.py +288 -0
  60. regscale/integrations/commercial/aws/kms_evidence.py +879 -0
  61. regscale/integrations/commercial/aws/ocsf/__init__.py +7 -0
  62. regscale/integrations/commercial/aws/ocsf/constants.py +115 -0
  63. regscale/integrations/commercial/aws/ocsf/mapper.py +435 -0
  64. regscale/integrations/commercial/aws/org_control_mappings.py +286 -0
  65. regscale/integrations/commercial/aws/org_evidence.py +666 -0
  66. regscale/integrations/commercial/aws/s3_control_mappings.py +356 -0
  67. regscale/integrations/commercial/aws/s3_evidence.py +632 -0
  68. regscale/integrations/commercial/aws/scanner.py +1072 -205
  69. regscale/integrations/commercial/aws/security_hub.py +319 -0
  70. regscale/integrations/commercial/aws/session_manager.py +282 -0
  71. regscale/integrations/commercial/aws/ssm_control_mappings.py +291 -0
  72. regscale/integrations/commercial/aws/ssm_evidence.py +492 -0
  73. regscale/integrations/commercial/jira.py +489 -153
  74. regscale/integrations/commercial/microsoft_defender/defender.py +326 -5
  75. regscale/integrations/commercial/microsoft_defender/defender_api.py +348 -14
  76. regscale/integrations/commercial/microsoft_defender/defender_constants.py +157 -0
  77. regscale/integrations/commercial/qualys/__init__.py +167 -68
  78. regscale/integrations/commercial/qualys/scanner.py +305 -39
  79. regscale/integrations/commercial/sarif/sairf_importer.py +432 -0
  80. regscale/integrations/commercial/sarif/sarif_converter.py +67 -0
  81. regscale/integrations/commercial/sicura/api.py +79 -42
  82. regscale/integrations/commercial/sicura/commands.py +8 -2
  83. regscale/integrations/commercial/sicura/scanner.py +83 -44
  84. regscale/integrations/commercial/stigv2/ckl_parser.py +5 -5
  85. regscale/integrations/commercial/synqly/assets.py +133 -16
  86. regscale/integrations/commercial/synqly/edr.py +2 -8
  87. regscale/integrations/commercial/synqly/query_builder.py +536 -0
  88. regscale/integrations/commercial/synqly/ticketing.py +27 -0
  89. regscale/integrations/commercial/synqly/vulnerabilities.py +165 -28
  90. regscale/integrations/commercial/tenablev2/cis_parsers.py +453 -0
  91. regscale/integrations/commercial/tenablev2/cis_scanner.py +447 -0
  92. regscale/integrations/commercial/tenablev2/commands.py +146 -5
  93. regscale/integrations/commercial/tenablev2/scanner.py +1 -3
  94. regscale/integrations/commercial/tenablev2/stig_parsers.py +113 -57
  95. regscale/integrations/commercial/wizv2/WizDataMixin.py +1 -1
  96. regscale/integrations/commercial/wizv2/click.py +191 -76
  97. regscale/integrations/commercial/wizv2/compliance/__init__.py +15 -0
  98. regscale/integrations/commercial/wizv2/{policy_compliance_helpers.py → compliance/helpers.py} +78 -60
  99. regscale/integrations/commercial/wizv2/compliance_report.py +1592 -0
  100. regscale/integrations/commercial/wizv2/core/__init__.py +133 -0
  101. regscale/integrations/commercial/wizv2/{async_client.py → core/client.py} +7 -3
  102. regscale/integrations/commercial/wizv2/{constants.py → core/constants.py} +92 -89
  103. regscale/integrations/commercial/wizv2/core/file_operations.py +237 -0
  104. regscale/integrations/commercial/wizv2/fetchers/__init__.py +11 -0
  105. regscale/integrations/commercial/wizv2/{data_fetcher.py → fetchers/policy_assessment.py} +66 -9
  106. regscale/integrations/commercial/wizv2/file_cleanup.py +104 -0
  107. regscale/integrations/commercial/wizv2/issue.py +776 -28
  108. regscale/integrations/commercial/wizv2/models/__init__.py +0 -0
  109. regscale/integrations/commercial/wizv2/parsers/__init__.py +34 -0
  110. regscale/integrations/commercial/wizv2/{parsers.py → parsers/main.py} +1 -1
  111. regscale/integrations/commercial/wizv2/processors/__init__.py +11 -0
  112. regscale/integrations/commercial/wizv2/{finding_processor.py → processors/finding.py} +1 -1
  113. regscale/integrations/commercial/wizv2/reports.py +243 -0
  114. regscale/integrations/commercial/wizv2/sbom.py +1 -1
  115. regscale/integrations/commercial/wizv2/scanner.py +1031 -441
  116. regscale/integrations/commercial/wizv2/utils/__init__.py +48 -0
  117. regscale/integrations/commercial/wizv2/{utils.py → utils/main.py} +116 -61
  118. regscale/integrations/commercial/wizv2/variables.py +89 -3
  119. regscale/integrations/compliance_integration.py +1036 -151
  120. regscale/integrations/control_matcher.py +432 -0
  121. regscale/integrations/due_date_handler.py +333 -0
  122. regscale/integrations/milestone_manager.py +291 -0
  123. regscale/integrations/public/__init__.py +14 -0
  124. regscale/integrations/public/cci_importer.py +834 -0
  125. regscale/integrations/public/csam/__init__.py +0 -0
  126. regscale/integrations/public/csam/csam.py +938 -0
  127. regscale/integrations/public/csam/csam_agency_defined.py +179 -0
  128. regscale/integrations/public/csam/csam_common.py +154 -0
  129. regscale/integrations/public/csam/csam_controls.py +432 -0
  130. regscale/integrations/public/csam/csam_poam.py +124 -0
  131. regscale/integrations/public/fedramp/click.py +77 -6
  132. regscale/integrations/public/fedramp/docx_parser.py +10 -1
  133. regscale/integrations/public/fedramp/fedramp_cis_crm.py +675 -289
  134. regscale/integrations/public/fedramp/fedramp_five.py +1 -1
  135. regscale/integrations/public/fedramp/poam/scanner.py +75 -7
  136. regscale/integrations/public/fedramp/poam_export_v5.py +888 -0
  137. regscale/integrations/scanner_integration.py +1961 -430
  138. regscale/models/integration_models/CCI_List.xml +1 -0
  139. regscale/models/integration_models/aqua.py +2 -2
  140. regscale/models/integration_models/cisa_kev_data.json +805 -11
  141. regscale/models/integration_models/flat_file_importer/__init__.py +5 -8
  142. regscale/models/integration_models/nexpose.py +36 -10
  143. regscale/models/integration_models/qualys.py +3 -4
  144. regscale/models/integration_models/synqly_models/capabilities.json +1 -1
  145. regscale/models/integration_models/synqly_models/connectors/vulnerabilities.py +87 -18
  146. regscale/models/integration_models/synqly_models/filter_parser.py +332 -0
  147. regscale/models/integration_models/synqly_models/ocsf_mapper.py +124 -25
  148. regscale/models/integration_models/synqly_models/synqly_model.py +89 -16
  149. regscale/models/locking.py +12 -8
  150. regscale/models/platform.py +4 -2
  151. regscale/models/regscale_models/__init__.py +7 -0
  152. regscale/models/regscale_models/assessment.py +2 -1
  153. regscale/models/regscale_models/catalog.py +1 -1
  154. regscale/models/regscale_models/compliance_settings.py +251 -1
  155. regscale/models/regscale_models/component.py +1 -0
  156. regscale/models/regscale_models/control_implementation.py +236 -41
  157. regscale/models/regscale_models/control_objective.py +74 -5
  158. regscale/models/regscale_models/file.py +2 -0
  159. regscale/models/regscale_models/form_field_value.py +5 -3
  160. regscale/models/regscale_models/inheritance.py +44 -0
  161. regscale/models/regscale_models/issue.py +301 -102
  162. regscale/models/regscale_models/milestone.py +33 -14
  163. regscale/models/regscale_models/organization.py +3 -0
  164. regscale/models/regscale_models/regscale_model.py +310 -73
  165. regscale/models/regscale_models/security_plan.py +4 -2
  166. regscale/models/regscale_models/vulnerability.py +3 -3
  167. regscale/regscale.py +25 -4
  168. regscale/templates/__init__.py +0 -0
  169. regscale/utils/threading/threadhandler.py +20 -15
  170. regscale/validation/record.py +23 -1
  171. {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/METADATA +17 -33
  172. {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/RECORD +310 -111
  173. tests/core/__init__.py +0 -0
  174. tests/core/utils/__init__.py +0 -0
  175. tests/core/utils/test_async_graphql_client.py +472 -0
  176. tests/fixtures/test_fixture.py +13 -8
  177. tests/regscale/core/test_login.py +171 -4
  178. tests/regscale/integrations/commercial/__init__.py +0 -0
  179. tests/regscale/integrations/commercial/aws/__init__.py +0 -0
  180. tests/regscale/integrations/commercial/aws/test_audit_manager_compliance.py +1304 -0
  181. tests/regscale/integrations/commercial/aws/test_audit_manager_evidence_aggregation.py +341 -0
  182. tests/regscale/integrations/commercial/aws/test_aws_analytics_collector.py +260 -0
  183. tests/regscale/integrations/commercial/aws/test_aws_applications_collector.py +242 -0
  184. tests/regscale/integrations/commercial/aws/test_aws_audit_manager_collector.py +1155 -0
  185. tests/regscale/integrations/commercial/aws/test_aws_cloudtrail_collector.py +534 -0
  186. tests/regscale/integrations/commercial/aws/test_aws_config_collector.py +400 -0
  187. tests/regscale/integrations/commercial/aws/test_aws_developer_tools_collector.py +203 -0
  188. tests/regscale/integrations/commercial/aws/test_aws_guardduty_collector.py +315 -0
  189. tests/regscale/integrations/commercial/aws/test_aws_iam_collector.py +458 -0
  190. tests/regscale/integrations/commercial/aws/test_aws_inspector_collector.py +353 -0
  191. tests/regscale/integrations/commercial/aws/test_aws_inventory_integration.py +530 -0
  192. tests/regscale/integrations/commercial/aws/test_aws_kms_collector.py +919 -0
  193. tests/regscale/integrations/commercial/aws/test_aws_machine_learning_collector.py +237 -0
  194. tests/regscale/integrations/commercial/aws/test_aws_s3_collector.py +722 -0
  195. tests/regscale/integrations/commercial/aws/test_aws_scanner_integration.py +722 -0
  196. tests/regscale/integrations/commercial/aws/test_aws_securityhub_collector.py +792 -0
  197. tests/regscale/integrations/commercial/aws/test_aws_systems_manager_collector.py +918 -0
  198. tests/regscale/integrations/commercial/aws/test_aws_vpc_collector.py +996 -0
  199. tests/regscale/integrations/commercial/aws/test_cli_evidence.py +431 -0
  200. tests/regscale/integrations/commercial/aws/test_cloudtrail_control_mappings.py +452 -0
  201. tests/regscale/integrations/commercial/aws/test_cloudtrail_evidence.py +788 -0
  202. tests/regscale/integrations/commercial/aws/test_config_compliance.py +298 -0
  203. tests/regscale/integrations/commercial/aws/test_conformance_pack_mappings.py +200 -0
  204. tests/regscale/integrations/commercial/aws/test_control_compliance_analyzer.py +375 -0
  205. tests/regscale/integrations/commercial/aws/test_datetime_parsing.py +223 -0
  206. tests/regscale/integrations/commercial/aws/test_evidence_generator.py +386 -0
  207. tests/regscale/integrations/commercial/aws/test_guardduty_control_mappings.py +564 -0
  208. tests/regscale/integrations/commercial/aws/test_guardduty_evidence.py +1041 -0
  209. tests/regscale/integrations/commercial/aws/test_iam_control_mappings.py +718 -0
  210. tests/regscale/integrations/commercial/aws/test_iam_evidence.py +1375 -0
  211. tests/regscale/integrations/commercial/aws/test_kms_control_mappings.py +656 -0
  212. tests/regscale/integrations/commercial/aws/test_kms_evidence.py +1163 -0
  213. tests/regscale/integrations/commercial/aws/test_ocsf_mapper.py +370 -0
  214. tests/regscale/integrations/commercial/aws/test_org_control_mappings.py +546 -0
  215. tests/regscale/integrations/commercial/aws/test_org_evidence.py +1240 -0
  216. tests/regscale/integrations/commercial/aws/test_s3_control_mappings.py +672 -0
  217. tests/regscale/integrations/commercial/aws/test_s3_evidence.py +987 -0
  218. tests/regscale/integrations/commercial/aws/test_scanner_evidence.py +373 -0
  219. tests/regscale/integrations/commercial/aws/test_security_hub_config_filtering.py +539 -0
  220. tests/regscale/integrations/commercial/aws/test_session_manager.py +516 -0
  221. tests/regscale/integrations/commercial/aws/test_ssm_control_mappings.py +588 -0
  222. tests/regscale/integrations/commercial/aws/test_ssm_evidence.py +735 -0
  223. tests/regscale/integrations/commercial/conftest.py +28 -0
  224. tests/regscale/integrations/commercial/microsoft_defender/__init__.py +1 -0
  225. tests/regscale/integrations/commercial/microsoft_defender/test_defender.py +1517 -0
  226. tests/regscale/integrations/commercial/microsoft_defender/test_defender_api.py +1748 -0
  227. tests/regscale/integrations/commercial/microsoft_defender/test_defender_constants.py +327 -0
  228. tests/regscale/integrations/commercial/microsoft_defender/test_defender_scanner.py +487 -0
  229. tests/regscale/integrations/commercial/test_aws.py +3742 -0
  230. tests/regscale/integrations/commercial/test_burp.py +48 -0
  231. tests/regscale/integrations/commercial/test_crowdstrike.py +49 -0
  232. tests/regscale/integrations/commercial/test_dependabot.py +341 -0
  233. tests/regscale/integrations/commercial/test_gcp.py +1543 -0
  234. tests/regscale/integrations/commercial/test_gitlab.py +549 -0
  235. tests/regscale/integrations/commercial/test_ip_mac_address_length.py +84 -0
  236. tests/regscale/integrations/commercial/test_jira.py +2204 -0
  237. tests/regscale/integrations/commercial/test_npm_audit.py +42 -0
  238. tests/regscale/integrations/commercial/test_okta.py +1228 -0
  239. tests/regscale/integrations/commercial/test_sarif_converter.py +251 -0
  240. tests/regscale/integrations/commercial/test_sicura.py +349 -0
  241. tests/regscale/integrations/commercial/test_snow.py +423 -0
  242. tests/regscale/integrations/commercial/test_sonarcloud.py +394 -0
  243. tests/regscale/integrations/commercial/test_sqlserver.py +186 -0
  244. tests/regscale/integrations/commercial/test_stig.py +33 -0
  245. tests/regscale/integrations/commercial/test_stig_mapper.py +153 -0
  246. tests/regscale/integrations/commercial/test_stigv2.py +406 -0
  247. tests/regscale/integrations/commercial/test_wiz.py +1365 -0
  248. tests/regscale/integrations/commercial/test_wiz_inventory.py +256 -0
  249. tests/regscale/integrations/commercial/wizv2/__init__.py +339 -0
  250. tests/regscale/integrations/commercial/wizv2/compliance/__init__.py +1 -0
  251. tests/regscale/integrations/commercial/wizv2/compliance/test_helpers.py +903 -0
  252. tests/regscale/integrations/commercial/wizv2/core/__init__.py +1 -0
  253. tests/regscale/integrations/commercial/wizv2/core/test_auth.py +701 -0
  254. tests/regscale/integrations/commercial/wizv2/core/test_client.py +1037 -0
  255. tests/regscale/integrations/commercial/wizv2/core/test_file_operations.py +989 -0
  256. tests/regscale/integrations/commercial/wizv2/fetchers/__init__.py +1 -0
  257. tests/regscale/integrations/commercial/wizv2/fetchers/test_policy_assessment.py +805 -0
  258. tests/regscale/integrations/commercial/wizv2/parsers/__init__.py +1 -0
  259. tests/regscale/integrations/commercial/wizv2/parsers/test_main.py +1153 -0
  260. tests/regscale/integrations/commercial/wizv2/processors/__init__.py +1 -0
  261. tests/regscale/integrations/commercial/wizv2/processors/test_finding.py +671 -0
  262. tests/regscale/integrations/commercial/wizv2/test_WizDataMixin.py +537 -0
  263. tests/regscale/integrations/commercial/wizv2/test_click_comprehensive.py +851 -0
  264. tests/regscale/integrations/commercial/wizv2/test_compliance_report_comprehensive.py +910 -0
  265. tests/regscale/integrations/commercial/wizv2/test_compliance_report_normalization.py +138 -0
  266. tests/regscale/integrations/commercial/wizv2/test_file_cleanup.py +283 -0
  267. tests/regscale/integrations/commercial/wizv2/test_file_operations.py +260 -0
  268. tests/regscale/integrations/commercial/wizv2/test_issue.py +343 -0
  269. tests/regscale/integrations/commercial/wizv2/test_issue_comprehensive.py +1203 -0
  270. tests/regscale/integrations/commercial/wizv2/test_reports.py +497 -0
  271. tests/regscale/integrations/commercial/wizv2/test_sbom.py +643 -0
  272. tests/regscale/integrations/commercial/wizv2/test_scanner_comprehensive.py +805 -0
  273. tests/regscale/integrations/commercial/wizv2/test_wiz_click_client_id.py +165 -0
  274. tests/regscale/integrations/commercial/wizv2/test_wiz_compliance_report.py +1394 -0
  275. tests/regscale/integrations/commercial/wizv2/test_wiz_compliance_unit.py +341 -0
  276. tests/regscale/integrations/commercial/wizv2/test_wiz_control_normalization.py +138 -0
  277. tests/regscale/integrations/commercial/wizv2/test_wiz_findings_comprehensive.py +364 -0
  278. tests/regscale/integrations/commercial/wizv2/test_wiz_inventory_comprehensive.py +644 -0
  279. tests/regscale/integrations/commercial/wizv2/test_wiz_status_mapping.py +149 -0
  280. tests/regscale/integrations/commercial/wizv2/test_wizv2.py +1218 -0
  281. tests/regscale/integrations/commercial/wizv2/test_wizv2_utils.py +519 -0
  282. tests/regscale/integrations/commercial/wizv2/utils/__init__.py +1 -0
  283. tests/regscale/integrations/commercial/wizv2/utils/test_main.py +1523 -0
  284. tests/regscale/integrations/public/__init__.py +0 -0
  285. tests/regscale/integrations/public/fedramp/__init__.py +1 -0
  286. tests/regscale/integrations/public/fedramp/test_gen_asset_list.py +150 -0
  287. tests/regscale/integrations/public/fedramp/test_poam_export_v5.py +1293 -0
  288. tests/regscale/integrations/public/test_alienvault.py +220 -0
  289. tests/regscale/integrations/public/test_cci.py +1053 -0
  290. tests/regscale/integrations/public/test_cisa.py +1021 -0
  291. tests/regscale/integrations/public/test_emass.py +518 -0
  292. tests/regscale/integrations/public/test_fedramp.py +1152 -0
  293. tests/regscale/integrations/public/test_fedramp_cis_crm.py +3661 -0
  294. tests/regscale/integrations/public/test_file_uploads.py +506 -0
  295. tests/regscale/integrations/public/test_oscal.py +453 -0
  296. tests/regscale/integrations/test_compliance_status_mapping.py +406 -0
  297. tests/regscale/integrations/test_control_matcher.py +1421 -0
  298. tests/regscale/integrations/test_control_matching.py +155 -0
  299. tests/regscale/integrations/test_milestone_manager.py +408 -0
  300. tests/regscale/models/test_control_implementation.py +118 -3
  301. tests/regscale/models/test_form_field_value_integration.py +304 -0
  302. tests/regscale/models/test_issue.py +378 -1
  303. tests/regscale/models/test_module_integration.py +582 -0
  304. tests/regscale/models/test_tenable_integrations.py +811 -105
  305. regscale/integrations/commercial/wizv2/policy_compliance.py +0 -3057
  306. regscale/integrations/public/fedramp/mappings/fedramp_r4_parts.json +0 -7388
  307. regscale/integrations/public/fedramp/mappings/fedramp_r5_parts.json +0 -9605
  308. regscale/integrations/public/fedramp/parts_mapper.py +0 -107
  309. /regscale/integrations/commercial/{amazon → sarif}/__init__.py +0 -0
  310. /regscale/integrations/commercial/wizv2/{wiz_auth.py → core/auth.py} +0 -0
  311. {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/LICENSE +0 -0
  312. {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/WHEEL +0 -0
  313. {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/entry_points.txt +0 -0
  314. {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/top_level.txt +0 -0
@@ -8,12 +8,13 @@ import json
8
8
  import math
9
9
  import re
10
10
  import shutil
11
- import tempfile
12
11
  from collections import Counter
13
12
  from concurrent.futures import as_completed
14
13
  from concurrent.futures.thread import ThreadPoolExecutor
15
14
  from datetime import datetime
15
+ from functools import lru_cache
16
16
  from pathlib import Path
17
+ from tempfile import gettempdir
17
18
  from threading import Thread
18
19
  from types import ModuleType
19
20
  from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, Optional, Tuple, TypeVar
@@ -24,7 +25,7 @@ from regscale.core.app.api import Api
24
25
  from regscale.core.app.utils.api_handler import APIInsertionError, APIUpdateError
25
26
  from regscale.core.app.utils.app_utils import compute_hash, create_progress_object, error_and_exit, get_current_datetime
26
27
  from regscale.core.utils.graphql import GraphQLQuery
27
- from regscale.integrations.public.fedramp.parts_mapper import PartMapper
28
+ from regscale.integrations.control_matcher import ControlMatcher
28
29
  from regscale.integrations.public.fedramp.ssp_logger import SSPLogger
29
30
  from regscale.models import ControlObjective, ImplementationObjective, ImportValidater, Parameter, Profile
30
31
  from regscale.models.regscale_models import (
@@ -43,14 +44,9 @@ from regscale.utils.version import RegscaleVersion
43
44
  if TYPE_CHECKING:
44
45
  import pandas as pd
45
46
 
46
- from functools import lru_cache
47
- from tempfile import gettempdir
48
-
49
47
  T = TypeVar("T")
50
48
 
51
49
  logger = SSPLogger()
52
- part_mapper_rev5 = PartMapper()
53
- part_mapper_rev4 = PartMapper()
54
50
  progress = create_progress_object()
55
51
 
56
52
  SERVICE_PROVIDER_CORPORATE = "Service Provider Corporate"
@@ -120,6 +116,112 @@ def get_pandas() -> ModuleType:
120
116
  return pd
121
117
 
122
118
 
119
+ def _build_potential_oscal_ids(variation: str) -> List[str]:
120
+ """
121
+ Build potential OSCAL ID formats from a control ID variation.
122
+
123
+ :param str variation: Control ID variation (e.g., "AC-1", "AC-01", "AC-1.a")
124
+ :return: List of potential OSCAL IDs
125
+ :rtype: List[str]
126
+ """
127
+ variation_lower = variation.lower()
128
+ oscal_ids = []
129
+
130
+ # Check if this is a control with a letter part (e.g., "ac-1.a")
131
+ if re.match(r"^[a-z]+-\d+\.[a-z]$", variation_lower):
132
+ # For letter parts, map to OSCAL format: ac-1.a -> ac-1_smt.a
133
+ base_control = variation_lower.rsplit(".", 1)[0] # Get "ac-1" from "ac-1.a"
134
+ letter_part = variation_lower.rsplit(".", 1)[1] # Get "a" from "ac-1.a"
135
+ oscal_ids.extend(
136
+ [
137
+ f"{base_control}_smt.{letter_part}", # ac-1_smt.a (primary format)
138
+ f"{variation_lower}_smt", # ac-1.a_smt (alternative format)
139
+ ]
140
+ )
141
+ else:
142
+ # Base control without letter part - include all potential letter variations
143
+ oscal_ids.extend(
144
+ [
145
+ f"{variation_lower}_smt",
146
+ f"{variation_lower}_smt.a",
147
+ f"{variation_lower}_smt.b",
148
+ f"{variation_lower}_smt.c",
149
+ ]
150
+ )
151
+
152
+ return oscal_ids
153
+
154
+
155
+ def _matches_oscal_id(obj_id: str, variation: str) -> bool:
156
+ """
157
+ Check if an objective's otherId matches any OSCAL ID format for the given variation.
158
+
159
+ :param str obj_id: The objective's otherId
160
+ :param str variation: Control ID variation
161
+ :return: True if matches, False otherwise
162
+ :rtype: bool
163
+ """
164
+ potential_ids = _build_potential_oscal_ids(variation)
165
+ return obj_id in potential_ids or obj_id.startswith(f"{variation.lower()}_smt")
166
+
167
+
168
+ def _find_matching_objectives(control_objectives: List[ControlObjective], variations: set) -> List[ControlObjective]:
169
+ """
170
+ Find objectives that match any of the control ID variations.
171
+
172
+ :param List[ControlObjective] control_objectives: List of objectives to search
173
+ :param set variations: Set of control ID variations
174
+ :return: List of matched objectives
175
+ :rtype: List[ControlObjective]
176
+ """
177
+ matched_objectives = []
178
+
179
+ for obj in control_objectives:
180
+ if not hasattr(obj, "otherId") or not obj.otherId:
181
+ continue
182
+
183
+ obj_id = obj.otherId
184
+ for variation in variations:
185
+ if _matches_oscal_id(obj_id, variation):
186
+ if obj not in matched_objectives:
187
+ matched_objectives.append(obj)
188
+ break
189
+
190
+ return matched_objectives
191
+
192
+
193
+ def find_objectives_using_control_matcher(
194
+ source: str, control_objectives: List[ControlObjective], control_matcher: ControlMatcher
195
+ ) -> Tuple[List[ControlObjective], str]:
196
+ """
197
+ Find control objectives using ControlMatcher for consistent control ID parsing and matching.
198
+
199
+ :param str source: The source control ID (e.g., "AC-1(a)", "AC-01 (a)")
200
+ :param List[ControlObjective] control_objectives: List of ControlObjective objects to search
201
+ :param ControlMatcher control_matcher: Instance of ControlMatcher for parsing and variations
202
+ :return: Tuple of (matched objectives list, status_message)
203
+ :rtype: Tuple[List[ControlObjective], str]
204
+ """
205
+ # Parse the control ID using ControlMatcher
206
+ parsed_id = control_matcher.parse_control_id(source)
207
+ if not parsed_id:
208
+ return [], f"Unable to parse control {source}"
209
+
210
+ # Get all variations of this control ID
211
+ # pylint: disable=protected-access # Using internal method for control ID variation matching
212
+ variations = control_matcher._get_control_id_variations(parsed_id)
213
+ if not variations:
214
+ return [], f"Unable to generate variations for {source}"
215
+
216
+ # Find matching objectives
217
+ matched_objectives = _find_matching_objectives(control_objectives, variations)
218
+
219
+ if matched_objectives:
220
+ return matched_objectives, f"Found {len(matched_objectives)} objective(s) for {source}"
221
+
222
+ return [], f"No database match found for {source} (parsed: {parsed_id})"
223
+
224
+
123
225
  def transform_control(control: str) -> str:
124
226
  """
125
227
  Function to parse the control string and transform it to the RegScale format
@@ -129,16 +231,15 @@ def transform_control(control: str) -> str:
129
231
  :return: Transformed control ID to match RegScale control ID format
130
232
  :rtype: str
131
233
  """
132
- # Use regex to match the pattern and capture the parts
133
- match = re.match(r"([A-Za-z]+)-(\d+)\s\((\d+|[a-z])\)", control)
134
- if match:
234
+ # Use regex to match the pattern and capture the parts (handle extra spaces)
235
+ # Now handles both uppercase and lowercase letters in parentheses
236
+ if match := re.match(r"([A-Z]+)-(\d+)\s*\(\s*(\d+|[A-Z])\s*\)", control, re.IGNORECASE):
135
237
  control_name = match.group(1).lower()
136
238
  control_number = match.group(2)
137
- sub_control = match.group(3)
138
-
139
- if sub_control.isdigit():
239
+ try:
240
+ sub_control = match.group(3).lower() # Normalize to lowercase
140
241
  transformed_control = f"{control_name}-{control_number}.{sub_control}"
141
- else:
242
+ except IndexError:
142
243
  transformed_control = f"{control_name}-{control_number}"
143
244
 
144
245
  return transformed_control
@@ -179,29 +280,76 @@ def new_leveraged_auth(
179
280
  return new_leveraged_auth_id.id
180
281
 
181
282
 
182
- def gen_key(control_id: str):
283
+ def gen_key(control_id: str) -> str:
183
284
  """
184
- Function to generate a key for the control ID
285
+ Function to generate a key for the control ID by stripping letter-based parts.
286
+ Handles both parentheses notation (AC-1(a)) and dot notation (ac-1.a).
287
+
288
+ Examples:
289
+ - AC-1 (a) -> AC-1
290
+ - ac-1.a -> ac-1
291
+ - AC-2(1) -> AC-2(1) (numeric enhancement preserved)
292
+ - AC-17.2 -> AC-17.2 (numeric enhancement preserved)
185
293
 
186
294
  :param str control_id: The control ID to generate a key for
187
- :return: The generated key
295
+ :return: The generated key with letter parts stripped
188
296
  :rtype: str
189
297
  """
190
- # Match pattern: captures everything up to either:
191
- # 1. The last (number) if it exists
298
+ # First, try parentheses notation: ALPHA-NUM(LETTER) -> ALPHA-NUM
299
+ # Captures everything up to either:
300
+ # 1. The last (number) if it exists (preserved)
192
301
  # 2. The main control number if no enhancement exists
193
- # And excludes any trailing (letter)
194
- pattern = r"^((?:\w+-\d+(?:\(\d+\))?))(?:\([a-zA-Z]\))?$"
302
+ # Excludes trailing (letter) - handles extra spaces like AC-6 ( 1 ) ( a )
303
+ pattern_paren = r"^(\w+-\d+(?:\s*\(\s*\d+\s*\))?)(?:\s*\(\s*[a-zA-Z]\s*\))?$"
304
+ if match := re.match(pattern_paren, control_id):
305
+ return match.group(1)
195
306
 
196
- match = re.match(pattern, control_id)
197
- if match:
307
+ # Try dot notation: alpha-num.letter -> alpha-num
308
+ # Preserves numeric enhancements (ac-17.2) but strips letter parts (ac-1.a)
309
+ pattern_dot = r"^([a-z]+-\d+)\.([a-z])$"
310
+ if match := re.match(pattern_dot, control_id, re.IGNORECASE):
311
+ # Check if the part after dot is a single letter (not a number)
198
312
  return match.group(1)
313
+
314
+ # No match, return as-is
199
315
  return control_id
200
316
 
201
317
 
318
+ def _is_letter_based_control_part(control_id: str) -> bool:
319
+ """
320
+ Check if a control ID is a letter-based part (e.g., AC-1(a), ac-1.a).
321
+ Returns True for ALPHA-NUMERIC(ALPHA) or alpha-numeric.alpha patterns.
322
+ Returns False for numeric enhancements (AC-1(1), ac-17.2).
323
+
324
+ :param str control_id: The control ID to check
325
+ :return: True if it's a letter-based control part
326
+ :rtype: bool
327
+ """
328
+ # Pattern 1: Parentheses notation - ALPHA-NUMERIC(ALPHA) like AC-1(a), AC-2(B)
329
+ pattern_paren = r"^[A-Za-z]+-\d+\s*\(\s*[a-zA-Z]\s*\)$"
330
+ if re.match(pattern_paren, control_id):
331
+ return True
332
+
333
+ # Pattern 2: Dot notation - alpha-numeric.alpha like ac-1.a, ac-2.b
334
+ # Exclude numeric enhancements like ac-17.2
335
+ pattern_dot = r"^[a-z]+-\d+\.([a-z])$"
336
+ match = re.match(pattern_dot, control_id, re.IGNORECASE)
337
+ if match and match.group(1).isalpha():
338
+ return True
339
+
340
+ return False
341
+
342
+
202
343
  def map_implementation_status(control_id: str, cis_data: dict) -> str:
203
344
  """
204
- Function to map the selected implementation status on the CIS worksheet to a RegScale status
345
+ Function to map the selected implementation status on the CIS worksheet to a RegScale status.
346
+ Aggregates letter-based control parts (AC-1(a), AC-1(b), AC-1(c)) into base control (AC-1).
347
+
348
+ Aggregation logic for letter-based parts:
349
+ - All "Implemented" → "Fully Implemented"
350
+ - Mix with at least one "Implemented" → "Partially Implemented"
351
+ - All "Not Implemented" or empty → "Not Implemented"
352
+ - Any "Planned" (no implemented) → "Planned"
205
353
 
206
354
  :param str control_id: The control ID from RegScale
207
355
  :param dict cis_data: Data from the CIS worksheet to map the status from
@@ -209,7 +357,7 @@ def map_implementation_status(control_id: str, cis_data: dict) -> str:
209
357
  :rtype: str
210
358
  """
211
359
 
212
- # Extract matching records
360
+ # Extract matching records (gen_key strips letter parts to match base control)
213
361
  cis_records = [
214
362
  value
215
363
  for value in cis_data.values()
@@ -221,31 +369,49 @@ def map_implementation_status(control_id: str, cis_data: dict) -> str:
221
369
  logger.debug("Found %d CIS records for control %s", len(cis_records), control_id)
222
370
 
223
371
  if not cis_records:
372
+ # Alerts if a control exists in regscale but is missing from CIS worksheet
224
373
  logger.warning(f"No CIS records found for control {control_id}")
225
374
  return status_ret
226
375
 
376
+ # Check if these are letter-based control parts that need aggregation
377
+ has_letter_parts = any(_is_letter_based_control_part(rec.get("control_id", "")) for rec in cis_records)
378
+
227
379
  # Count implementation statuses
228
380
  status_counts = Counter(record.get("implementation_status", "") for record in cis_records)
229
- logger.debug("Status distribution for %s: %s", control_id, dict(status_counts))
381
+ logger.debug("Status distribution for %s: %s (letter parts: %s)", control_id, dict(status_counts), has_letter_parts)
230
382
 
231
- # Early returns for simple cases
383
+ # Early return for simple case: all same status
232
384
  if len(status_counts) == 1:
233
385
  status = next(iter(status_counts))
234
- return STATUS_MAPPING.get(status, ControlImplementationStatus.NotImplemented)
386
+ mapped_status = STATUS_MAPPING.get(status, ControlImplementationStatus.NotImplemented)
387
+ # If all letter parts have same status and it's "Implemented", return FullyImplemented
388
+ if has_letter_parts and status == "Implemented":
389
+ return ControlImplementationStatus.FullyImplemented
390
+ return mapped_status
235
391
 
392
+ # Aggregate statuses for letter-based control parts or multiple records
236
393
  implemented_count = status_counts.get("Implemented", 0)
394
+ not_implemented_count = status_counts.get("", 0) # Empty status counts as not implemented
395
+ partially_implemented_count = status_counts.get("Partially Implemented", 0)
396
+ planned_count = status_counts.get("Planned", 0)
237
397
  total_count = sum(status_counts.values())
238
398
 
399
+ # Aggregation logic
239
400
  if implemented_count == total_count:
401
+ # All parts are implemented
240
402
  return ControlImplementationStatus.FullyImplemented
241
- elif implemented_count > 0 or any(status == "Partially Implemented" for status in status_counts):
242
- status_ret = ControlImplementationStatus.PartiallyImplemented
243
- elif any(status == "Planned" for status in status_counts):
244
- status_ret = ControlImplementationStatus.Planned
403
+ elif implemented_count > 0 or partially_implemented_count > 0:
404
+ # Mix of implemented and other statuses, or any partially implemented
405
+ return ControlImplementationStatus.PartiallyImplemented
406
+ elif planned_count > 0 and not_implemented_count == 0:
407
+ # All are planned (no not-implemented)
408
+ return ControlImplementationStatus.Planned
245
409
  elif any(status in ["N/A", ALTERNATIVE_IMPLEMENTATION] for status in status_counts):
246
- status_ret = ControlImplementationStatus.NA
247
-
248
- return status_ret
410
+ # Any N/A or Alternative
411
+ return ControlImplementationStatus.NA
412
+ else:
413
+ # Default: not implemented
414
+ return ControlImplementationStatus.NotImplemented
249
415
 
250
416
 
251
417
  def map_origination(control_id: str, cis_data: dict) -> dict:
@@ -268,7 +434,7 @@ def map_origination(control_id: str, cis_data: dict) -> dict:
268
434
  }
269
435
 
270
436
  # Initialize result with all flags set to False
271
- result = {key: False for key in origination_mapping.values()}
437
+ result = dict.fromkeys(origination_mapping.values(), False)
272
438
  result["record_text"] = ""
273
439
 
274
440
  # Find matching CIS records
@@ -345,6 +511,103 @@ def get_multi_status(record: dict) -> str:
345
511
  return status_map.get(implementation_status, NOT_IMPLEMENTED)
346
512
 
347
513
 
514
+ def _calculate_responsibility(control_originations: List[str], imp: ControlImplementation) -> str:
515
+ """
516
+ Calculate responsibility from control originations.
517
+
518
+ :param List[str] control_originations: List of control origination values
519
+ :param ControlImplementation imp: Control implementation
520
+ :return: Calculated responsibility value
521
+ :rtype: str
522
+ """
523
+ try:
524
+ if RegscaleVersion.meets_minimum_version("6.20.17.0"):
525
+ return ",".join(control_originations)
526
+ return next(iter(control_originations))
527
+ except StopIteration:
528
+ if imp.responsibility:
529
+ return imp.responsibility.split(",")[0]
530
+ return SERVICE_PROVIDER_CORPORATE
531
+
532
+
533
+ def _create_new_implementation_objective(
534
+ leverage_auth_id: int,
535
+ imp: ControlImplementation,
536
+ objective: ControlObjective,
537
+ cis_record: dict,
538
+ responsibility: str,
539
+ cloud_responsibility: str,
540
+ customer_responsibility: str,
541
+ can_be_inherited_from_csp: str,
542
+ ) -> ImplementationObjective:
543
+ """
544
+ Create a new implementation objective.
545
+
546
+ :param int leverage_auth_id: Leveraged authorization ID
547
+ :param ControlImplementation imp: Control implementation
548
+ :param ControlObjective objective: Control objective
549
+ :param dict cis_record: CIS record data
550
+ :param str responsibility: Responsibility value
551
+ :param str cloud_responsibility: Cloud responsibility value
552
+ :param str customer_responsibility: Customer responsibility value
553
+ :param str can_be_inherited_from_csp: Can be inherited flag
554
+ :return: New implementation objective
555
+ :rtype: ImplementationObjective
556
+ """
557
+ imp_obj = ImplementationObjective(
558
+ id=0,
559
+ uuid="",
560
+ inherited=can_be_inherited_from_csp in ["Yes", "Partial"],
561
+ implementationId=imp.id,
562
+ status=get_multi_status(cis_record),
563
+ objectiveId=objective.id,
564
+ notes=objective.name,
565
+ securityControlId=objective.securityControlId,
566
+ securityPlanId=REGSCALE_SSP_ID,
567
+ responsibility=responsibility,
568
+ cloudResponsibility=cloud_responsibility,
569
+ customerResponsibility=customer_responsibility,
570
+ authorizationId=leverage_auth_id,
571
+ parentObjectiveId=objective.parentObjectiveId,
572
+ )
573
+ logger.debug(
574
+ "Creating new Implementation Objective for Control %s with status: %s responsibility: %s",
575
+ imp_obj.securityControlId,
576
+ imp_obj.status,
577
+ imp_obj.responsibility,
578
+ )
579
+ return imp_obj
580
+
581
+
582
+ def _update_existing_implementation_objective(
583
+ ex_obj: ImplementationObjective,
584
+ cis_record: dict,
585
+ responsibility: str,
586
+ cloud_responsibility: str,
587
+ customer_responsibility: str,
588
+ ) -> None:
589
+ """
590
+ Update an existing implementation objective.
591
+
592
+ :param ImplementationObjective ex_obj: Existing implementation objective
593
+ :param dict cis_record: CIS record data
594
+ :param str responsibility: Responsibility value
595
+ :param str cloud_responsibility: Cloud responsibility value
596
+ :param str customer_responsibility: Customer responsibility value
597
+ :rtype: None
598
+ """
599
+ ex_obj.status = get_multi_status(cis_record)
600
+ ex_obj.responsibility = responsibility
601
+ if cloud_responsibility.strip():
602
+ logger.debug(f"Updating Implementation Objective #{ex_obj.id} with responsibility: {responsibility}")
603
+ ex_obj.cloudResponsibility = cloud_responsibility
604
+ if customer_responsibility.strip():
605
+ logger.debug(
606
+ f"Updating Implementation Objective #{ex_obj.id} with cloud responsibility: {cloud_responsibility}"
607
+ )
608
+ ex_obj.customerResponsibility = customer_responsibility
609
+
610
+
348
611
  def update_imp_objective(
349
612
  leverage_auth_id: int,
350
613
  existing_imp_obj: List[ImplementationObjective],
@@ -363,80 +626,49 @@ def update_imp_objective(
363
626
  :rtype: None
364
627
  :return: None
365
628
  """
366
-
367
629
  cis_record = record.get("cis", {})
368
630
  crm_record = record.get("crm", {})
369
- # There could be multiples, take the first one as regscale will not allow multiples at the objective level.
370
- control_originations = cis_record.get("control_origination", "").split(",")
371
- for ix, control_origination in enumerate(control_originations):
372
- control_originations[ix] = control_origination.strip()
373
631
 
374
- try:
375
- if RegscaleVersion.meets_minimum_version("6.20.17.0"):
376
- responsibility = ",".join(control_originations)
377
- else:
378
- responsibility = next(origin for origin in control_originations)
632
+ # Parse and clean control originations
633
+ control_originations = [orig.strip() for orig in cis_record.get("control_origination", "").split(",")]
379
634
 
380
- except StopIteration:
381
- if imp.responsibility:
382
- responsibility = imp.responsibility.split(",")[0] # only one responsiblity allowed here.
383
- else:
384
- responsibility = SERVICE_PROVIDER_CORPORATE
635
+ # Calculate responsibility
636
+ responsibility = _calculate_responsibility(control_originations, imp)
385
637
 
638
+ # Parse responsibility fields
386
639
  customer_responsibility = clean_customer_responsibility(
387
640
  crm_record.get("specific_inheritance_and_customer_agency_csp_responsibilities")
388
641
  )
389
- existing_pairs = {(obj.objectiveId, obj.implementationId) for obj in existing_imp_obj}
390
- logger.debug(f"CRM Record: {crm_record}")
391
642
  can_be_inherited_from_csp: str = crm_record.get("can_be_inherited_from_csp") or ""
392
643
  cloud_responsibility = customer_responsibility if can_be_inherited_from_csp.lower() == "yes" else ""
393
644
  customer_responsibility = customer_responsibility if can_be_inherited_from_csp.lower() != "yes" else ""
645
+
646
+ existing_pairs = {(obj.objectiveId, obj.implementationId) for obj in existing_imp_obj}
647
+ logger.debug(f"CRM Record: {crm_record}")
648
+
394
649
  for objective in objectives:
650
+ if objective.securityControlId != imp.controlID:
651
+ continue
652
+
395
653
  current_pair = (objective.id, imp.id)
396
654
  if current_pair not in existing_pairs:
397
- if objective.securityControlId != imp.controlID:
398
- # This is a bad match, do not save.
399
- continue
400
-
401
- imp_obj = ImplementationObjective(
402
- id=0,
403
- uuid="",
404
- inherited=can_be_inherited_from_csp in ["Yes", "Partial"],
405
- implementationId=imp.id,
406
- status=get_multi_status(cis_record),
407
- objectiveId=objective.id,
408
- notes=objective.name,
409
- securityControlId=objective.securityControlId,
410
- securityPlanId=REGSCALE_SSP_ID,
411
- responsibility=responsibility,
412
- cloudResponsibility=cloud_responsibility,
413
- customerResponsibility=customer_responsibility,
414
- authorizationId=leverage_auth_id,
415
- parentObjectiveId=objective.parentObjectiveId,
416
- )
417
- logger.debug(
418
- "Creating new Implementation Objective for Control %s with status: %s responsibility: %s",
419
- imp_obj.securityControlId,
420
- imp_obj.status,
421
- imp_obj.responsibility,
655
+ imp_obj = _create_new_implementation_objective(
656
+ leverage_auth_id,
657
+ imp,
658
+ objective,
659
+ cis_record,
660
+ responsibility,
661
+ cloud_responsibility,
662
+ customer_responsibility,
663
+ can_be_inherited_from_csp,
422
664
  )
423
665
  UPDATED_IMPLEMENTATION_OBJECTIVES.add(imp_obj)
424
666
  else:
425
667
  ex_obj = next((obj for obj in existing_imp_obj if obj.objectiveId == objective.id), None)
426
668
  if ex_obj:
427
- ex_obj.status = get_multi_status(cis_record)
428
- ex_obj.responsibility = responsibility
429
- if cloud_responsibility.strip():
430
- logger.debug(
431
- f"Updating Implementation Objective #{ex_obj.id} with responsibility: {responsibility}"
432
- )
433
- ex_obj.cloudResponsibility = cloud_responsibility
434
- if customer_responsibility.strip():
435
- logger.debug(
436
- f"Updating Implementation Objective #{ex_obj.id} with cloud responsibility: {cloud_responsibility}"
437
- )
438
- ex_obj.customerResponsibility = customer_responsibility
439
-
669
+ _update_existing_implementation_objective(
670
+ ex_obj, cis_record, responsibility, cloud_responsibility, customer_responsibility
671
+ )
440
672
  UPDATED_IMPLEMENTATION_OBJECTIVES.add(ex_obj)
441
673
 
442
674
 
@@ -565,8 +797,6 @@ def get_all_imps(api: Api, cis_data: dict, version: Literal["rev4", "rev5"]) ->
565
797
  :return: None
566
798
  :rtype: None
567
799
  """
568
- from requests import RequestException
569
-
570
800
  # Check if the response is successful
571
801
  if EXISTING_IMPLEMENTATIONS:
572
802
  # Get Control Implementations For SSP
@@ -641,6 +871,9 @@ def update_all_objectives(
641
871
  """
642
872
 
643
873
  all_control_objectives = get_all_control_objectives(imps=EXISTING_IMPLEMENTATIONS.values())
874
+ # Create ControlMatcher instance for consistent control ID parsing
875
+ control_matcher = ControlMatcher()
876
+
644
877
  error_set = set()
645
878
  process_task = progress.add_task(
646
879
  "[cyan]Processing control objectives...", total=len(EXISTING_IMPLEMENTATIONS.values())
@@ -653,7 +886,13 @@ def update_all_objectives(
653
886
  # Submit all tasks
654
887
  future_to_control = {
655
888
  executor.submit(
656
- process_implementation, leveraged_auth_id, imp, combined_data, version, all_control_objectives
889
+ process_implementation,
890
+ leveraged_auth_id,
891
+ imp,
892
+ combined_data,
893
+ version,
894
+ all_control_objectives,
895
+ control_matcher,
657
896
  ): imp
658
897
  for imp in EXISTING_IMPLEMENTATIONS.values()
659
898
  }
@@ -709,6 +948,7 @@ def process_implementation(
709
948
  sheet_data: dict,
710
949
  version: Literal["rev4", "rev5"],
711
950
  all_objectives: List[ControlObjective],
951
+ control_matcher: ControlMatcher,
712
952
  ) -> Tuple[List[str], List[ImplementationObjective]]:
713
953
  """
714
954
  Processes a single implementation and its associated records.
@@ -718,6 +958,7 @@ def process_implementation(
718
958
  :param dict sheet_data: The CIS/CRM data to process
719
959
  :param Literal["rev4", "rev5"] version: The version of the workbook
720
960
  :param List[ControlObjective] all_objectives: all the control objectives
961
+ :param ControlMatcher control_matcher: ControlMatcher instance for control ID parsing
721
962
  :rtype Tuple[List[str], List[ImplementationObjective]]
722
963
  :returns A list of updated implementation objectives
723
964
  """
@@ -725,7 +966,7 @@ def process_implementation(
725
966
  errors = []
726
967
  processed_objectives = []
727
968
 
728
- existing_objectives, filtered_records = gen_filtered_records(implementation, sheet_data, version)
969
+ existing_objectives, filtered_records = gen_filtered_records(implementation, sheet_data, control_matcher)
729
970
  result = None
730
971
  for record in filtered_records:
731
972
  res = process_single_record(
@@ -735,6 +976,7 @@ def process_implementation(
735
976
  control_objectives=all_objectives,
736
977
  existing_objectives=existing_objectives,
737
978
  version=version,
979
+ control_matcher=control_matcher,
738
980
  )
739
981
  if isinstance(res, tuple):
740
982
  method_errors, result = res
@@ -745,39 +987,67 @@ def process_implementation(
745
987
  return errors, processed_objectives
746
988
 
747
989
 
990
+ def _extract_base_control_id(control_id: str) -> str:
991
+ """
992
+ Extract the base control ID from a control ID that may have a letter part.
993
+
994
+ Examples:
995
+ - "AC-1.a" -> "AC-1"
996
+ - "AC-17.2" -> "AC-17.2" (numeric parts are preserved)
997
+ - "AC-1" -> "AC-1"
998
+
999
+ :param str control_id: Control ID that may have a letter part
1000
+ :return: Base control ID without letter part
1001
+ :rtype: str
1002
+ """
1003
+ # Check if the control has a letter part (e.g., AC-1.a)
1004
+ match = re.match(r"^([A-Z]+-\d+)\.[A-Z]$", control_id, re.IGNORECASE)
1005
+ if match:
1006
+ return match.group(1)
1007
+ return control_id
1008
+
1009
+
748
1010
  def gen_filtered_records(
749
- implementation: ControlImplementation, sheet_data: dict, version: Literal["rev4", "rev5"]
1011
+ implementation: ControlImplementation, sheet_data: dict, control_matcher: ControlMatcher
750
1012
  ) -> Tuple[List[ImplementationObjective], List[Dict[str, str]]]:
751
1013
  """
752
- Generates filtered records for a given implementation.
1014
+ Generates filtered records for a given implementation using ControlMatcher.
753
1015
 
754
1016
  :param ControlImplementation implementation: The control implementation to filter records for
755
1017
  :param dict sheet_data: The CIS/CRM data to filter
756
- :param Literal["rev4", "rev5"] version: The version of the workbook
1018
+ :param ControlMatcher control_matcher: ControlMatcher instance for control ID matching
757
1019
  :returns A tuple of existing objectives, and filtered records
758
1020
  :rtype Tuple[List[ImplementationObjective], List[Dict[str, str]]]
759
1021
  """
760
1022
  security_control = SecurityControl.get_object(implementation.controlID)
761
1023
  existing_objectives = ImplementationObjective.get_by_control(implementation.id)
762
- if version == "rev5":
763
- filtered_records = filter(
764
- lambda r: extract_control_name(r["cis"]["regscale_control_id"]).lower()
765
- == security_control.controlId.lower(),
766
- sheet_data.values(),
767
- )
768
- else:
769
- try:
770
- control_label = next(
771
- dat
772
- for dat in part_mapper_rev4.data
773
- if dat.get("Oscal Control ID") == security_control.controlId.lower()
774
- ).get("CONTROLLABEL")
775
- except StopIteration:
776
- control_label = None
777
- if control_label:
778
- filtered_records = [r for r in sheet_data.values() if r["cis"]["regscale_control_id"] == control_label]
1024
+
1025
+ # Get all variations of the control ID using ControlMatcher
1026
+ # pylint: disable=protected-access # Using internal method for control ID variation matching
1027
+ control_variations = control_matcher._get_control_id_variations(security_control.controlId)
1028
+
1029
+ # Filter records that match any variation of the control ID
1030
+ filtered_records = []
1031
+ for record in sheet_data.values():
1032
+ record_control_id = record["cis"].get("regscale_control_id", "")
1033
+ # Parse the record's control ID
1034
+ parsed_record_id = control_matcher.parse_control_id(record_control_id)
1035
+ if not parsed_record_id:
1036
+ continue
1037
+ # Get variations for the parsed record ID
1038
+ # pylint: disable=protected-access # Using internal method for control ID variation matching
1039
+ record_variations = control_matcher._get_control_id_variations(parsed_record_id)
1040
+
1041
+ # Check if the parsed record control ID matches any variation
1042
+ if control_variations & record_variations:
1043
+ filtered_records.append(record)
779
1044
  else:
780
- filtered_records = []
1045
+ # If no direct match and record has a letter part, try matching the base control
1046
+ base_control_id = _extract_base_control_id(parsed_record_id)
1047
+ if base_control_id != parsed_record_id:
1048
+ base_variations = control_matcher._get_control_id_variations(base_control_id)
1049
+ if control_variations & base_variations:
1050
+ filtered_records.append(record)
781
1051
 
782
1052
  return existing_objectives, filtered_records
783
1053
 
@@ -801,54 +1071,28 @@ def process_single_record(**kwargs) -> Tuple[List[str], Optional[ImplementationO
801
1071
  :rtype Tuple[List[str], Optional[ImplementationObjective]]
802
1072
  :returns A list of errors and the Implementation Objective if successful, otherwise None
803
1073
  """
804
- # for pytest
805
- if not part_mapper_rev5.data:
806
- part_mapper_rev5.load_fedramp_version_5_mapping()
807
- if not part_mapper_rev4.data:
808
- part_mapper_rev4.load_fedramp_version_4_mapping()
809
-
810
1074
  errors = []
811
- version = kwargs.get("version")
812
1075
  leveraged_auth_id: int = kwargs.get("leveraged_auth_id")
813
1076
  implementation: ControlImplementation = kwargs.get("implementation")
814
1077
  record: dict = kwargs.get("record")
815
1078
  control_objectives: List[ControlObjective] = kwargs.get("control_objectives")
816
1079
  existing_objectives: List[ImplementationObjective] = kwargs.get("existing_objectives")
817
- mapped_objectives: List[ControlObjective] = []
1080
+ control_matcher: ControlMatcher = kwargs.get("control_matcher")
818
1081
  result = None
819
- parts = []
820
- # Note: The Control ID from the CIS/CRM can be in non-standard formats, as compared to the example sheet on fedramp.
821
- if version == "rev5":
822
- key = record["cis"]["control_id"].replace(" ", "")
823
- source = part_mapper_rev5.find_by_source(key)
824
- else:
825
- key = record["cis"]["control_id"]
826
- source = part_mapper_rev4.find_by_source(key)
827
- if parts := part_mapper_rev4.find_sub_parts(key):
828
- for part in parts:
829
- try:
830
- if version == "rev5":
831
- mapped_objectives.append(next(obj for obj in control_objectives if obj.name == part))
832
- else:
833
- mapped_objectives.append(next(obj for obj in control_objectives if obj.otherId == part))
834
- except StopIteration:
835
- errors.append(f"Unable to find part {part} for control {key}")
836
- if not source and not parts:
837
- errors.append(f"Unable to find source and part for control {key}")
838
-
839
- if source and not parts:
840
- try:
841
- objective = next(
842
- obj
843
- for obj in control_objectives
844
- if (obj.otherId == source and version in ["rev5", "rev4"]) or (obj.name == source and version == "rev4")
845
- )
846
- mapped_objectives.append(objective)
847
- except StopIteration:
848
- logger.debug(f"Missing Source: {source}")
849
- errors.append(f"Unable to find objective for control {key} ({source})")
850
1082
 
851
- if mapped_objectives:
1083
+ # Get the control ID from the CIS/CRM record
1084
+ key = record["cis"]["control_id"]
1085
+
1086
+ # Use ControlMatcher to find matching objectives
1087
+ mapped_objectives, status = find_objectives_using_control_matcher(key, control_objectives, control_matcher)
1088
+
1089
+ logger.debug(f"Control matching result for {key}: {status}")
1090
+
1091
+ # Add to errors list if no objectives found
1092
+ if not mapped_objectives:
1093
+ errors.append(f"{key}: {status}")
1094
+ else:
1095
+ # Update implementation objectives with the matched control objectives
852
1096
  update_imp_objective(
853
1097
  leverage_auth_id=leveraged_auth_id,
854
1098
  existing_imp_obj=existing_objectives,
@@ -898,46 +1142,53 @@ def parse_crm_worksheet(file_path: click.Path, crm_sheet_name: str, version: Lit
898
1142
 
899
1143
  logger.debug(f"Skipping {skip_rows} rows in CRM worksheet")
900
1144
 
901
- # only use thse coloumns
902
- usecols = [
1145
+ # Define required columns
1146
+ required_columns = [
903
1147
  CONTROL_ID,
904
1148
  "Can Be Inherited from CSP",
905
1149
  "Specific Inheritance and Customer Agency/CSP Responsibilities",
906
1150
  ]
907
1151
 
908
1152
  try:
909
- # Verify that the columns are in the dataframe
910
- header_row = validator.data.iloc[skip_rows - 1 :].iloc[0]
1153
+ # Get the header row (which is at skip_rows - 1)
1154
+ header_row = validator.data.iloc[skip_rows - 1]
911
1155
 
912
- # Check if we have enough columns
913
- if len(header_row) < len(usecols):
914
- error_and_exit(
915
- f"Not enough columns found in CRM worksheet. Expected {len(usecols)} columns but found {len(header_row)}."
916
- )
1156
+ # Get data rows starting from skip_rows
1157
+ data = validator.data.iloc[skip_rows:].reset_index(drop=True)
917
1158
 
918
- # Verify each required column exists in the correct position
1159
+ # Set column names from header row
1160
+ data.columns = header_row
1161
+
1162
+ # Find required columns by name (case-insensitive, handle extra columns in AWS CIS/CRM)
1163
+ available_columns = data.columns.tolist()
1164
+ columns_to_use = []
919
1165
  missing_columns = []
920
- for i, expected_col in enumerate(usecols):
921
- if header_row.iloc[i] != expected_col:
922
- missing_columns.append(
923
- f"Expected '{expected_col}' at position {i + 1} but found '{header_row.iloc[i]}'"
924
- )
1166
+
1167
+ for required_col in required_columns:
1168
+ # Find column that matches (case-insensitive, strip whitespace)
1169
+ matching_col = next(
1170
+ (col for col in available_columns if str(col).strip().lower() == required_col.lower()), None
1171
+ )
1172
+ if matching_col is not None:
1173
+ columns_to_use.append(matching_col)
1174
+ else:
1175
+ missing_columns.append(required_col)
925
1176
 
926
1177
  if missing_columns:
927
- error_msg = "Required columns not found in the CRM worksheet:\n" + "\n".join(missing_columns)
1178
+ error_msg = (
1179
+ f"Required columns not found in the CRM worksheet: {', '.join(missing_columns)}\n"
1180
+ f"Available columns: {', '.join(str(col) for col in available_columns)}"
1181
+ )
928
1182
  error_and_exit(error_msg)
929
1183
 
930
- logger.debug("Verified all required columns exist in CRM worksheet")
931
-
932
- # Reindex the dataframe and skip some rows
933
- data = validator.data.iloc[skip_rows:]
1184
+ logger.debug(f"Found all required columns in CRM worksheet: {', '.join(required_columns)}")
934
1185
 
935
- # Keep only the first three columns
936
- data = data.iloc[:, :3]
1186
+ # Keep only the required columns
1187
+ data = data[columns_to_use]
937
1188
 
938
- # Rename the columns to match usecols
939
- data.columns = usecols
940
- logger.debug(f"Kept only required columns: {', '.join(usecols)}")
1189
+ # Rename the columns to standardize names
1190
+ data.columns = required_columns
1191
+ logger.debug(f"Using columns: {', '.join(required_columns)}")
941
1192
 
942
1193
  except KeyError as e:
943
1194
  error_and_exit(f"KeyError: {e} - One or more columns specified in usecols are not found in the dataframe.")
@@ -973,59 +1224,216 @@ def parse_crm_worksheet(file_path: click.Path, crm_sheet_name: str, version: Lit
973
1224
  return formatted_crm
974
1225
 
975
1226
 
976
- def parse_cis_worksheet(file_path: click.Path, cis_sheet_name: str) -> dict:
1227
+ def _get_expected_cis_columns() -> List[str]:
977
1228
  """
978
- Function to parse and format the CIS worksheet content
1229
+ Get the expected column names for CIS worksheet in order.
1230
+ These match the FedRAMP Rev 5 CIS worksheet format.
979
1231
 
980
- :param click.Path file_path: The file path to the FedRAMP CIS CRM workbook
981
- :param str cis_sheet_name: The name of the CIS sheet to parse
982
- :return: Formatted CIS content
983
- :rtype: dict
1232
+ :return: List of expected column names
1233
+ :rtype: List[str]
1234
+ """
1235
+ return [
1236
+ CONTROL_ID, # "Control ID"
1237
+ "Implemented",
1238
+ ControlImplementationStatus.PartiallyImplemented, # "Partially Implemented"
1239
+ "Planned",
1240
+ ALTERNATIVE_IMPLEMENTATION, # "Alternative Implementation"
1241
+ ControlImplementationStatus.NA, # "N/A"
1242
+ SERVICE_PROVIDER_CORPORATE,
1243
+ SERVICE_PROVIDER_SYSTEM_SPECIFIC,
1244
+ SERVICE_PROVIDER_HYBRID,
1245
+ CONFIGURED_BY_CUSTOMER,
1246
+ PROVIDED_BY_CUSTOMER,
1247
+ SHARED,
1248
+ INHERITED, # "Inherited from pre-existing FedRAMP Authorization"
1249
+ ]
1250
+
1251
+
1252
+ def _normalize_cis_columns(cis_df: "pd.DataFrame", expected_columns: List[str]) -> "pd.DataFrame":
1253
+ """
1254
+ Normalize CIS dataframe columns by matching expected columns and handling missing ones.
1255
+ Uses fuzzy matching to handle truncated column names from merged cells.
1256
+
1257
+ :param pd.DataFrame cis_df: The CIS dataframe
1258
+ :param List[str] expected_columns: List of expected column names
1259
+ :return: Normalized dataframe with standardized column names
1260
+ :rtype: pd.DataFrame
1261
+ """
1262
+ available_columns = cis_df.columns.tolist()
1263
+ columns_to_keep = []
1264
+
1265
+ logger.debug(f"Available CIS columns: {available_columns}")
1266
+
1267
+ for expected_col in expected_columns:
1268
+ matching_col = None
1269
+
1270
+ # Try exact match first (case-insensitive)
1271
+ matching_col = next(
1272
+ (col for col in available_columns if str(col).strip().lower() == expected_col.lower()), None
1273
+ )
1274
+
1275
+ # If no exact match, try partial/fuzzy match for truncated column names
1276
+ if matching_col is None:
1277
+ # Create a simplified version for matching (first few significant words)
1278
+ # Filter out common words and take first 3 significant words
1279
+ skip_words = {"from", "by", "to", "the", "and", "or", "a", "an"}
1280
+ expected_words = [w for w in expected_col.lower().split() if w not in skip_words][:3]
1281
+
1282
+ for col in available_columns:
1283
+ col_str = str(col).lower()
1284
+ # Check if at least 2 of the significant words are in the column name (handles truncation & variations)
1285
+ matches = sum(1 for word in expected_words if word in col_str)
1286
+ if matches >= min(2, len(expected_words)): # Need at least 2 matches, or all if less than 2 words
1287
+ matching_col = col
1288
+ logger.debug(
1289
+ f"Fuzzy matched '{expected_col}' to '{col}' (matched {matches}/{len(expected_words)} words)"
1290
+ )
1291
+ break
1292
+
1293
+ if matching_col is not None:
1294
+ columns_to_keep.append(matching_col)
1295
+ else:
1296
+ logger.info(f"Expected column '{expected_col}' not found in CIS worksheet. Using empty values.")
1297
+ cis_df[expected_col] = ""
1298
+ columns_to_keep.append(expected_col)
1299
+
1300
+ cis_df = cis_df[columns_to_keep]
1301
+ cis_df.columns = expected_columns
1302
+ return cis_df.fillna("")
1303
+
1304
+
1305
+ def _find_control_id_row_index(df: "pd.DataFrame") -> Optional[int]:
1306
+ """
1307
+ Find the row index containing 'Control ID' in the first column.
1308
+
1309
+ :param pd.DataFrame df: The dataframe to search
1310
+ :return: Row index if found, None otherwise
1311
+ :rtype: Optional[int]
1312
+ """
1313
+ for idx, row in df.iterrows():
1314
+ if row.iloc[0] == CONTROL_ID:
1315
+ return idx
1316
+ return None
1317
+
1318
+
1319
+ def _merge_header_rows(header_row, sub_header_row) -> List[str]:
1320
+ """
1321
+ Merge two header rows into a single list of column names.
1322
+
1323
+ FedRAMP Rev5 has a two-row header structure where main headers span multiple columns
1324
+ and sub-headers provide specific column names.
1325
+
1326
+ :param header_row: The main header row (categories)
1327
+ :param sub_header_row: The sub-header row (specific columns)
1328
+ :return: List of merged column names
1329
+ :rtype: List[str]
984
1330
  """
985
1331
  pd = get_pandas()
986
- logger.info("Parsing CIS worksheet...")
987
- skip_rows = 2
1332
+ merged_headers = []
1333
+ current_category = None
988
1334
 
989
- validator = ImportValidater(
990
- file_path=file_path,
991
- disable_mapping=True,
992
- required_headers=[],
993
- mapping_file_path=gettempdir(),
994
- prompt=False,
995
- ignore_unnamed=True,
996
- worksheet_name=cis_sheet_name,
997
- warn_extra_headers=False,
998
- )
999
- if validator.data.empty:
1000
- return {}
1335
+ for i, (main, sub) in enumerate(zip(header_row, sub_header_row)):
1336
+ # Update current category if main header has a value
1337
+ if pd.notna(main) and main and str(main).strip():
1338
+ current_category = str(main)
1001
1339
 
1002
- skip_rows = determine_skip_row(original_df=validator.data, text_to_find=CONTROL_ID, original_skip=skip_rows)
1340
+ # Determine which header value to use
1341
+ header_value = _select_header_value(pd, main, sub, current_category, i)
1342
+ merged_headers.append(header_value)
1343
+
1344
+ return merged_headers
1003
1345
 
1004
- # Parse the worksheet named 'CIS GovCloud U.S.+DoD (H)', skipping the initial rows
1005
- original_cis = validator.data
1006
1346
 
1007
- cis_df = original_cis.iloc[skip_rows:].reset_index(drop=True)
1347
+ def _select_header_value(pd: "pd.DataFrame", main, sub, current_category: Optional[str], index: int) -> str:
1348
+ """
1349
+ Select the appropriate header value based on priority: sub-header > main header > category > unnamed.
1008
1350
 
1009
- # Set the appropriate headers
1010
- cis_df.columns = cis_df.iloc[0]
1351
+ :param pd.DataFrame pd: The pandas dataframe
1352
+ :param main: Main header value
1353
+ :param sub: Sub-header value
1354
+ :param Optional[str] current_category: Current category from merged cells
1355
+ :param int index: Column index for fallback naming
1356
+ :return: Selected header value
1357
+ :rtype: str
1358
+ """
1359
+ if pd.notna(sub) and sub and str(sub).strip():
1360
+ return str(sub)
1361
+ if pd.notna(main) and main and str(main).strip():
1362
+ return str(main)
1363
+ if current_category:
1364
+ return f"{current_category}_{index}"
1365
+ return f"Unnamed_{index}"
1011
1366
 
1012
- # Drop any fully empty rows
1013
- cis_df.dropna(how="all", inplace=True)
1014
1367
 
1015
- # Reset the index
1368
+ def _load_and_prepare_cis_dataframe(file_path: click.Path, cis_sheet_name: str, skip_rows: int):
1369
+ """
1370
+ Load and prepare the CIS dataframe from the workbook.
1371
+
1372
+ :param click.Path file_path: The file path to the workbook
1373
+ :param str cis_sheet_name: The sheet name to parse
1374
+ :param int skip_rows: Number of rows to skip
1375
+ :return: Tuple of (prepared dataframe, updated skip_rows) or (None, skip_rows) if empty
1376
+ """
1377
+ # Read the Excel file directly with pandas to preserve "N/A" as string
1378
+ pd = get_pandas()
1379
+ df = pd.read_excel(file_path, sheet_name=cis_sheet_name, header=None, keep_default_na=False)
1380
+
1381
+ if df.empty:
1382
+ return None, skip_rows
1383
+
1384
+ # Find the row with "Control ID"
1385
+ control_id_row_idx = _find_control_id_row_index(df)
1386
+ if control_id_row_idx is None:
1387
+ logger.error("Could not find 'Control ID' in CIS worksheet")
1388
+ return None, skip_rows
1389
+
1390
+ # Extract and merge the two header rows
1391
+ header_row = df.iloc[control_id_row_idx]
1392
+ sub_header_row = df.iloc[control_id_row_idx + 1]
1393
+ merged_headers = _merge_header_rows(header_row, sub_header_row)
1394
+
1395
+ # Get data starting from two rows after the main header row
1396
+ cis_df = df.iloc[control_id_row_idx + 2 :].reset_index(drop=True)
1397
+ cis_df.columns = merged_headers
1398
+ cis_df.dropna(how="all", inplace=True)
1016
1399
  cis_df.reset_index(drop=True, inplace=True)
1017
1400
 
1018
- # Only keep the first 13 columns
1019
- cis_df = cis_df.iloc[:, :13]
1401
+ skip_rows = control_id_row_idx + 2
1020
1402
 
1021
- # Rename columns to standardize names
1022
- cis_df.columns = [
1023
- CONTROL_ID,
1403
+ return cis_df, skip_rows
1404
+
1405
+
1406
+ def _extract_status(data_row) -> str:
1407
+ """
1408
+ Extract the first non-empty implementation status from the CIS worksheet.
1409
+
1410
+ :param data_row: The data row to extract the status from
1411
+ :return: The implementation status
1412
+ :rtype: str
1413
+ """
1414
+ selected_status = []
1415
+ for col in [
1024
1416
  "Implemented",
1025
1417
  ControlImplementationStatus.PartiallyImplemented,
1026
1418
  "Planned",
1027
- ALT_IMPLEMENTATION,
1419
+ ALTERNATIVE_IMPLEMENTATION, # Use the correct constant
1028
1420
  ControlImplementationStatus.NA,
1421
+ ]:
1422
+ if data_row[col]:
1423
+ selected_status.append(col)
1424
+ return ", ".join(selected_status) if selected_status else ""
1425
+
1426
+
1427
+ def _extract_origination(data_row) -> str:
1428
+ """
1429
+ Extract the first non-empty control origination from the CIS worksheet.
1430
+
1431
+ :param data_row: The data row to extract the origination from
1432
+ :return: The control origination
1433
+ :rtype: str
1434
+ """
1435
+ selected_origination = []
1436
+ for col in [
1029
1437
  SERVICE_PROVIDER_CORPORATE,
1030
1438
  SERVICE_PROVIDER_SYSTEM_SPECIFIC,
1031
1439
  SERVICE_PROVIDER_HYBRID,
@@ -1033,75 +1441,53 @@ def parse_cis_worksheet(file_path: click.Path, cis_sheet_name: str) -> dict:
1033
1441
  PROVIDED_BY_CUSTOMER,
1034
1442
  SHARED,
1035
1443
  INHERITED,
1036
- ]
1444
+ ]:
1445
+ if data_row[col]:
1446
+ selected_origination.append(col)
1447
+ return ", ".join(selected_origination) if selected_origination else ""
1037
1448
 
1038
- # Fill NaN values with an empty string for processing
1039
- cis_df = cis_df.fillna("")
1040
-
1041
- # Function to extract the first non-empty implementation status
1042
- def _extract_status(data_row: pd.Series) -> str:
1043
- """
1044
- Function to extract the first non-empty implementation status from the CIS worksheet
1045
-
1046
- :param pd.Series data_row: The data row to extract the status from
1047
- :return: The implementation status
1048
- :rtype: str
1049
- """
1050
- selected_status = []
1051
- for col in [
1052
- "Implemented",
1053
- ControlImplementationStatus.PartiallyImplemented,
1054
- "Planned",
1055
- ALT_IMPLEMENTATION,
1056
- ControlImplementationStatus.NA,
1057
- ]:
1058
- if data_row[col]:
1059
- selected_status.append(col)
1060
- return ", ".join(selected_status) if selected_status else ""
1061
-
1062
- # Function to extract the first non-empty control origination
1063
- def _extract_origination(data_row: pd.Series) -> str:
1064
- """
1065
- Function to extract the first non-empty control origination from the CIS worksheet
1066
-
1067
- :param pd.Series data_row: The data row to extract the origination from
1068
- :return: The control origination
1069
- :rtype: str
1070
- """
1071
- selected_origination = []
1072
- for col in [
1073
- SERVICE_PROVIDER_CORPORATE,
1074
- SERVICE_PROVIDER_SYSTEM_SPECIFIC,
1075
- SERVICE_PROVIDER_HYBRID,
1076
- CONFIGURED_BY_CUSTOMER,
1077
- PROVIDED_BY_CUSTOMER,
1078
- SHARED,
1079
- INHERITED,
1080
- ]:
1081
- if data_row[col]:
1082
- selected_origination.append(col)
1083
- return ", ".join(selected_origination) if selected_origination else ""
1084
-
1085
- def _process_row(row: pd.Series) -> dict:
1086
- """
1087
- Function to process a row from the CIS worksheet
1088
-
1089
- :param pd.Series row: The row to process
1090
- :return: The processed row
1091
- :rtype: dict
1092
- """
1093
- return {
1094
- "control_id": row[CONTROL_ID],
1095
- "regscale_control_id": transform_control(row[CONTROL_ID]),
1096
- "implementation_status": _extract_status(row),
1097
- "control_origination": _extract_origination(row),
1098
- }
1099
1449
 
1100
- # use a threadexecutor to process the rows in parallel
1450
+ def _process_cis_row(row) -> dict:
1451
+ """
1452
+ Process a row from the CIS worksheet.
1453
+
1454
+ :param row: The row to process
1455
+ :return: The processed row
1456
+ :rtype: dict
1457
+ """
1458
+ return {
1459
+ "control_id": row[CONTROL_ID],
1460
+ "regscale_control_id": transform_control(row[CONTROL_ID]),
1461
+ "implementation_status": _extract_status(row),
1462
+ "control_origination": _extract_origination(row),
1463
+ }
1464
+
1465
+
1466
+ def parse_cis_worksheet(file_path: click.Path, cis_sheet_name: str) -> dict:
1467
+ """
1468
+ Function to parse and format the CIS worksheet content
1469
+
1470
+ :param click.Path file_path: The file path to the FedRAMP CIS CRM workbook
1471
+ :param str cis_sheet_name: The name of the CIS sheet to parse
1472
+ :return: Formatted CIS content
1473
+ :rtype: dict
1474
+ """
1475
+ logger.info("Parsing CIS worksheet...")
1476
+
1477
+ # Load and prepare the dataframe
1478
+ cis_df, _ = _load_and_prepare_cis_dataframe(file_path, cis_sheet_name, skip_rows=2)
1479
+ if cis_df is None:
1480
+ return {}
1481
+
1482
+ # Get expected columns and normalize the dataframe
1483
+ expected_columns = _get_expected_cis_columns()
1484
+ cis_df = _normalize_cis_columns(cis_df, expected_columns)
1485
+
1486
+ # Process rows in parallel
1101
1487
  with ThreadPoolExecutor() as executor:
1102
- results = list(executor.map(_process_row, [row for _, row in cis_df.iterrows()]))
1488
+ results = list(executor.map(_process_cis_row, [row for _, row in cis_df.iterrows()]))
1103
1489
 
1104
- # iterate the results and index by control_id
1490
+ # Index by control_id
1105
1491
  return {clean_key(result["control_id"]): result for result in results}
1106
1492
 
1107
1493
 
@@ -1325,7 +1711,7 @@ def extract_control_name(control_string: str) -> str:
1325
1711
  :return: The extracted control name
1326
1712
  :rtype: str
1327
1713
  """
1328
- pattern = r"^[A-Za-z]{2}-\d{1,3}(?:\(\d+\))?"
1714
+ pattern = r"^[A-Za-z]{2}-\d{1,3}(?:\s*\(\s*\d+\s*\))?"
1329
1715
  match = re.match(pattern, control_string.upper())
1330
1716
  return match.group() if match else ""
1331
1717
 
@@ -1338,8 +1724,8 @@ def rev_4_map(control_id: str) -> Optional[str]:
1338
1724
  :return: The mapped control ID or None if not found
1339
1725
  :rtype: Optional[str]
1340
1726
  """
1341
- # Regex pattern to match different control ID formats
1342
- pattern = r"^([A-Z]{2})-(\d{2})\s*(?:\((\d{2})\))?\s*(?:\(([a-z])\))?$"
1727
+ # Regex pattern to match different control ID formats - handles extra spaces like AC-6 ( 1 ) ( a )
1728
+ pattern = r"^([A-Z]{2})-(\d{2})\s*(?:\(\s*(\d{2})\s*\))?\s*(?:\(\s*([a-z])\s*\))?$"
1343
1729
 
1344
1730
  match = re.match(pattern, control_id, re.IGNORECASE)
1345
1731
 
@@ -1464,7 +1850,7 @@ def create_new_security_plan(profile_id: int, system_name: str):
1464
1850
 
1465
1851
  else:
1466
1852
  INITIAL_IMPORT = False
1467
- ret = next((plan for plan in existing_plan), None)
1853
+ ret = next(iter(existing_plan), None)
1468
1854
  logger.info(f"Found existing SSP# {ret.id}")
1469
1855
  create_backup_file(ret.id)
1470
1856
  existing_imps = ControlImplementation.get_list_by_plan(ret.id)
@@ -1549,7 +1935,7 @@ def copy_and_rename_file(file_path: Path, new_name: str) -> Path:
1549
1935
  """
1550
1936
  Copy and rename a file.
1551
1937
  """
1552
- temp_folder = Path(tempfile.gettempdir()) / "regscale"
1938
+ temp_folder = Path(gettempdir()) / "regscale"
1553
1939
  temp_folder.mkdir(exist_ok=True) # Ensure directory exists
1554
1940
 
1555
1941
  new_file_path = temp_folder / new_name
@@ -1610,10 +1996,9 @@ def parse_and_import_ciscrm(
1610
1996
 
1611
1997
  if "5" in version:
1612
1998
  version = "rev5"
1613
- part_mapper_rev5.load_fedramp_version_5_mapping()
1614
1999
  else:
1615
2000
  version = "rev4"
1616
- part_mapper_rev4.load_fedramp_version_4_mapping()
2001
+ # No longer loading JSON mappings - using smart algorithm only
1617
2002
  # parse the instructions worksheet to get the csp name, system name, and other data
1618
2003
  instructions_data = parse_instructions_worksheet(df=df, version=version) # type: ignore
1619
2004
 
@@ -1654,7 +2039,8 @@ def parse_and_import_ciscrm(
1654
2039
  cis_data = parse_cis_worksheet(file_path=file_path, cis_sheet_name=cis_sheet_name)
1655
2040
  crm_data = {}
1656
2041
  if crm_sheet_name:
1657
- crm_data = parse_crm_worksheet(file_path=file_path, crm_sheet_name=crm_sheet_name, version=version) # type: ignore
2042
+ # type: ignore
2043
+ crm_data = parse_crm_worksheet(file_path=file_path, crm_sheet_name=crm_sheet_name, version=version)
1658
2044
  if leveraged_auth_id == 0:
1659
2045
  auths = LeveragedAuthorization.get_all_by_parent(ssp.id)
1660
2046
  if auths: