regscale-cli 6.21.2.0__py3-none-any.whl → 6.28.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- regscale/_version.py +1 -1
- regscale/airflow/hierarchy.py +2 -2
- regscale/core/app/api.py +5 -2
- regscale/core/app/application.py +36 -6
- regscale/core/app/internal/control_editor.py +73 -21
- regscale/core/app/internal/evidence.py +727 -204
- regscale/core/app/internal/login.py +4 -2
- regscale/core/app/internal/model_editor.py +219 -64
- regscale/core/app/utils/app_utils.py +86 -12
- regscale/core/app/utils/catalog_utils/common.py +1 -1
- regscale/core/login.py +21 -4
- regscale/core/utils/async_graphql_client.py +363 -0
- regscale/core/utils/date.py +77 -1
- regscale/dev/cli.py +26 -0
- regscale/dev/code_gen.py +109 -24
- regscale/dev/version.py +72 -0
- regscale/integrations/commercial/__init__.py +30 -2
- regscale/integrations/commercial/aws/audit_manager_compliance.py +3908 -0
- regscale/integrations/commercial/aws/cli.py +3107 -54
- regscale/integrations/commercial/aws/cloudtrail_control_mappings.py +333 -0
- regscale/integrations/commercial/aws/cloudtrail_evidence.py +501 -0
- regscale/integrations/commercial/aws/cloudwatch_control_mappings.py +357 -0
- regscale/integrations/commercial/aws/cloudwatch_evidence.py +490 -0
- regscale/integrations/commercial/{amazon → aws}/common.py +71 -19
- regscale/integrations/commercial/aws/config_compliance.py +914 -0
- regscale/integrations/commercial/aws/conformance_pack_mappings.py +198 -0
- regscale/integrations/commercial/aws/control_compliance_analyzer.py +439 -0
- regscale/integrations/commercial/aws/evidence_generator.py +283 -0
- regscale/integrations/commercial/aws/guardduty_control_mappings.py +340 -0
- regscale/integrations/commercial/aws/guardduty_evidence.py +1053 -0
- regscale/integrations/commercial/aws/iam_control_mappings.py +368 -0
- regscale/integrations/commercial/aws/iam_evidence.py +574 -0
- regscale/integrations/commercial/aws/inventory/__init__.py +338 -22
- regscale/integrations/commercial/aws/inventory/base.py +107 -5
- regscale/integrations/commercial/aws/inventory/resources/analytics.py +390 -0
- regscale/integrations/commercial/aws/inventory/resources/applications.py +234 -0
- regscale/integrations/commercial/aws/inventory/resources/audit_manager.py +513 -0
- regscale/integrations/commercial/aws/inventory/resources/cloudtrail.py +315 -0
- regscale/integrations/commercial/aws/inventory/resources/cloudtrail_logs_metadata.py +476 -0
- regscale/integrations/commercial/aws/inventory/resources/cloudwatch.py +191 -0
- regscale/integrations/commercial/aws/inventory/resources/compute.py +328 -9
- regscale/integrations/commercial/aws/inventory/resources/config.py +464 -0
- regscale/integrations/commercial/aws/inventory/resources/containers.py +74 -9
- regscale/integrations/commercial/aws/inventory/resources/database.py +481 -31
- regscale/integrations/commercial/aws/inventory/resources/developer_tools.py +253 -0
- regscale/integrations/commercial/aws/inventory/resources/guardduty.py +286 -0
- regscale/integrations/commercial/aws/inventory/resources/iam.py +470 -0
- regscale/integrations/commercial/aws/inventory/resources/inspector.py +476 -0
- regscale/integrations/commercial/aws/inventory/resources/integration.py +175 -61
- regscale/integrations/commercial/aws/inventory/resources/kms.py +447 -0
- regscale/integrations/commercial/aws/inventory/resources/machine_learning.py +358 -0
- regscale/integrations/commercial/aws/inventory/resources/networking.py +390 -67
- regscale/integrations/commercial/aws/inventory/resources/s3.py +394 -0
- regscale/integrations/commercial/aws/inventory/resources/security.py +268 -72
- regscale/integrations/commercial/aws/inventory/resources/securityhub.py +473 -0
- regscale/integrations/commercial/aws/inventory/resources/storage.py +288 -29
- regscale/integrations/commercial/aws/inventory/resources/systems_manager.py +657 -0
- regscale/integrations/commercial/aws/inventory/resources/vpc.py +655 -0
- regscale/integrations/commercial/aws/kms_control_mappings.py +288 -0
- regscale/integrations/commercial/aws/kms_evidence.py +879 -0
- regscale/integrations/commercial/aws/ocsf/__init__.py +7 -0
- regscale/integrations/commercial/aws/ocsf/constants.py +115 -0
- regscale/integrations/commercial/aws/ocsf/mapper.py +435 -0
- regscale/integrations/commercial/aws/org_control_mappings.py +286 -0
- regscale/integrations/commercial/aws/org_evidence.py +666 -0
- regscale/integrations/commercial/aws/s3_control_mappings.py +356 -0
- regscale/integrations/commercial/aws/s3_evidence.py +632 -0
- regscale/integrations/commercial/aws/scanner.py +1072 -205
- regscale/integrations/commercial/aws/security_hub.py +319 -0
- regscale/integrations/commercial/aws/session_manager.py +282 -0
- regscale/integrations/commercial/aws/ssm_control_mappings.py +291 -0
- regscale/integrations/commercial/aws/ssm_evidence.py +492 -0
- regscale/integrations/commercial/jira.py +489 -153
- regscale/integrations/commercial/microsoft_defender/defender.py +326 -5
- regscale/integrations/commercial/microsoft_defender/defender_api.py +348 -14
- regscale/integrations/commercial/microsoft_defender/defender_constants.py +157 -0
- regscale/integrations/commercial/qualys/__init__.py +167 -68
- regscale/integrations/commercial/qualys/scanner.py +305 -39
- regscale/integrations/commercial/sarif/sairf_importer.py +432 -0
- regscale/integrations/commercial/sarif/sarif_converter.py +67 -0
- regscale/integrations/commercial/sicura/api.py +79 -42
- regscale/integrations/commercial/sicura/commands.py +8 -2
- regscale/integrations/commercial/sicura/scanner.py +83 -44
- regscale/integrations/commercial/stigv2/ckl_parser.py +5 -5
- regscale/integrations/commercial/synqly/assets.py +133 -16
- regscale/integrations/commercial/synqly/edr.py +2 -8
- regscale/integrations/commercial/synqly/query_builder.py +536 -0
- regscale/integrations/commercial/synqly/ticketing.py +27 -0
- regscale/integrations/commercial/synqly/vulnerabilities.py +165 -28
- regscale/integrations/commercial/tenablev2/cis_parsers.py +453 -0
- regscale/integrations/commercial/tenablev2/cis_scanner.py +447 -0
- regscale/integrations/commercial/tenablev2/commands.py +146 -5
- regscale/integrations/commercial/tenablev2/scanner.py +1 -3
- regscale/integrations/commercial/tenablev2/stig_parsers.py +113 -57
- regscale/integrations/commercial/wizv2/WizDataMixin.py +1 -1
- regscale/integrations/commercial/wizv2/click.py +191 -76
- regscale/integrations/commercial/wizv2/compliance/__init__.py +15 -0
- regscale/integrations/commercial/wizv2/{policy_compliance_helpers.py → compliance/helpers.py} +78 -60
- regscale/integrations/commercial/wizv2/compliance_report.py +1592 -0
- regscale/integrations/commercial/wizv2/core/__init__.py +133 -0
- regscale/integrations/commercial/wizv2/{async_client.py → core/client.py} +7 -3
- regscale/integrations/commercial/wizv2/{constants.py → core/constants.py} +92 -89
- regscale/integrations/commercial/wizv2/core/file_operations.py +237 -0
- regscale/integrations/commercial/wizv2/fetchers/__init__.py +11 -0
- regscale/integrations/commercial/wizv2/{data_fetcher.py → fetchers/policy_assessment.py} +66 -9
- regscale/integrations/commercial/wizv2/file_cleanup.py +104 -0
- regscale/integrations/commercial/wizv2/issue.py +776 -28
- regscale/integrations/commercial/wizv2/models/__init__.py +0 -0
- regscale/integrations/commercial/wizv2/parsers/__init__.py +34 -0
- regscale/integrations/commercial/wizv2/{parsers.py → parsers/main.py} +1 -1
- regscale/integrations/commercial/wizv2/processors/__init__.py +11 -0
- regscale/integrations/commercial/wizv2/{finding_processor.py → processors/finding.py} +1 -1
- regscale/integrations/commercial/wizv2/reports.py +243 -0
- regscale/integrations/commercial/wizv2/sbom.py +1 -1
- regscale/integrations/commercial/wizv2/scanner.py +1031 -441
- regscale/integrations/commercial/wizv2/utils/__init__.py +48 -0
- regscale/integrations/commercial/wizv2/{utils.py → utils/main.py} +116 -61
- regscale/integrations/commercial/wizv2/variables.py +89 -3
- regscale/integrations/compliance_integration.py +1036 -151
- regscale/integrations/control_matcher.py +432 -0
- regscale/integrations/due_date_handler.py +333 -0
- regscale/integrations/milestone_manager.py +291 -0
- regscale/integrations/public/__init__.py +14 -0
- regscale/integrations/public/cci_importer.py +834 -0
- regscale/integrations/public/csam/__init__.py +0 -0
- regscale/integrations/public/csam/csam.py +938 -0
- regscale/integrations/public/csam/csam_agency_defined.py +179 -0
- regscale/integrations/public/csam/csam_common.py +154 -0
- regscale/integrations/public/csam/csam_controls.py +432 -0
- regscale/integrations/public/csam/csam_poam.py +124 -0
- regscale/integrations/public/fedramp/click.py +77 -6
- regscale/integrations/public/fedramp/docx_parser.py +10 -1
- regscale/integrations/public/fedramp/fedramp_cis_crm.py +675 -289
- regscale/integrations/public/fedramp/fedramp_five.py +1 -1
- regscale/integrations/public/fedramp/poam/scanner.py +75 -7
- regscale/integrations/public/fedramp/poam_export_v5.py +888 -0
- regscale/integrations/scanner_integration.py +1961 -430
- regscale/models/integration_models/CCI_List.xml +1 -0
- regscale/models/integration_models/aqua.py +2 -2
- regscale/models/integration_models/cisa_kev_data.json +805 -11
- regscale/models/integration_models/flat_file_importer/__init__.py +5 -8
- regscale/models/integration_models/nexpose.py +36 -10
- regscale/models/integration_models/qualys.py +3 -4
- regscale/models/integration_models/synqly_models/capabilities.json +1 -1
- regscale/models/integration_models/synqly_models/connectors/vulnerabilities.py +87 -18
- regscale/models/integration_models/synqly_models/filter_parser.py +332 -0
- regscale/models/integration_models/synqly_models/ocsf_mapper.py +124 -25
- regscale/models/integration_models/synqly_models/synqly_model.py +89 -16
- regscale/models/locking.py +12 -8
- regscale/models/platform.py +4 -2
- regscale/models/regscale_models/__init__.py +7 -0
- regscale/models/regscale_models/assessment.py +2 -1
- regscale/models/regscale_models/catalog.py +1 -1
- regscale/models/regscale_models/compliance_settings.py +251 -1
- regscale/models/regscale_models/component.py +1 -0
- regscale/models/regscale_models/control_implementation.py +236 -41
- regscale/models/regscale_models/control_objective.py +74 -5
- regscale/models/regscale_models/file.py +2 -0
- regscale/models/regscale_models/form_field_value.py +5 -3
- regscale/models/regscale_models/inheritance.py +44 -0
- regscale/models/regscale_models/issue.py +301 -102
- regscale/models/regscale_models/milestone.py +33 -14
- regscale/models/regscale_models/organization.py +3 -0
- regscale/models/regscale_models/regscale_model.py +310 -73
- regscale/models/regscale_models/security_plan.py +4 -2
- regscale/models/regscale_models/vulnerability.py +3 -3
- regscale/regscale.py +25 -4
- regscale/templates/__init__.py +0 -0
- regscale/utils/threading/threadhandler.py +20 -15
- regscale/validation/record.py +23 -1
- {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/METADATA +17 -33
- {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/RECORD +310 -111
- tests/core/__init__.py +0 -0
- tests/core/utils/__init__.py +0 -0
- tests/core/utils/test_async_graphql_client.py +472 -0
- tests/fixtures/test_fixture.py +13 -8
- tests/regscale/core/test_login.py +171 -4
- tests/regscale/integrations/commercial/__init__.py +0 -0
- tests/regscale/integrations/commercial/aws/__init__.py +0 -0
- tests/regscale/integrations/commercial/aws/test_audit_manager_compliance.py +1304 -0
- tests/regscale/integrations/commercial/aws/test_audit_manager_evidence_aggregation.py +341 -0
- tests/regscale/integrations/commercial/aws/test_aws_analytics_collector.py +260 -0
- tests/regscale/integrations/commercial/aws/test_aws_applications_collector.py +242 -0
- tests/regscale/integrations/commercial/aws/test_aws_audit_manager_collector.py +1155 -0
- tests/regscale/integrations/commercial/aws/test_aws_cloudtrail_collector.py +534 -0
- tests/regscale/integrations/commercial/aws/test_aws_config_collector.py +400 -0
- tests/regscale/integrations/commercial/aws/test_aws_developer_tools_collector.py +203 -0
- tests/regscale/integrations/commercial/aws/test_aws_guardduty_collector.py +315 -0
- tests/regscale/integrations/commercial/aws/test_aws_iam_collector.py +458 -0
- tests/regscale/integrations/commercial/aws/test_aws_inspector_collector.py +353 -0
- tests/regscale/integrations/commercial/aws/test_aws_inventory_integration.py +530 -0
- tests/regscale/integrations/commercial/aws/test_aws_kms_collector.py +919 -0
- tests/regscale/integrations/commercial/aws/test_aws_machine_learning_collector.py +237 -0
- tests/regscale/integrations/commercial/aws/test_aws_s3_collector.py +722 -0
- tests/regscale/integrations/commercial/aws/test_aws_scanner_integration.py +722 -0
- tests/regscale/integrations/commercial/aws/test_aws_securityhub_collector.py +792 -0
- tests/regscale/integrations/commercial/aws/test_aws_systems_manager_collector.py +918 -0
- tests/regscale/integrations/commercial/aws/test_aws_vpc_collector.py +996 -0
- tests/regscale/integrations/commercial/aws/test_cli_evidence.py +431 -0
- tests/regscale/integrations/commercial/aws/test_cloudtrail_control_mappings.py +452 -0
- tests/regscale/integrations/commercial/aws/test_cloudtrail_evidence.py +788 -0
- tests/regscale/integrations/commercial/aws/test_config_compliance.py +298 -0
- tests/regscale/integrations/commercial/aws/test_conformance_pack_mappings.py +200 -0
- tests/regscale/integrations/commercial/aws/test_control_compliance_analyzer.py +375 -0
- tests/regscale/integrations/commercial/aws/test_datetime_parsing.py +223 -0
- tests/regscale/integrations/commercial/aws/test_evidence_generator.py +386 -0
- tests/regscale/integrations/commercial/aws/test_guardduty_control_mappings.py +564 -0
- tests/regscale/integrations/commercial/aws/test_guardduty_evidence.py +1041 -0
- tests/regscale/integrations/commercial/aws/test_iam_control_mappings.py +718 -0
- tests/regscale/integrations/commercial/aws/test_iam_evidence.py +1375 -0
- tests/regscale/integrations/commercial/aws/test_kms_control_mappings.py +656 -0
- tests/regscale/integrations/commercial/aws/test_kms_evidence.py +1163 -0
- tests/regscale/integrations/commercial/aws/test_ocsf_mapper.py +370 -0
- tests/regscale/integrations/commercial/aws/test_org_control_mappings.py +546 -0
- tests/regscale/integrations/commercial/aws/test_org_evidence.py +1240 -0
- tests/regscale/integrations/commercial/aws/test_s3_control_mappings.py +672 -0
- tests/regscale/integrations/commercial/aws/test_s3_evidence.py +987 -0
- tests/regscale/integrations/commercial/aws/test_scanner_evidence.py +373 -0
- tests/regscale/integrations/commercial/aws/test_security_hub_config_filtering.py +539 -0
- tests/regscale/integrations/commercial/aws/test_session_manager.py +516 -0
- tests/regscale/integrations/commercial/aws/test_ssm_control_mappings.py +588 -0
- tests/regscale/integrations/commercial/aws/test_ssm_evidence.py +735 -0
- tests/regscale/integrations/commercial/conftest.py +28 -0
- tests/regscale/integrations/commercial/microsoft_defender/__init__.py +1 -0
- tests/regscale/integrations/commercial/microsoft_defender/test_defender.py +1517 -0
- tests/regscale/integrations/commercial/microsoft_defender/test_defender_api.py +1748 -0
- tests/regscale/integrations/commercial/microsoft_defender/test_defender_constants.py +327 -0
- tests/regscale/integrations/commercial/microsoft_defender/test_defender_scanner.py +487 -0
- tests/regscale/integrations/commercial/test_aws.py +3742 -0
- tests/regscale/integrations/commercial/test_burp.py +48 -0
- tests/regscale/integrations/commercial/test_crowdstrike.py +49 -0
- tests/regscale/integrations/commercial/test_dependabot.py +341 -0
- tests/regscale/integrations/commercial/test_gcp.py +1543 -0
- tests/regscale/integrations/commercial/test_gitlab.py +549 -0
- tests/regscale/integrations/commercial/test_ip_mac_address_length.py +84 -0
- tests/regscale/integrations/commercial/test_jira.py +2204 -0
- tests/regscale/integrations/commercial/test_npm_audit.py +42 -0
- tests/regscale/integrations/commercial/test_okta.py +1228 -0
- tests/regscale/integrations/commercial/test_sarif_converter.py +251 -0
- tests/regscale/integrations/commercial/test_sicura.py +349 -0
- tests/regscale/integrations/commercial/test_snow.py +423 -0
- tests/regscale/integrations/commercial/test_sonarcloud.py +394 -0
- tests/regscale/integrations/commercial/test_sqlserver.py +186 -0
- tests/regscale/integrations/commercial/test_stig.py +33 -0
- tests/regscale/integrations/commercial/test_stig_mapper.py +153 -0
- tests/regscale/integrations/commercial/test_stigv2.py +406 -0
- tests/regscale/integrations/commercial/test_wiz.py +1365 -0
- tests/regscale/integrations/commercial/test_wiz_inventory.py +256 -0
- tests/regscale/integrations/commercial/wizv2/__init__.py +339 -0
- tests/regscale/integrations/commercial/wizv2/compliance/__init__.py +1 -0
- tests/regscale/integrations/commercial/wizv2/compliance/test_helpers.py +903 -0
- tests/regscale/integrations/commercial/wizv2/core/__init__.py +1 -0
- tests/regscale/integrations/commercial/wizv2/core/test_auth.py +701 -0
- tests/regscale/integrations/commercial/wizv2/core/test_client.py +1037 -0
- tests/regscale/integrations/commercial/wizv2/core/test_file_operations.py +989 -0
- tests/regscale/integrations/commercial/wizv2/fetchers/__init__.py +1 -0
- tests/regscale/integrations/commercial/wizv2/fetchers/test_policy_assessment.py +805 -0
- tests/regscale/integrations/commercial/wizv2/parsers/__init__.py +1 -0
- tests/regscale/integrations/commercial/wizv2/parsers/test_main.py +1153 -0
- tests/regscale/integrations/commercial/wizv2/processors/__init__.py +1 -0
- tests/regscale/integrations/commercial/wizv2/processors/test_finding.py +671 -0
- tests/regscale/integrations/commercial/wizv2/test_WizDataMixin.py +537 -0
- tests/regscale/integrations/commercial/wizv2/test_click_comprehensive.py +851 -0
- tests/regscale/integrations/commercial/wizv2/test_compliance_report_comprehensive.py +910 -0
- tests/regscale/integrations/commercial/wizv2/test_compliance_report_normalization.py +138 -0
- tests/regscale/integrations/commercial/wizv2/test_file_cleanup.py +283 -0
- tests/regscale/integrations/commercial/wizv2/test_file_operations.py +260 -0
- tests/regscale/integrations/commercial/wizv2/test_issue.py +343 -0
- tests/regscale/integrations/commercial/wizv2/test_issue_comprehensive.py +1203 -0
- tests/regscale/integrations/commercial/wizv2/test_reports.py +497 -0
- tests/regscale/integrations/commercial/wizv2/test_sbom.py +643 -0
- tests/regscale/integrations/commercial/wizv2/test_scanner_comprehensive.py +805 -0
- tests/regscale/integrations/commercial/wizv2/test_wiz_click_client_id.py +165 -0
- tests/regscale/integrations/commercial/wizv2/test_wiz_compliance_report.py +1394 -0
- tests/regscale/integrations/commercial/wizv2/test_wiz_compliance_unit.py +341 -0
- tests/regscale/integrations/commercial/wizv2/test_wiz_control_normalization.py +138 -0
- tests/regscale/integrations/commercial/wizv2/test_wiz_findings_comprehensive.py +364 -0
- tests/regscale/integrations/commercial/wizv2/test_wiz_inventory_comprehensive.py +644 -0
- tests/regscale/integrations/commercial/wizv2/test_wiz_status_mapping.py +149 -0
- tests/regscale/integrations/commercial/wizv2/test_wizv2.py +1218 -0
- tests/regscale/integrations/commercial/wizv2/test_wizv2_utils.py +519 -0
- tests/regscale/integrations/commercial/wizv2/utils/__init__.py +1 -0
- tests/regscale/integrations/commercial/wizv2/utils/test_main.py +1523 -0
- tests/regscale/integrations/public/__init__.py +0 -0
- tests/regscale/integrations/public/fedramp/__init__.py +1 -0
- tests/regscale/integrations/public/fedramp/test_gen_asset_list.py +150 -0
- tests/regscale/integrations/public/fedramp/test_poam_export_v5.py +1293 -0
- tests/regscale/integrations/public/test_alienvault.py +220 -0
- tests/regscale/integrations/public/test_cci.py +1053 -0
- tests/regscale/integrations/public/test_cisa.py +1021 -0
- tests/regscale/integrations/public/test_emass.py +518 -0
- tests/regscale/integrations/public/test_fedramp.py +1152 -0
- tests/regscale/integrations/public/test_fedramp_cis_crm.py +3661 -0
- tests/regscale/integrations/public/test_file_uploads.py +506 -0
- tests/regscale/integrations/public/test_oscal.py +453 -0
- tests/regscale/integrations/test_compliance_status_mapping.py +406 -0
- tests/regscale/integrations/test_control_matcher.py +1421 -0
- tests/regscale/integrations/test_control_matching.py +155 -0
- tests/regscale/integrations/test_milestone_manager.py +408 -0
- tests/regscale/models/test_control_implementation.py +118 -3
- tests/regscale/models/test_form_field_value_integration.py +304 -0
- tests/regscale/models/test_issue.py +378 -1
- tests/regscale/models/test_module_integration.py +582 -0
- tests/regscale/models/test_tenable_integrations.py +811 -105
- regscale/integrations/commercial/wizv2/policy_compliance.py +0 -3057
- regscale/integrations/public/fedramp/mappings/fedramp_r4_parts.json +0 -7388
- regscale/integrations/public/fedramp/mappings/fedramp_r5_parts.json +0 -9605
- regscale/integrations/public/fedramp/parts_mapper.py +0 -107
- /regscale/integrations/commercial/{amazon → sarif}/__init__.py +0 -0
- /regscale/integrations/commercial/wizv2/{wiz_auth.py → core/auth.py} +0 -0
- {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/LICENSE +0 -0
- {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/WHEEL +0 -0
- {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/entry_points.txt +0 -0
- {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/top_level.txt +0 -0
|
@@ -8,12 +8,13 @@ import json
|
|
|
8
8
|
import math
|
|
9
9
|
import re
|
|
10
10
|
import shutil
|
|
11
|
-
import tempfile
|
|
12
11
|
from collections import Counter
|
|
13
12
|
from concurrent.futures import as_completed
|
|
14
13
|
from concurrent.futures.thread import ThreadPoolExecutor
|
|
15
14
|
from datetime import datetime
|
|
15
|
+
from functools import lru_cache
|
|
16
16
|
from pathlib import Path
|
|
17
|
+
from tempfile import gettempdir
|
|
17
18
|
from threading import Thread
|
|
18
19
|
from types import ModuleType
|
|
19
20
|
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, Optional, Tuple, TypeVar
|
|
@@ -24,7 +25,7 @@ from regscale.core.app.api import Api
|
|
|
24
25
|
from regscale.core.app.utils.api_handler import APIInsertionError, APIUpdateError
|
|
25
26
|
from regscale.core.app.utils.app_utils import compute_hash, create_progress_object, error_and_exit, get_current_datetime
|
|
26
27
|
from regscale.core.utils.graphql import GraphQLQuery
|
|
27
|
-
from regscale.integrations.
|
|
28
|
+
from regscale.integrations.control_matcher import ControlMatcher
|
|
28
29
|
from regscale.integrations.public.fedramp.ssp_logger import SSPLogger
|
|
29
30
|
from regscale.models import ControlObjective, ImplementationObjective, ImportValidater, Parameter, Profile
|
|
30
31
|
from regscale.models.regscale_models import (
|
|
@@ -43,14 +44,9 @@ from regscale.utils.version import RegscaleVersion
|
|
|
43
44
|
if TYPE_CHECKING:
|
|
44
45
|
import pandas as pd
|
|
45
46
|
|
|
46
|
-
from functools import lru_cache
|
|
47
|
-
from tempfile import gettempdir
|
|
48
|
-
|
|
49
47
|
T = TypeVar("T")
|
|
50
48
|
|
|
51
49
|
logger = SSPLogger()
|
|
52
|
-
part_mapper_rev5 = PartMapper()
|
|
53
|
-
part_mapper_rev4 = PartMapper()
|
|
54
50
|
progress = create_progress_object()
|
|
55
51
|
|
|
56
52
|
SERVICE_PROVIDER_CORPORATE = "Service Provider Corporate"
|
|
@@ -120,6 +116,112 @@ def get_pandas() -> ModuleType:
|
|
|
120
116
|
return pd
|
|
121
117
|
|
|
122
118
|
|
|
119
|
+
def _build_potential_oscal_ids(variation: str) -> List[str]:
|
|
120
|
+
"""
|
|
121
|
+
Build potential OSCAL ID formats from a control ID variation.
|
|
122
|
+
|
|
123
|
+
:param str variation: Control ID variation (e.g., "AC-1", "AC-01", "AC-1.a")
|
|
124
|
+
:return: List of potential OSCAL IDs
|
|
125
|
+
:rtype: List[str]
|
|
126
|
+
"""
|
|
127
|
+
variation_lower = variation.lower()
|
|
128
|
+
oscal_ids = []
|
|
129
|
+
|
|
130
|
+
# Check if this is a control with a letter part (e.g., "ac-1.a")
|
|
131
|
+
if re.match(r"^[a-z]+-\d+\.[a-z]$", variation_lower):
|
|
132
|
+
# For letter parts, map to OSCAL format: ac-1.a -> ac-1_smt.a
|
|
133
|
+
base_control = variation_lower.rsplit(".", 1)[0] # Get "ac-1" from "ac-1.a"
|
|
134
|
+
letter_part = variation_lower.rsplit(".", 1)[1] # Get "a" from "ac-1.a"
|
|
135
|
+
oscal_ids.extend(
|
|
136
|
+
[
|
|
137
|
+
f"{base_control}_smt.{letter_part}", # ac-1_smt.a (primary format)
|
|
138
|
+
f"{variation_lower}_smt", # ac-1.a_smt (alternative format)
|
|
139
|
+
]
|
|
140
|
+
)
|
|
141
|
+
else:
|
|
142
|
+
# Base control without letter part - include all potential letter variations
|
|
143
|
+
oscal_ids.extend(
|
|
144
|
+
[
|
|
145
|
+
f"{variation_lower}_smt",
|
|
146
|
+
f"{variation_lower}_smt.a",
|
|
147
|
+
f"{variation_lower}_smt.b",
|
|
148
|
+
f"{variation_lower}_smt.c",
|
|
149
|
+
]
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
return oscal_ids
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _matches_oscal_id(obj_id: str, variation: str) -> bool:
|
|
156
|
+
"""
|
|
157
|
+
Check if an objective's otherId matches any OSCAL ID format for the given variation.
|
|
158
|
+
|
|
159
|
+
:param str obj_id: The objective's otherId
|
|
160
|
+
:param str variation: Control ID variation
|
|
161
|
+
:return: True if matches, False otherwise
|
|
162
|
+
:rtype: bool
|
|
163
|
+
"""
|
|
164
|
+
potential_ids = _build_potential_oscal_ids(variation)
|
|
165
|
+
return obj_id in potential_ids or obj_id.startswith(f"{variation.lower()}_smt")
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _find_matching_objectives(control_objectives: List[ControlObjective], variations: set) -> List[ControlObjective]:
|
|
169
|
+
"""
|
|
170
|
+
Find objectives that match any of the control ID variations.
|
|
171
|
+
|
|
172
|
+
:param List[ControlObjective] control_objectives: List of objectives to search
|
|
173
|
+
:param set variations: Set of control ID variations
|
|
174
|
+
:return: List of matched objectives
|
|
175
|
+
:rtype: List[ControlObjective]
|
|
176
|
+
"""
|
|
177
|
+
matched_objectives = []
|
|
178
|
+
|
|
179
|
+
for obj in control_objectives:
|
|
180
|
+
if not hasattr(obj, "otherId") or not obj.otherId:
|
|
181
|
+
continue
|
|
182
|
+
|
|
183
|
+
obj_id = obj.otherId
|
|
184
|
+
for variation in variations:
|
|
185
|
+
if _matches_oscal_id(obj_id, variation):
|
|
186
|
+
if obj not in matched_objectives:
|
|
187
|
+
matched_objectives.append(obj)
|
|
188
|
+
break
|
|
189
|
+
|
|
190
|
+
return matched_objectives
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def find_objectives_using_control_matcher(
|
|
194
|
+
source: str, control_objectives: List[ControlObjective], control_matcher: ControlMatcher
|
|
195
|
+
) -> Tuple[List[ControlObjective], str]:
|
|
196
|
+
"""
|
|
197
|
+
Find control objectives using ControlMatcher for consistent control ID parsing and matching.
|
|
198
|
+
|
|
199
|
+
:param str source: The source control ID (e.g., "AC-1(a)", "AC-01 (a)")
|
|
200
|
+
:param List[ControlObjective] control_objectives: List of ControlObjective objects to search
|
|
201
|
+
:param ControlMatcher control_matcher: Instance of ControlMatcher for parsing and variations
|
|
202
|
+
:return: Tuple of (matched objectives list, status_message)
|
|
203
|
+
:rtype: Tuple[List[ControlObjective], str]
|
|
204
|
+
"""
|
|
205
|
+
# Parse the control ID using ControlMatcher
|
|
206
|
+
parsed_id = control_matcher.parse_control_id(source)
|
|
207
|
+
if not parsed_id:
|
|
208
|
+
return [], f"Unable to parse control {source}"
|
|
209
|
+
|
|
210
|
+
# Get all variations of this control ID
|
|
211
|
+
# pylint: disable=protected-access # Using internal method for control ID variation matching
|
|
212
|
+
variations = control_matcher._get_control_id_variations(parsed_id)
|
|
213
|
+
if not variations:
|
|
214
|
+
return [], f"Unable to generate variations for {source}"
|
|
215
|
+
|
|
216
|
+
# Find matching objectives
|
|
217
|
+
matched_objectives = _find_matching_objectives(control_objectives, variations)
|
|
218
|
+
|
|
219
|
+
if matched_objectives:
|
|
220
|
+
return matched_objectives, f"Found {len(matched_objectives)} objective(s) for {source}"
|
|
221
|
+
|
|
222
|
+
return [], f"No database match found for {source} (parsed: {parsed_id})"
|
|
223
|
+
|
|
224
|
+
|
|
123
225
|
def transform_control(control: str) -> str:
|
|
124
226
|
"""
|
|
125
227
|
Function to parse the control string and transform it to the RegScale format
|
|
@@ -129,16 +231,15 @@ def transform_control(control: str) -> str:
|
|
|
129
231
|
:return: Transformed control ID to match RegScale control ID format
|
|
130
232
|
:rtype: str
|
|
131
233
|
"""
|
|
132
|
-
# Use regex to match the pattern and capture the parts
|
|
133
|
-
|
|
134
|
-
if match:
|
|
234
|
+
# Use regex to match the pattern and capture the parts (handle extra spaces)
|
|
235
|
+
# Now handles both uppercase and lowercase letters in parentheses
|
|
236
|
+
if match := re.match(r"([A-Z]+)-(\d+)\s*\(\s*(\d+|[A-Z])\s*\)", control, re.IGNORECASE):
|
|
135
237
|
control_name = match.group(1).lower()
|
|
136
238
|
control_number = match.group(2)
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
if sub_control.isdigit():
|
|
239
|
+
try:
|
|
240
|
+
sub_control = match.group(3).lower() # Normalize to lowercase
|
|
140
241
|
transformed_control = f"{control_name}-{control_number}.{sub_control}"
|
|
141
|
-
|
|
242
|
+
except IndexError:
|
|
142
243
|
transformed_control = f"{control_name}-{control_number}"
|
|
143
244
|
|
|
144
245
|
return transformed_control
|
|
@@ -179,29 +280,76 @@ def new_leveraged_auth(
|
|
|
179
280
|
return new_leveraged_auth_id.id
|
|
180
281
|
|
|
181
282
|
|
|
182
|
-
def gen_key(control_id: str):
|
|
283
|
+
def gen_key(control_id: str) -> str:
|
|
183
284
|
"""
|
|
184
|
-
Function to generate a key for the control ID
|
|
285
|
+
Function to generate a key for the control ID by stripping letter-based parts.
|
|
286
|
+
Handles both parentheses notation (AC-1(a)) and dot notation (ac-1.a).
|
|
287
|
+
|
|
288
|
+
Examples:
|
|
289
|
+
- AC-1 (a) -> AC-1
|
|
290
|
+
- ac-1.a -> ac-1
|
|
291
|
+
- AC-2(1) -> AC-2(1) (numeric enhancement preserved)
|
|
292
|
+
- AC-17.2 -> AC-17.2 (numeric enhancement preserved)
|
|
185
293
|
|
|
186
294
|
:param str control_id: The control ID to generate a key for
|
|
187
|
-
:return: The generated key
|
|
295
|
+
:return: The generated key with letter parts stripped
|
|
188
296
|
:rtype: str
|
|
189
297
|
"""
|
|
190
|
-
#
|
|
191
|
-
#
|
|
298
|
+
# First, try parentheses notation: ALPHA-NUM(LETTER) -> ALPHA-NUM
|
|
299
|
+
# Captures everything up to either:
|
|
300
|
+
# 1. The last (number) if it exists (preserved)
|
|
192
301
|
# 2. The main control number if no enhancement exists
|
|
193
|
-
#
|
|
194
|
-
|
|
302
|
+
# Excludes trailing (letter) - handles extra spaces like AC-6 ( 1 ) ( a )
|
|
303
|
+
pattern_paren = r"^(\w+-\d+(?:\s*\(\s*\d+\s*\))?)(?:\s*\(\s*[a-zA-Z]\s*\))?$"
|
|
304
|
+
if match := re.match(pattern_paren, control_id):
|
|
305
|
+
return match.group(1)
|
|
195
306
|
|
|
196
|
-
|
|
197
|
-
|
|
307
|
+
# Try dot notation: alpha-num.letter -> alpha-num
|
|
308
|
+
# Preserves numeric enhancements (ac-17.2) but strips letter parts (ac-1.a)
|
|
309
|
+
pattern_dot = r"^([a-z]+-\d+)\.([a-z])$"
|
|
310
|
+
if match := re.match(pattern_dot, control_id, re.IGNORECASE):
|
|
311
|
+
# Check if the part after dot is a single letter (not a number)
|
|
198
312
|
return match.group(1)
|
|
313
|
+
|
|
314
|
+
# No match, return as-is
|
|
199
315
|
return control_id
|
|
200
316
|
|
|
201
317
|
|
|
318
|
+
def _is_letter_based_control_part(control_id: str) -> bool:
|
|
319
|
+
"""
|
|
320
|
+
Check if a control ID is a letter-based part (e.g., AC-1(a), ac-1.a).
|
|
321
|
+
Returns True for ALPHA-NUMERIC(ALPHA) or alpha-numeric.alpha patterns.
|
|
322
|
+
Returns False for numeric enhancements (AC-1(1), ac-17.2).
|
|
323
|
+
|
|
324
|
+
:param str control_id: The control ID to check
|
|
325
|
+
:return: True if it's a letter-based control part
|
|
326
|
+
:rtype: bool
|
|
327
|
+
"""
|
|
328
|
+
# Pattern 1: Parentheses notation - ALPHA-NUMERIC(ALPHA) like AC-1(a), AC-2(B)
|
|
329
|
+
pattern_paren = r"^[A-Za-z]+-\d+\s*\(\s*[a-zA-Z]\s*\)$"
|
|
330
|
+
if re.match(pattern_paren, control_id):
|
|
331
|
+
return True
|
|
332
|
+
|
|
333
|
+
# Pattern 2: Dot notation - alpha-numeric.alpha like ac-1.a, ac-2.b
|
|
334
|
+
# Exclude numeric enhancements like ac-17.2
|
|
335
|
+
pattern_dot = r"^[a-z]+-\d+\.([a-z])$"
|
|
336
|
+
match = re.match(pattern_dot, control_id, re.IGNORECASE)
|
|
337
|
+
if match and match.group(1).isalpha():
|
|
338
|
+
return True
|
|
339
|
+
|
|
340
|
+
return False
|
|
341
|
+
|
|
342
|
+
|
|
202
343
|
def map_implementation_status(control_id: str, cis_data: dict) -> str:
|
|
203
344
|
"""
|
|
204
|
-
Function to map the selected implementation status on the CIS worksheet to a RegScale status
|
|
345
|
+
Function to map the selected implementation status on the CIS worksheet to a RegScale status.
|
|
346
|
+
Aggregates letter-based control parts (AC-1(a), AC-1(b), AC-1(c)) into base control (AC-1).
|
|
347
|
+
|
|
348
|
+
Aggregation logic for letter-based parts:
|
|
349
|
+
- All "Implemented" → "Fully Implemented"
|
|
350
|
+
- Mix with at least one "Implemented" → "Partially Implemented"
|
|
351
|
+
- All "Not Implemented" or empty → "Not Implemented"
|
|
352
|
+
- Any "Planned" (no implemented) → "Planned"
|
|
205
353
|
|
|
206
354
|
:param str control_id: The control ID from RegScale
|
|
207
355
|
:param dict cis_data: Data from the CIS worksheet to map the status from
|
|
@@ -209,7 +357,7 @@ def map_implementation_status(control_id: str, cis_data: dict) -> str:
|
|
|
209
357
|
:rtype: str
|
|
210
358
|
"""
|
|
211
359
|
|
|
212
|
-
# Extract matching records
|
|
360
|
+
# Extract matching records (gen_key strips letter parts to match base control)
|
|
213
361
|
cis_records = [
|
|
214
362
|
value
|
|
215
363
|
for value in cis_data.values()
|
|
@@ -221,31 +369,49 @@ def map_implementation_status(control_id: str, cis_data: dict) -> str:
|
|
|
221
369
|
logger.debug("Found %d CIS records for control %s", len(cis_records), control_id)
|
|
222
370
|
|
|
223
371
|
if not cis_records:
|
|
372
|
+
# Alerts if a control exists in regscale but is missing from CIS worksheet
|
|
224
373
|
logger.warning(f"No CIS records found for control {control_id}")
|
|
225
374
|
return status_ret
|
|
226
375
|
|
|
376
|
+
# Check if these are letter-based control parts that need aggregation
|
|
377
|
+
has_letter_parts = any(_is_letter_based_control_part(rec.get("control_id", "")) for rec in cis_records)
|
|
378
|
+
|
|
227
379
|
# Count implementation statuses
|
|
228
380
|
status_counts = Counter(record.get("implementation_status", "") for record in cis_records)
|
|
229
|
-
logger.debug("Status distribution for %s: %s", control_id, dict(status_counts))
|
|
381
|
+
logger.debug("Status distribution for %s: %s (letter parts: %s)", control_id, dict(status_counts), has_letter_parts)
|
|
230
382
|
|
|
231
|
-
# Early
|
|
383
|
+
# Early return for simple case: all same status
|
|
232
384
|
if len(status_counts) == 1:
|
|
233
385
|
status = next(iter(status_counts))
|
|
234
|
-
|
|
386
|
+
mapped_status = STATUS_MAPPING.get(status, ControlImplementationStatus.NotImplemented)
|
|
387
|
+
# If all letter parts have same status and it's "Implemented", return FullyImplemented
|
|
388
|
+
if has_letter_parts and status == "Implemented":
|
|
389
|
+
return ControlImplementationStatus.FullyImplemented
|
|
390
|
+
return mapped_status
|
|
235
391
|
|
|
392
|
+
# Aggregate statuses for letter-based control parts or multiple records
|
|
236
393
|
implemented_count = status_counts.get("Implemented", 0)
|
|
394
|
+
not_implemented_count = status_counts.get("", 0) # Empty status counts as not implemented
|
|
395
|
+
partially_implemented_count = status_counts.get("Partially Implemented", 0)
|
|
396
|
+
planned_count = status_counts.get("Planned", 0)
|
|
237
397
|
total_count = sum(status_counts.values())
|
|
238
398
|
|
|
399
|
+
# Aggregation logic
|
|
239
400
|
if implemented_count == total_count:
|
|
401
|
+
# All parts are implemented
|
|
240
402
|
return ControlImplementationStatus.FullyImplemented
|
|
241
|
-
elif implemented_count > 0 or
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
403
|
+
elif implemented_count > 0 or partially_implemented_count > 0:
|
|
404
|
+
# Mix of implemented and other statuses, or any partially implemented
|
|
405
|
+
return ControlImplementationStatus.PartiallyImplemented
|
|
406
|
+
elif planned_count > 0 and not_implemented_count == 0:
|
|
407
|
+
# All are planned (no not-implemented)
|
|
408
|
+
return ControlImplementationStatus.Planned
|
|
245
409
|
elif any(status in ["N/A", ALTERNATIVE_IMPLEMENTATION] for status in status_counts):
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
410
|
+
# Any N/A or Alternative
|
|
411
|
+
return ControlImplementationStatus.NA
|
|
412
|
+
else:
|
|
413
|
+
# Default: not implemented
|
|
414
|
+
return ControlImplementationStatus.NotImplemented
|
|
249
415
|
|
|
250
416
|
|
|
251
417
|
def map_origination(control_id: str, cis_data: dict) -> dict:
|
|
@@ -268,7 +434,7 @@ def map_origination(control_id: str, cis_data: dict) -> dict:
|
|
|
268
434
|
}
|
|
269
435
|
|
|
270
436
|
# Initialize result with all flags set to False
|
|
271
|
-
result =
|
|
437
|
+
result = dict.fromkeys(origination_mapping.values(), False)
|
|
272
438
|
result["record_text"] = ""
|
|
273
439
|
|
|
274
440
|
# Find matching CIS records
|
|
@@ -345,6 +511,103 @@ def get_multi_status(record: dict) -> str:
|
|
|
345
511
|
return status_map.get(implementation_status, NOT_IMPLEMENTED)
|
|
346
512
|
|
|
347
513
|
|
|
514
|
+
def _calculate_responsibility(control_originations: List[str], imp: ControlImplementation) -> str:
|
|
515
|
+
"""
|
|
516
|
+
Calculate responsibility from control originations.
|
|
517
|
+
|
|
518
|
+
:param List[str] control_originations: List of control origination values
|
|
519
|
+
:param ControlImplementation imp: Control implementation
|
|
520
|
+
:return: Calculated responsibility value
|
|
521
|
+
:rtype: str
|
|
522
|
+
"""
|
|
523
|
+
try:
|
|
524
|
+
if RegscaleVersion.meets_minimum_version("6.20.17.0"):
|
|
525
|
+
return ",".join(control_originations)
|
|
526
|
+
return next(iter(control_originations))
|
|
527
|
+
except StopIteration:
|
|
528
|
+
if imp.responsibility:
|
|
529
|
+
return imp.responsibility.split(",")[0]
|
|
530
|
+
return SERVICE_PROVIDER_CORPORATE
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def _create_new_implementation_objective(
|
|
534
|
+
leverage_auth_id: int,
|
|
535
|
+
imp: ControlImplementation,
|
|
536
|
+
objective: ControlObjective,
|
|
537
|
+
cis_record: dict,
|
|
538
|
+
responsibility: str,
|
|
539
|
+
cloud_responsibility: str,
|
|
540
|
+
customer_responsibility: str,
|
|
541
|
+
can_be_inherited_from_csp: str,
|
|
542
|
+
) -> ImplementationObjective:
|
|
543
|
+
"""
|
|
544
|
+
Create a new implementation objective.
|
|
545
|
+
|
|
546
|
+
:param int leverage_auth_id: Leveraged authorization ID
|
|
547
|
+
:param ControlImplementation imp: Control implementation
|
|
548
|
+
:param ControlObjective objective: Control objective
|
|
549
|
+
:param dict cis_record: CIS record data
|
|
550
|
+
:param str responsibility: Responsibility value
|
|
551
|
+
:param str cloud_responsibility: Cloud responsibility value
|
|
552
|
+
:param str customer_responsibility: Customer responsibility value
|
|
553
|
+
:param str can_be_inherited_from_csp: Can be inherited flag
|
|
554
|
+
:return: New implementation objective
|
|
555
|
+
:rtype: ImplementationObjective
|
|
556
|
+
"""
|
|
557
|
+
imp_obj = ImplementationObjective(
|
|
558
|
+
id=0,
|
|
559
|
+
uuid="",
|
|
560
|
+
inherited=can_be_inherited_from_csp in ["Yes", "Partial"],
|
|
561
|
+
implementationId=imp.id,
|
|
562
|
+
status=get_multi_status(cis_record),
|
|
563
|
+
objectiveId=objective.id,
|
|
564
|
+
notes=objective.name,
|
|
565
|
+
securityControlId=objective.securityControlId,
|
|
566
|
+
securityPlanId=REGSCALE_SSP_ID,
|
|
567
|
+
responsibility=responsibility,
|
|
568
|
+
cloudResponsibility=cloud_responsibility,
|
|
569
|
+
customerResponsibility=customer_responsibility,
|
|
570
|
+
authorizationId=leverage_auth_id,
|
|
571
|
+
parentObjectiveId=objective.parentObjectiveId,
|
|
572
|
+
)
|
|
573
|
+
logger.debug(
|
|
574
|
+
"Creating new Implementation Objective for Control %s with status: %s responsibility: %s",
|
|
575
|
+
imp_obj.securityControlId,
|
|
576
|
+
imp_obj.status,
|
|
577
|
+
imp_obj.responsibility,
|
|
578
|
+
)
|
|
579
|
+
return imp_obj
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def _update_existing_implementation_objective(
|
|
583
|
+
ex_obj: ImplementationObjective,
|
|
584
|
+
cis_record: dict,
|
|
585
|
+
responsibility: str,
|
|
586
|
+
cloud_responsibility: str,
|
|
587
|
+
customer_responsibility: str,
|
|
588
|
+
) -> None:
|
|
589
|
+
"""
|
|
590
|
+
Update an existing implementation objective.
|
|
591
|
+
|
|
592
|
+
:param ImplementationObjective ex_obj: Existing implementation objective
|
|
593
|
+
:param dict cis_record: CIS record data
|
|
594
|
+
:param str responsibility: Responsibility value
|
|
595
|
+
:param str cloud_responsibility: Cloud responsibility value
|
|
596
|
+
:param str customer_responsibility: Customer responsibility value
|
|
597
|
+
:rtype: None
|
|
598
|
+
"""
|
|
599
|
+
ex_obj.status = get_multi_status(cis_record)
|
|
600
|
+
ex_obj.responsibility = responsibility
|
|
601
|
+
if cloud_responsibility.strip():
|
|
602
|
+
logger.debug(f"Updating Implementation Objective #{ex_obj.id} with responsibility: {responsibility}")
|
|
603
|
+
ex_obj.cloudResponsibility = cloud_responsibility
|
|
604
|
+
if customer_responsibility.strip():
|
|
605
|
+
logger.debug(
|
|
606
|
+
f"Updating Implementation Objective #{ex_obj.id} with cloud responsibility: {cloud_responsibility}"
|
|
607
|
+
)
|
|
608
|
+
ex_obj.customerResponsibility = customer_responsibility
|
|
609
|
+
|
|
610
|
+
|
|
348
611
|
def update_imp_objective(
|
|
349
612
|
leverage_auth_id: int,
|
|
350
613
|
existing_imp_obj: List[ImplementationObjective],
|
|
@@ -363,80 +626,49 @@ def update_imp_objective(
|
|
|
363
626
|
:rtype: None
|
|
364
627
|
:return: None
|
|
365
628
|
"""
|
|
366
|
-
|
|
367
629
|
cis_record = record.get("cis", {})
|
|
368
630
|
crm_record = record.get("crm", {})
|
|
369
|
-
# There could be multiples, take the first one as regscale will not allow multiples at the objective level.
|
|
370
|
-
control_originations = cis_record.get("control_origination", "").split(",")
|
|
371
|
-
for ix, control_origination in enumerate(control_originations):
|
|
372
|
-
control_originations[ix] = control_origination.strip()
|
|
373
631
|
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
responsibility = ",".join(control_originations)
|
|
377
|
-
else:
|
|
378
|
-
responsibility = next(origin for origin in control_originations)
|
|
632
|
+
# Parse and clean control originations
|
|
633
|
+
control_originations = [orig.strip() for orig in cis_record.get("control_origination", "").split(",")]
|
|
379
634
|
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
responsibility = imp.responsibility.split(",")[0] # only one responsiblity allowed here.
|
|
383
|
-
else:
|
|
384
|
-
responsibility = SERVICE_PROVIDER_CORPORATE
|
|
635
|
+
# Calculate responsibility
|
|
636
|
+
responsibility = _calculate_responsibility(control_originations, imp)
|
|
385
637
|
|
|
638
|
+
# Parse responsibility fields
|
|
386
639
|
customer_responsibility = clean_customer_responsibility(
|
|
387
640
|
crm_record.get("specific_inheritance_and_customer_agency_csp_responsibilities")
|
|
388
641
|
)
|
|
389
|
-
existing_pairs = {(obj.objectiveId, obj.implementationId) for obj in existing_imp_obj}
|
|
390
|
-
logger.debug(f"CRM Record: {crm_record}")
|
|
391
642
|
can_be_inherited_from_csp: str = crm_record.get("can_be_inherited_from_csp") or ""
|
|
392
643
|
cloud_responsibility = customer_responsibility if can_be_inherited_from_csp.lower() == "yes" else ""
|
|
393
644
|
customer_responsibility = customer_responsibility if can_be_inherited_from_csp.lower() != "yes" else ""
|
|
645
|
+
|
|
646
|
+
existing_pairs = {(obj.objectiveId, obj.implementationId) for obj in existing_imp_obj}
|
|
647
|
+
logger.debug(f"CRM Record: {crm_record}")
|
|
648
|
+
|
|
394
649
|
for objective in objectives:
|
|
650
|
+
if objective.securityControlId != imp.controlID:
|
|
651
|
+
continue
|
|
652
|
+
|
|
395
653
|
current_pair = (objective.id, imp.id)
|
|
396
654
|
if current_pair not in existing_pairs:
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
status=get_multi_status(cis_record),
|
|
407
|
-
objectiveId=objective.id,
|
|
408
|
-
notes=objective.name,
|
|
409
|
-
securityControlId=objective.securityControlId,
|
|
410
|
-
securityPlanId=REGSCALE_SSP_ID,
|
|
411
|
-
responsibility=responsibility,
|
|
412
|
-
cloudResponsibility=cloud_responsibility,
|
|
413
|
-
customerResponsibility=customer_responsibility,
|
|
414
|
-
authorizationId=leverage_auth_id,
|
|
415
|
-
parentObjectiveId=objective.parentObjectiveId,
|
|
416
|
-
)
|
|
417
|
-
logger.debug(
|
|
418
|
-
"Creating new Implementation Objective for Control %s with status: %s responsibility: %s",
|
|
419
|
-
imp_obj.securityControlId,
|
|
420
|
-
imp_obj.status,
|
|
421
|
-
imp_obj.responsibility,
|
|
655
|
+
imp_obj = _create_new_implementation_objective(
|
|
656
|
+
leverage_auth_id,
|
|
657
|
+
imp,
|
|
658
|
+
objective,
|
|
659
|
+
cis_record,
|
|
660
|
+
responsibility,
|
|
661
|
+
cloud_responsibility,
|
|
662
|
+
customer_responsibility,
|
|
663
|
+
can_be_inherited_from_csp,
|
|
422
664
|
)
|
|
423
665
|
UPDATED_IMPLEMENTATION_OBJECTIVES.add(imp_obj)
|
|
424
666
|
else:
|
|
425
667
|
ex_obj = next((obj for obj in existing_imp_obj if obj.objectiveId == objective.id), None)
|
|
426
668
|
if ex_obj:
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
logger.debug(
|
|
431
|
-
f"Updating Implementation Objective #{ex_obj.id} with responsibility: {responsibility}"
|
|
432
|
-
)
|
|
433
|
-
ex_obj.cloudResponsibility = cloud_responsibility
|
|
434
|
-
if customer_responsibility.strip():
|
|
435
|
-
logger.debug(
|
|
436
|
-
f"Updating Implementation Objective #{ex_obj.id} with cloud responsibility: {cloud_responsibility}"
|
|
437
|
-
)
|
|
438
|
-
ex_obj.customerResponsibility = customer_responsibility
|
|
439
|
-
|
|
669
|
+
_update_existing_implementation_objective(
|
|
670
|
+
ex_obj, cis_record, responsibility, cloud_responsibility, customer_responsibility
|
|
671
|
+
)
|
|
440
672
|
UPDATED_IMPLEMENTATION_OBJECTIVES.add(ex_obj)
|
|
441
673
|
|
|
442
674
|
|
|
@@ -565,8 +797,6 @@ def get_all_imps(api: Api, cis_data: dict, version: Literal["rev4", "rev5"]) ->
|
|
|
565
797
|
:return: None
|
|
566
798
|
:rtype: None
|
|
567
799
|
"""
|
|
568
|
-
from requests import RequestException
|
|
569
|
-
|
|
570
800
|
# Check if the response is successful
|
|
571
801
|
if EXISTING_IMPLEMENTATIONS:
|
|
572
802
|
# Get Control Implementations For SSP
|
|
@@ -641,6 +871,9 @@ def update_all_objectives(
|
|
|
641
871
|
"""
|
|
642
872
|
|
|
643
873
|
all_control_objectives = get_all_control_objectives(imps=EXISTING_IMPLEMENTATIONS.values())
|
|
874
|
+
# Create ControlMatcher instance for consistent control ID parsing
|
|
875
|
+
control_matcher = ControlMatcher()
|
|
876
|
+
|
|
644
877
|
error_set = set()
|
|
645
878
|
process_task = progress.add_task(
|
|
646
879
|
"[cyan]Processing control objectives...", total=len(EXISTING_IMPLEMENTATIONS.values())
|
|
@@ -653,7 +886,13 @@ def update_all_objectives(
|
|
|
653
886
|
# Submit all tasks
|
|
654
887
|
future_to_control = {
|
|
655
888
|
executor.submit(
|
|
656
|
-
process_implementation,
|
|
889
|
+
process_implementation,
|
|
890
|
+
leveraged_auth_id,
|
|
891
|
+
imp,
|
|
892
|
+
combined_data,
|
|
893
|
+
version,
|
|
894
|
+
all_control_objectives,
|
|
895
|
+
control_matcher,
|
|
657
896
|
): imp
|
|
658
897
|
for imp in EXISTING_IMPLEMENTATIONS.values()
|
|
659
898
|
}
|
|
@@ -709,6 +948,7 @@ def process_implementation(
|
|
|
709
948
|
sheet_data: dict,
|
|
710
949
|
version: Literal["rev4", "rev5"],
|
|
711
950
|
all_objectives: List[ControlObjective],
|
|
951
|
+
control_matcher: ControlMatcher,
|
|
712
952
|
) -> Tuple[List[str], List[ImplementationObjective]]:
|
|
713
953
|
"""
|
|
714
954
|
Processes a single implementation and its associated records.
|
|
@@ -718,6 +958,7 @@ def process_implementation(
|
|
|
718
958
|
:param dict sheet_data: The CIS/CRM data to process
|
|
719
959
|
:param Literal["rev4", "rev5"] version: The version of the workbook
|
|
720
960
|
:param List[ControlObjective] all_objectives: all the control objectives
|
|
961
|
+
:param ControlMatcher control_matcher: ControlMatcher instance for control ID parsing
|
|
721
962
|
:rtype Tuple[List[str], List[ImplementationObjective]]
|
|
722
963
|
:returns A list of updated implementation objectives
|
|
723
964
|
"""
|
|
@@ -725,7 +966,7 @@ def process_implementation(
|
|
|
725
966
|
errors = []
|
|
726
967
|
processed_objectives = []
|
|
727
968
|
|
|
728
|
-
existing_objectives, filtered_records = gen_filtered_records(implementation, sheet_data,
|
|
969
|
+
existing_objectives, filtered_records = gen_filtered_records(implementation, sheet_data, control_matcher)
|
|
729
970
|
result = None
|
|
730
971
|
for record in filtered_records:
|
|
731
972
|
res = process_single_record(
|
|
@@ -735,6 +976,7 @@ def process_implementation(
|
|
|
735
976
|
control_objectives=all_objectives,
|
|
736
977
|
existing_objectives=existing_objectives,
|
|
737
978
|
version=version,
|
|
979
|
+
control_matcher=control_matcher,
|
|
738
980
|
)
|
|
739
981
|
if isinstance(res, tuple):
|
|
740
982
|
method_errors, result = res
|
|
@@ -745,39 +987,67 @@ def process_implementation(
|
|
|
745
987
|
return errors, processed_objectives
|
|
746
988
|
|
|
747
989
|
|
|
990
|
+
def _extract_base_control_id(control_id: str) -> str:
|
|
991
|
+
"""
|
|
992
|
+
Extract the base control ID from a control ID that may have a letter part.
|
|
993
|
+
|
|
994
|
+
Examples:
|
|
995
|
+
- "AC-1.a" -> "AC-1"
|
|
996
|
+
- "AC-17.2" -> "AC-17.2" (numeric parts are preserved)
|
|
997
|
+
- "AC-1" -> "AC-1"
|
|
998
|
+
|
|
999
|
+
:param str control_id: Control ID that may have a letter part
|
|
1000
|
+
:return: Base control ID without letter part
|
|
1001
|
+
:rtype: str
|
|
1002
|
+
"""
|
|
1003
|
+
# Check if the control has a letter part (e.g., AC-1.a)
|
|
1004
|
+
match = re.match(r"^([A-Z]+-\d+)\.[A-Z]$", control_id, re.IGNORECASE)
|
|
1005
|
+
if match:
|
|
1006
|
+
return match.group(1)
|
|
1007
|
+
return control_id
|
|
1008
|
+
|
|
1009
|
+
|
|
748
1010
|
def gen_filtered_records(
|
|
749
|
-
implementation: ControlImplementation, sheet_data: dict,
|
|
1011
|
+
implementation: ControlImplementation, sheet_data: dict, control_matcher: ControlMatcher
|
|
750
1012
|
) -> Tuple[List[ImplementationObjective], List[Dict[str, str]]]:
|
|
751
1013
|
"""
|
|
752
|
-
Generates filtered records for a given implementation.
|
|
1014
|
+
Generates filtered records for a given implementation using ControlMatcher.
|
|
753
1015
|
|
|
754
1016
|
:param ControlImplementation implementation: The control implementation to filter records for
|
|
755
1017
|
:param dict sheet_data: The CIS/CRM data to filter
|
|
756
|
-
:param
|
|
1018
|
+
:param ControlMatcher control_matcher: ControlMatcher instance for control ID matching
|
|
757
1019
|
:returns A tuple of existing objectives, and filtered records
|
|
758
1020
|
:rtype Tuple[List[ImplementationObjective], List[Dict[str, str]]]
|
|
759
1021
|
"""
|
|
760
1022
|
security_control = SecurityControl.get_object(implementation.controlID)
|
|
761
1023
|
existing_objectives = ImplementationObjective.get_by_control(implementation.id)
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
1024
|
+
|
|
1025
|
+
# Get all variations of the control ID using ControlMatcher
|
|
1026
|
+
# pylint: disable=protected-access # Using internal method for control ID variation matching
|
|
1027
|
+
control_variations = control_matcher._get_control_id_variations(security_control.controlId)
|
|
1028
|
+
|
|
1029
|
+
# Filter records that match any variation of the control ID
|
|
1030
|
+
filtered_records = []
|
|
1031
|
+
for record in sheet_data.values():
|
|
1032
|
+
record_control_id = record["cis"].get("regscale_control_id", "")
|
|
1033
|
+
# Parse the record's control ID
|
|
1034
|
+
parsed_record_id = control_matcher.parse_control_id(record_control_id)
|
|
1035
|
+
if not parsed_record_id:
|
|
1036
|
+
continue
|
|
1037
|
+
# Get variations for the parsed record ID
|
|
1038
|
+
# pylint: disable=protected-access # Using internal method for control ID variation matching
|
|
1039
|
+
record_variations = control_matcher._get_control_id_variations(parsed_record_id)
|
|
1040
|
+
|
|
1041
|
+
# Check if the parsed record control ID matches any variation
|
|
1042
|
+
if control_variations & record_variations:
|
|
1043
|
+
filtered_records.append(record)
|
|
779
1044
|
else:
|
|
780
|
-
|
|
1045
|
+
# If no direct match and record has a letter part, try matching the base control
|
|
1046
|
+
base_control_id = _extract_base_control_id(parsed_record_id)
|
|
1047
|
+
if base_control_id != parsed_record_id:
|
|
1048
|
+
base_variations = control_matcher._get_control_id_variations(base_control_id)
|
|
1049
|
+
if control_variations & base_variations:
|
|
1050
|
+
filtered_records.append(record)
|
|
781
1051
|
|
|
782
1052
|
return existing_objectives, filtered_records
|
|
783
1053
|
|
|
@@ -801,54 +1071,28 @@ def process_single_record(**kwargs) -> Tuple[List[str], Optional[ImplementationO
|
|
|
801
1071
|
:rtype Tuple[List[str], Optional[ImplementationObjective]]
|
|
802
1072
|
:returns A list of errors and the Implementation Objective if successful, otherwise None
|
|
803
1073
|
"""
|
|
804
|
-
# for pytest
|
|
805
|
-
if not part_mapper_rev5.data:
|
|
806
|
-
part_mapper_rev5.load_fedramp_version_5_mapping()
|
|
807
|
-
if not part_mapper_rev4.data:
|
|
808
|
-
part_mapper_rev4.load_fedramp_version_4_mapping()
|
|
809
|
-
|
|
810
1074
|
errors = []
|
|
811
|
-
version = kwargs.get("version")
|
|
812
1075
|
leveraged_auth_id: int = kwargs.get("leveraged_auth_id")
|
|
813
1076
|
implementation: ControlImplementation = kwargs.get("implementation")
|
|
814
1077
|
record: dict = kwargs.get("record")
|
|
815
1078
|
control_objectives: List[ControlObjective] = kwargs.get("control_objectives")
|
|
816
1079
|
existing_objectives: List[ImplementationObjective] = kwargs.get("existing_objectives")
|
|
817
|
-
|
|
1080
|
+
control_matcher: ControlMatcher = kwargs.get("control_matcher")
|
|
818
1081
|
result = None
|
|
819
|
-
parts = []
|
|
820
|
-
# Note: The Control ID from the CIS/CRM can be in non-standard formats, as compared to the example sheet on fedramp.
|
|
821
|
-
if version == "rev5":
|
|
822
|
-
key = record["cis"]["control_id"].replace(" ", "")
|
|
823
|
-
source = part_mapper_rev5.find_by_source(key)
|
|
824
|
-
else:
|
|
825
|
-
key = record["cis"]["control_id"]
|
|
826
|
-
source = part_mapper_rev4.find_by_source(key)
|
|
827
|
-
if parts := part_mapper_rev4.find_sub_parts(key):
|
|
828
|
-
for part in parts:
|
|
829
|
-
try:
|
|
830
|
-
if version == "rev5":
|
|
831
|
-
mapped_objectives.append(next(obj for obj in control_objectives if obj.name == part))
|
|
832
|
-
else:
|
|
833
|
-
mapped_objectives.append(next(obj for obj in control_objectives if obj.otherId == part))
|
|
834
|
-
except StopIteration:
|
|
835
|
-
errors.append(f"Unable to find part {part} for control {key}")
|
|
836
|
-
if not source and not parts:
|
|
837
|
-
errors.append(f"Unable to find source and part for control {key}")
|
|
838
|
-
|
|
839
|
-
if source and not parts:
|
|
840
|
-
try:
|
|
841
|
-
objective = next(
|
|
842
|
-
obj
|
|
843
|
-
for obj in control_objectives
|
|
844
|
-
if (obj.otherId == source and version in ["rev5", "rev4"]) or (obj.name == source and version == "rev4")
|
|
845
|
-
)
|
|
846
|
-
mapped_objectives.append(objective)
|
|
847
|
-
except StopIteration:
|
|
848
|
-
logger.debug(f"Missing Source: {source}")
|
|
849
|
-
errors.append(f"Unable to find objective for control {key} ({source})")
|
|
850
1082
|
|
|
851
|
-
|
|
1083
|
+
# Get the control ID from the CIS/CRM record
|
|
1084
|
+
key = record["cis"]["control_id"]
|
|
1085
|
+
|
|
1086
|
+
# Use ControlMatcher to find matching objectives
|
|
1087
|
+
mapped_objectives, status = find_objectives_using_control_matcher(key, control_objectives, control_matcher)
|
|
1088
|
+
|
|
1089
|
+
logger.debug(f"Control matching result for {key}: {status}")
|
|
1090
|
+
|
|
1091
|
+
# Add to errors list if no objectives found
|
|
1092
|
+
if not mapped_objectives:
|
|
1093
|
+
errors.append(f"{key}: {status}")
|
|
1094
|
+
else:
|
|
1095
|
+
# Update implementation objectives with the matched control objectives
|
|
852
1096
|
update_imp_objective(
|
|
853
1097
|
leverage_auth_id=leveraged_auth_id,
|
|
854
1098
|
existing_imp_obj=existing_objectives,
|
|
@@ -898,46 +1142,53 @@ def parse_crm_worksheet(file_path: click.Path, crm_sheet_name: str, version: Lit
|
|
|
898
1142
|
|
|
899
1143
|
logger.debug(f"Skipping {skip_rows} rows in CRM worksheet")
|
|
900
1144
|
|
|
901
|
-
#
|
|
902
|
-
|
|
1145
|
+
# Define required columns
|
|
1146
|
+
required_columns = [
|
|
903
1147
|
CONTROL_ID,
|
|
904
1148
|
"Can Be Inherited from CSP",
|
|
905
1149
|
"Specific Inheritance and Customer Agency/CSP Responsibilities",
|
|
906
1150
|
]
|
|
907
1151
|
|
|
908
1152
|
try:
|
|
909
|
-
#
|
|
910
|
-
header_row = validator.data.iloc[skip_rows - 1
|
|
1153
|
+
# Get the header row (which is at skip_rows - 1)
|
|
1154
|
+
header_row = validator.data.iloc[skip_rows - 1]
|
|
911
1155
|
|
|
912
|
-
#
|
|
913
|
-
|
|
914
|
-
error_and_exit(
|
|
915
|
-
f"Not enough columns found in CRM worksheet. Expected {len(usecols)} columns but found {len(header_row)}."
|
|
916
|
-
)
|
|
1156
|
+
# Get data rows starting from skip_rows
|
|
1157
|
+
data = validator.data.iloc[skip_rows:].reset_index(drop=True)
|
|
917
1158
|
|
|
918
|
-
#
|
|
1159
|
+
# Set column names from header row
|
|
1160
|
+
data.columns = header_row
|
|
1161
|
+
|
|
1162
|
+
# Find required columns by name (case-insensitive, handle extra columns in AWS CIS/CRM)
|
|
1163
|
+
available_columns = data.columns.tolist()
|
|
1164
|
+
columns_to_use = []
|
|
919
1165
|
missing_columns = []
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
)
|
|
1166
|
+
|
|
1167
|
+
for required_col in required_columns:
|
|
1168
|
+
# Find column that matches (case-insensitive, strip whitespace)
|
|
1169
|
+
matching_col = next(
|
|
1170
|
+
(col for col in available_columns if str(col).strip().lower() == required_col.lower()), None
|
|
1171
|
+
)
|
|
1172
|
+
if matching_col is not None:
|
|
1173
|
+
columns_to_use.append(matching_col)
|
|
1174
|
+
else:
|
|
1175
|
+
missing_columns.append(required_col)
|
|
925
1176
|
|
|
926
1177
|
if missing_columns:
|
|
927
|
-
error_msg =
|
|
1178
|
+
error_msg = (
|
|
1179
|
+
f"Required columns not found in the CRM worksheet: {', '.join(missing_columns)}\n"
|
|
1180
|
+
f"Available columns: {', '.join(str(col) for col in available_columns)}"
|
|
1181
|
+
)
|
|
928
1182
|
error_and_exit(error_msg)
|
|
929
1183
|
|
|
930
|
-
logger.debug("
|
|
931
|
-
|
|
932
|
-
# Reindex the dataframe and skip some rows
|
|
933
|
-
data = validator.data.iloc[skip_rows:]
|
|
1184
|
+
logger.debug(f"Found all required columns in CRM worksheet: {', '.join(required_columns)}")
|
|
934
1185
|
|
|
935
|
-
# Keep only the
|
|
936
|
-
data = data
|
|
1186
|
+
# Keep only the required columns
|
|
1187
|
+
data = data[columns_to_use]
|
|
937
1188
|
|
|
938
|
-
# Rename the columns to
|
|
939
|
-
data.columns =
|
|
940
|
-
logger.debug(f"
|
|
1189
|
+
# Rename the columns to standardize names
|
|
1190
|
+
data.columns = required_columns
|
|
1191
|
+
logger.debug(f"Using columns: {', '.join(required_columns)}")
|
|
941
1192
|
|
|
942
1193
|
except KeyError as e:
|
|
943
1194
|
error_and_exit(f"KeyError: {e} - One or more columns specified in usecols are not found in the dataframe.")
|
|
@@ -973,59 +1224,216 @@ def parse_crm_worksheet(file_path: click.Path, crm_sheet_name: str, version: Lit
|
|
|
973
1224
|
return formatted_crm
|
|
974
1225
|
|
|
975
1226
|
|
|
976
|
-
def
|
|
1227
|
+
def _get_expected_cis_columns() -> List[str]:
|
|
977
1228
|
"""
|
|
978
|
-
|
|
1229
|
+
Get the expected column names for CIS worksheet in order.
|
|
1230
|
+
These match the FedRAMP Rev 5 CIS worksheet format.
|
|
979
1231
|
|
|
980
|
-
:
|
|
981
|
-
:
|
|
982
|
-
|
|
983
|
-
|
|
1232
|
+
:return: List of expected column names
|
|
1233
|
+
:rtype: List[str]
|
|
1234
|
+
"""
|
|
1235
|
+
return [
|
|
1236
|
+
CONTROL_ID, # "Control ID"
|
|
1237
|
+
"Implemented",
|
|
1238
|
+
ControlImplementationStatus.PartiallyImplemented, # "Partially Implemented"
|
|
1239
|
+
"Planned",
|
|
1240
|
+
ALTERNATIVE_IMPLEMENTATION, # "Alternative Implementation"
|
|
1241
|
+
ControlImplementationStatus.NA, # "N/A"
|
|
1242
|
+
SERVICE_PROVIDER_CORPORATE,
|
|
1243
|
+
SERVICE_PROVIDER_SYSTEM_SPECIFIC,
|
|
1244
|
+
SERVICE_PROVIDER_HYBRID,
|
|
1245
|
+
CONFIGURED_BY_CUSTOMER,
|
|
1246
|
+
PROVIDED_BY_CUSTOMER,
|
|
1247
|
+
SHARED,
|
|
1248
|
+
INHERITED, # "Inherited from pre-existing FedRAMP Authorization"
|
|
1249
|
+
]
|
|
1250
|
+
|
|
1251
|
+
|
|
1252
|
+
def _normalize_cis_columns(cis_df: "pd.DataFrame", expected_columns: List[str]) -> "pd.DataFrame":
|
|
1253
|
+
"""
|
|
1254
|
+
Normalize CIS dataframe columns by matching expected columns and handling missing ones.
|
|
1255
|
+
Uses fuzzy matching to handle truncated column names from merged cells.
|
|
1256
|
+
|
|
1257
|
+
:param pd.DataFrame cis_df: The CIS dataframe
|
|
1258
|
+
:param List[str] expected_columns: List of expected column names
|
|
1259
|
+
:return: Normalized dataframe with standardized column names
|
|
1260
|
+
:rtype: pd.DataFrame
|
|
1261
|
+
"""
|
|
1262
|
+
available_columns = cis_df.columns.tolist()
|
|
1263
|
+
columns_to_keep = []
|
|
1264
|
+
|
|
1265
|
+
logger.debug(f"Available CIS columns: {available_columns}")
|
|
1266
|
+
|
|
1267
|
+
for expected_col in expected_columns:
|
|
1268
|
+
matching_col = None
|
|
1269
|
+
|
|
1270
|
+
# Try exact match first (case-insensitive)
|
|
1271
|
+
matching_col = next(
|
|
1272
|
+
(col for col in available_columns if str(col).strip().lower() == expected_col.lower()), None
|
|
1273
|
+
)
|
|
1274
|
+
|
|
1275
|
+
# If no exact match, try partial/fuzzy match for truncated column names
|
|
1276
|
+
if matching_col is None:
|
|
1277
|
+
# Create a simplified version for matching (first few significant words)
|
|
1278
|
+
# Filter out common words and take first 3 significant words
|
|
1279
|
+
skip_words = {"from", "by", "to", "the", "and", "or", "a", "an"}
|
|
1280
|
+
expected_words = [w for w in expected_col.lower().split() if w not in skip_words][:3]
|
|
1281
|
+
|
|
1282
|
+
for col in available_columns:
|
|
1283
|
+
col_str = str(col).lower()
|
|
1284
|
+
# Check if at least 2 of the significant words are in the column name (handles truncation & variations)
|
|
1285
|
+
matches = sum(1 for word in expected_words if word in col_str)
|
|
1286
|
+
if matches >= min(2, len(expected_words)): # Need at least 2 matches, or all if less than 2 words
|
|
1287
|
+
matching_col = col
|
|
1288
|
+
logger.debug(
|
|
1289
|
+
f"Fuzzy matched '{expected_col}' to '{col}' (matched {matches}/{len(expected_words)} words)"
|
|
1290
|
+
)
|
|
1291
|
+
break
|
|
1292
|
+
|
|
1293
|
+
if matching_col is not None:
|
|
1294
|
+
columns_to_keep.append(matching_col)
|
|
1295
|
+
else:
|
|
1296
|
+
logger.info(f"Expected column '{expected_col}' not found in CIS worksheet. Using empty values.")
|
|
1297
|
+
cis_df[expected_col] = ""
|
|
1298
|
+
columns_to_keep.append(expected_col)
|
|
1299
|
+
|
|
1300
|
+
cis_df = cis_df[columns_to_keep]
|
|
1301
|
+
cis_df.columns = expected_columns
|
|
1302
|
+
return cis_df.fillna("")
|
|
1303
|
+
|
|
1304
|
+
|
|
1305
|
+
def _find_control_id_row_index(df: "pd.DataFrame") -> Optional[int]:
|
|
1306
|
+
"""
|
|
1307
|
+
Find the row index containing 'Control ID' in the first column.
|
|
1308
|
+
|
|
1309
|
+
:param pd.DataFrame df: The dataframe to search
|
|
1310
|
+
:return: Row index if found, None otherwise
|
|
1311
|
+
:rtype: Optional[int]
|
|
1312
|
+
"""
|
|
1313
|
+
for idx, row in df.iterrows():
|
|
1314
|
+
if row.iloc[0] == CONTROL_ID:
|
|
1315
|
+
return idx
|
|
1316
|
+
return None
|
|
1317
|
+
|
|
1318
|
+
|
|
1319
|
+
def _merge_header_rows(header_row, sub_header_row) -> List[str]:
|
|
1320
|
+
"""
|
|
1321
|
+
Merge two header rows into a single list of column names.
|
|
1322
|
+
|
|
1323
|
+
FedRAMP Rev5 has a two-row header structure where main headers span multiple columns
|
|
1324
|
+
and sub-headers provide specific column names.
|
|
1325
|
+
|
|
1326
|
+
:param header_row: The main header row (categories)
|
|
1327
|
+
:param sub_header_row: The sub-header row (specific columns)
|
|
1328
|
+
:return: List of merged column names
|
|
1329
|
+
:rtype: List[str]
|
|
984
1330
|
"""
|
|
985
1331
|
pd = get_pandas()
|
|
986
|
-
|
|
987
|
-
|
|
1332
|
+
merged_headers = []
|
|
1333
|
+
current_category = None
|
|
988
1334
|
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
mapping_file_path=gettempdir(),
|
|
994
|
-
prompt=False,
|
|
995
|
-
ignore_unnamed=True,
|
|
996
|
-
worksheet_name=cis_sheet_name,
|
|
997
|
-
warn_extra_headers=False,
|
|
998
|
-
)
|
|
999
|
-
if validator.data.empty:
|
|
1000
|
-
return {}
|
|
1335
|
+
for i, (main, sub) in enumerate(zip(header_row, sub_header_row)):
|
|
1336
|
+
# Update current category if main header has a value
|
|
1337
|
+
if pd.notna(main) and main and str(main).strip():
|
|
1338
|
+
current_category = str(main)
|
|
1001
1339
|
|
|
1002
|
-
|
|
1340
|
+
# Determine which header value to use
|
|
1341
|
+
header_value = _select_header_value(pd, main, sub, current_category, i)
|
|
1342
|
+
merged_headers.append(header_value)
|
|
1343
|
+
|
|
1344
|
+
return merged_headers
|
|
1003
1345
|
|
|
1004
|
-
# Parse the worksheet named 'CIS GovCloud U.S.+DoD (H)', skipping the initial rows
|
|
1005
|
-
original_cis = validator.data
|
|
1006
1346
|
|
|
1007
|
-
|
|
1347
|
+
def _select_header_value(pd: "pd.DataFrame", main, sub, current_category: Optional[str], index: int) -> str:
|
|
1348
|
+
"""
|
|
1349
|
+
Select the appropriate header value based on priority: sub-header > main header > category > unnamed.
|
|
1008
1350
|
|
|
1009
|
-
|
|
1010
|
-
|
|
1351
|
+
:param pd.DataFrame pd: The pandas dataframe
|
|
1352
|
+
:param main: Main header value
|
|
1353
|
+
:param sub: Sub-header value
|
|
1354
|
+
:param Optional[str] current_category: Current category from merged cells
|
|
1355
|
+
:param int index: Column index for fallback naming
|
|
1356
|
+
:return: Selected header value
|
|
1357
|
+
:rtype: str
|
|
1358
|
+
"""
|
|
1359
|
+
if pd.notna(sub) and sub and str(sub).strip():
|
|
1360
|
+
return str(sub)
|
|
1361
|
+
if pd.notna(main) and main and str(main).strip():
|
|
1362
|
+
return str(main)
|
|
1363
|
+
if current_category:
|
|
1364
|
+
return f"{current_category}_{index}"
|
|
1365
|
+
return f"Unnamed_{index}"
|
|
1011
1366
|
|
|
1012
|
-
# Drop any fully empty rows
|
|
1013
|
-
cis_df.dropna(how="all", inplace=True)
|
|
1014
1367
|
|
|
1015
|
-
|
|
1368
|
+
def _load_and_prepare_cis_dataframe(file_path: click.Path, cis_sheet_name: str, skip_rows: int):
|
|
1369
|
+
"""
|
|
1370
|
+
Load and prepare the CIS dataframe from the workbook.
|
|
1371
|
+
|
|
1372
|
+
:param click.Path file_path: The file path to the workbook
|
|
1373
|
+
:param str cis_sheet_name: The sheet name to parse
|
|
1374
|
+
:param int skip_rows: Number of rows to skip
|
|
1375
|
+
:return: Tuple of (prepared dataframe, updated skip_rows) or (None, skip_rows) if empty
|
|
1376
|
+
"""
|
|
1377
|
+
# Read the Excel file directly with pandas to preserve "N/A" as string
|
|
1378
|
+
pd = get_pandas()
|
|
1379
|
+
df = pd.read_excel(file_path, sheet_name=cis_sheet_name, header=None, keep_default_na=False)
|
|
1380
|
+
|
|
1381
|
+
if df.empty:
|
|
1382
|
+
return None, skip_rows
|
|
1383
|
+
|
|
1384
|
+
# Find the row with "Control ID"
|
|
1385
|
+
control_id_row_idx = _find_control_id_row_index(df)
|
|
1386
|
+
if control_id_row_idx is None:
|
|
1387
|
+
logger.error("Could not find 'Control ID' in CIS worksheet")
|
|
1388
|
+
return None, skip_rows
|
|
1389
|
+
|
|
1390
|
+
# Extract and merge the two header rows
|
|
1391
|
+
header_row = df.iloc[control_id_row_idx]
|
|
1392
|
+
sub_header_row = df.iloc[control_id_row_idx + 1]
|
|
1393
|
+
merged_headers = _merge_header_rows(header_row, sub_header_row)
|
|
1394
|
+
|
|
1395
|
+
# Get data starting from two rows after the main header row
|
|
1396
|
+
cis_df = df.iloc[control_id_row_idx + 2 :].reset_index(drop=True)
|
|
1397
|
+
cis_df.columns = merged_headers
|
|
1398
|
+
cis_df.dropna(how="all", inplace=True)
|
|
1016
1399
|
cis_df.reset_index(drop=True, inplace=True)
|
|
1017
1400
|
|
|
1018
|
-
|
|
1019
|
-
cis_df = cis_df.iloc[:, :13]
|
|
1401
|
+
skip_rows = control_id_row_idx + 2
|
|
1020
1402
|
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1403
|
+
return cis_df, skip_rows
|
|
1404
|
+
|
|
1405
|
+
|
|
1406
|
+
def _extract_status(data_row) -> str:
|
|
1407
|
+
"""
|
|
1408
|
+
Extract the first non-empty implementation status from the CIS worksheet.
|
|
1409
|
+
|
|
1410
|
+
:param data_row: The data row to extract the status from
|
|
1411
|
+
:return: The implementation status
|
|
1412
|
+
:rtype: str
|
|
1413
|
+
"""
|
|
1414
|
+
selected_status = []
|
|
1415
|
+
for col in [
|
|
1024
1416
|
"Implemented",
|
|
1025
1417
|
ControlImplementationStatus.PartiallyImplemented,
|
|
1026
1418
|
"Planned",
|
|
1027
|
-
|
|
1419
|
+
ALTERNATIVE_IMPLEMENTATION, # Use the correct constant
|
|
1028
1420
|
ControlImplementationStatus.NA,
|
|
1421
|
+
]:
|
|
1422
|
+
if data_row[col]:
|
|
1423
|
+
selected_status.append(col)
|
|
1424
|
+
return ", ".join(selected_status) if selected_status else ""
|
|
1425
|
+
|
|
1426
|
+
|
|
1427
|
+
def _extract_origination(data_row) -> str:
|
|
1428
|
+
"""
|
|
1429
|
+
Extract the first non-empty control origination from the CIS worksheet.
|
|
1430
|
+
|
|
1431
|
+
:param data_row: The data row to extract the origination from
|
|
1432
|
+
:return: The control origination
|
|
1433
|
+
:rtype: str
|
|
1434
|
+
"""
|
|
1435
|
+
selected_origination = []
|
|
1436
|
+
for col in [
|
|
1029
1437
|
SERVICE_PROVIDER_CORPORATE,
|
|
1030
1438
|
SERVICE_PROVIDER_SYSTEM_SPECIFIC,
|
|
1031
1439
|
SERVICE_PROVIDER_HYBRID,
|
|
@@ -1033,75 +1441,53 @@ def parse_cis_worksheet(file_path: click.Path, cis_sheet_name: str) -> dict:
|
|
|
1033
1441
|
PROVIDED_BY_CUSTOMER,
|
|
1034
1442
|
SHARED,
|
|
1035
1443
|
INHERITED,
|
|
1036
|
-
]
|
|
1444
|
+
]:
|
|
1445
|
+
if data_row[col]:
|
|
1446
|
+
selected_origination.append(col)
|
|
1447
|
+
return ", ".join(selected_origination) if selected_origination else ""
|
|
1037
1448
|
|
|
1038
|
-
# Fill NaN values with an empty string for processing
|
|
1039
|
-
cis_df = cis_df.fillna("")
|
|
1040
|
-
|
|
1041
|
-
# Function to extract the first non-empty implementation status
|
|
1042
|
-
def _extract_status(data_row: pd.Series) -> str:
|
|
1043
|
-
"""
|
|
1044
|
-
Function to extract the first non-empty implementation status from the CIS worksheet
|
|
1045
|
-
|
|
1046
|
-
:param pd.Series data_row: The data row to extract the status from
|
|
1047
|
-
:return: The implementation status
|
|
1048
|
-
:rtype: str
|
|
1049
|
-
"""
|
|
1050
|
-
selected_status = []
|
|
1051
|
-
for col in [
|
|
1052
|
-
"Implemented",
|
|
1053
|
-
ControlImplementationStatus.PartiallyImplemented,
|
|
1054
|
-
"Planned",
|
|
1055
|
-
ALT_IMPLEMENTATION,
|
|
1056
|
-
ControlImplementationStatus.NA,
|
|
1057
|
-
]:
|
|
1058
|
-
if data_row[col]:
|
|
1059
|
-
selected_status.append(col)
|
|
1060
|
-
return ", ".join(selected_status) if selected_status else ""
|
|
1061
|
-
|
|
1062
|
-
# Function to extract the first non-empty control origination
|
|
1063
|
-
def _extract_origination(data_row: pd.Series) -> str:
|
|
1064
|
-
"""
|
|
1065
|
-
Function to extract the first non-empty control origination from the CIS worksheet
|
|
1066
|
-
|
|
1067
|
-
:param pd.Series data_row: The data row to extract the origination from
|
|
1068
|
-
:return: The control origination
|
|
1069
|
-
:rtype: str
|
|
1070
|
-
"""
|
|
1071
|
-
selected_origination = []
|
|
1072
|
-
for col in [
|
|
1073
|
-
SERVICE_PROVIDER_CORPORATE,
|
|
1074
|
-
SERVICE_PROVIDER_SYSTEM_SPECIFIC,
|
|
1075
|
-
SERVICE_PROVIDER_HYBRID,
|
|
1076
|
-
CONFIGURED_BY_CUSTOMER,
|
|
1077
|
-
PROVIDED_BY_CUSTOMER,
|
|
1078
|
-
SHARED,
|
|
1079
|
-
INHERITED,
|
|
1080
|
-
]:
|
|
1081
|
-
if data_row[col]:
|
|
1082
|
-
selected_origination.append(col)
|
|
1083
|
-
return ", ".join(selected_origination) if selected_origination else ""
|
|
1084
|
-
|
|
1085
|
-
def _process_row(row: pd.Series) -> dict:
|
|
1086
|
-
"""
|
|
1087
|
-
Function to process a row from the CIS worksheet
|
|
1088
|
-
|
|
1089
|
-
:param pd.Series row: The row to process
|
|
1090
|
-
:return: The processed row
|
|
1091
|
-
:rtype: dict
|
|
1092
|
-
"""
|
|
1093
|
-
return {
|
|
1094
|
-
"control_id": row[CONTROL_ID],
|
|
1095
|
-
"regscale_control_id": transform_control(row[CONTROL_ID]),
|
|
1096
|
-
"implementation_status": _extract_status(row),
|
|
1097
|
-
"control_origination": _extract_origination(row),
|
|
1098
|
-
}
|
|
1099
1449
|
|
|
1100
|
-
|
|
1450
|
+
def _process_cis_row(row) -> dict:
|
|
1451
|
+
"""
|
|
1452
|
+
Process a row from the CIS worksheet.
|
|
1453
|
+
|
|
1454
|
+
:param row: The row to process
|
|
1455
|
+
:return: The processed row
|
|
1456
|
+
:rtype: dict
|
|
1457
|
+
"""
|
|
1458
|
+
return {
|
|
1459
|
+
"control_id": row[CONTROL_ID],
|
|
1460
|
+
"regscale_control_id": transform_control(row[CONTROL_ID]),
|
|
1461
|
+
"implementation_status": _extract_status(row),
|
|
1462
|
+
"control_origination": _extract_origination(row),
|
|
1463
|
+
}
|
|
1464
|
+
|
|
1465
|
+
|
|
1466
|
+
def parse_cis_worksheet(file_path: click.Path, cis_sheet_name: str) -> dict:
|
|
1467
|
+
"""
|
|
1468
|
+
Function to parse and format the CIS worksheet content
|
|
1469
|
+
|
|
1470
|
+
:param click.Path file_path: The file path to the FedRAMP CIS CRM workbook
|
|
1471
|
+
:param str cis_sheet_name: The name of the CIS sheet to parse
|
|
1472
|
+
:return: Formatted CIS content
|
|
1473
|
+
:rtype: dict
|
|
1474
|
+
"""
|
|
1475
|
+
logger.info("Parsing CIS worksheet...")
|
|
1476
|
+
|
|
1477
|
+
# Load and prepare the dataframe
|
|
1478
|
+
cis_df, _ = _load_and_prepare_cis_dataframe(file_path, cis_sheet_name, skip_rows=2)
|
|
1479
|
+
if cis_df is None:
|
|
1480
|
+
return {}
|
|
1481
|
+
|
|
1482
|
+
# Get expected columns and normalize the dataframe
|
|
1483
|
+
expected_columns = _get_expected_cis_columns()
|
|
1484
|
+
cis_df = _normalize_cis_columns(cis_df, expected_columns)
|
|
1485
|
+
|
|
1486
|
+
# Process rows in parallel
|
|
1101
1487
|
with ThreadPoolExecutor() as executor:
|
|
1102
|
-
results = list(executor.map(
|
|
1488
|
+
results = list(executor.map(_process_cis_row, [row for _, row in cis_df.iterrows()]))
|
|
1103
1489
|
|
|
1104
|
-
#
|
|
1490
|
+
# Index by control_id
|
|
1105
1491
|
return {clean_key(result["control_id"]): result for result in results}
|
|
1106
1492
|
|
|
1107
1493
|
|
|
@@ -1325,7 +1711,7 @@ def extract_control_name(control_string: str) -> str:
|
|
|
1325
1711
|
:return: The extracted control name
|
|
1326
1712
|
:rtype: str
|
|
1327
1713
|
"""
|
|
1328
|
-
pattern = r"^[A-Za-z]{2}-\d{1,3}(?:\(\d+\))?"
|
|
1714
|
+
pattern = r"^[A-Za-z]{2}-\d{1,3}(?:\s*\(\s*\d+\s*\))?"
|
|
1329
1715
|
match = re.match(pattern, control_string.upper())
|
|
1330
1716
|
return match.group() if match else ""
|
|
1331
1717
|
|
|
@@ -1338,8 +1724,8 @@ def rev_4_map(control_id: str) -> Optional[str]:
|
|
|
1338
1724
|
:return: The mapped control ID or None if not found
|
|
1339
1725
|
:rtype: Optional[str]
|
|
1340
1726
|
"""
|
|
1341
|
-
# Regex pattern to match different control ID formats
|
|
1342
|
-
pattern = r"^([A-Z]{2})-(\d{2})\s*(?:\((\d{2})\))?\s*(?:\(([a-z])\))?$"
|
|
1727
|
+
# Regex pattern to match different control ID formats - handles extra spaces like AC-6 ( 1 ) ( a )
|
|
1728
|
+
pattern = r"^([A-Z]{2})-(\d{2})\s*(?:\(\s*(\d{2})\s*\))?\s*(?:\(\s*([a-z])\s*\))?$"
|
|
1343
1729
|
|
|
1344
1730
|
match = re.match(pattern, control_id, re.IGNORECASE)
|
|
1345
1731
|
|
|
@@ -1464,7 +1850,7 @@ def create_new_security_plan(profile_id: int, system_name: str):
|
|
|
1464
1850
|
|
|
1465
1851
|
else:
|
|
1466
1852
|
INITIAL_IMPORT = False
|
|
1467
|
-
ret = next((
|
|
1853
|
+
ret = next(iter(existing_plan), None)
|
|
1468
1854
|
logger.info(f"Found existing SSP# {ret.id}")
|
|
1469
1855
|
create_backup_file(ret.id)
|
|
1470
1856
|
existing_imps = ControlImplementation.get_list_by_plan(ret.id)
|
|
@@ -1549,7 +1935,7 @@ def copy_and_rename_file(file_path: Path, new_name: str) -> Path:
|
|
|
1549
1935
|
"""
|
|
1550
1936
|
Copy and rename a file.
|
|
1551
1937
|
"""
|
|
1552
|
-
temp_folder = Path(
|
|
1938
|
+
temp_folder = Path(gettempdir()) / "regscale"
|
|
1553
1939
|
temp_folder.mkdir(exist_ok=True) # Ensure directory exists
|
|
1554
1940
|
|
|
1555
1941
|
new_file_path = temp_folder / new_name
|
|
@@ -1610,10 +1996,9 @@ def parse_and_import_ciscrm(
|
|
|
1610
1996
|
|
|
1611
1997
|
if "5" in version:
|
|
1612
1998
|
version = "rev5"
|
|
1613
|
-
part_mapper_rev5.load_fedramp_version_5_mapping()
|
|
1614
1999
|
else:
|
|
1615
2000
|
version = "rev4"
|
|
1616
|
-
|
|
2001
|
+
# No longer loading JSON mappings - using smart algorithm only
|
|
1617
2002
|
# parse the instructions worksheet to get the csp name, system name, and other data
|
|
1618
2003
|
instructions_data = parse_instructions_worksheet(df=df, version=version) # type: ignore
|
|
1619
2004
|
|
|
@@ -1654,7 +2039,8 @@ def parse_and_import_ciscrm(
|
|
|
1654
2039
|
cis_data = parse_cis_worksheet(file_path=file_path, cis_sheet_name=cis_sheet_name)
|
|
1655
2040
|
crm_data = {}
|
|
1656
2041
|
if crm_sheet_name:
|
|
1657
|
-
|
|
2042
|
+
# type: ignore
|
|
2043
|
+
crm_data = parse_crm_worksheet(file_path=file_path, crm_sheet_name=crm_sheet_name, version=version)
|
|
1658
2044
|
if leveraged_auth_id == 0:
|
|
1659
2045
|
auths = LeveragedAuthorization.get_all_by_parent(ssp.id)
|
|
1660
2046
|
if auths:
|