regscale-cli 6.21.2.0__py3-none-any.whl → 6.28.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- regscale/_version.py +1 -1
- regscale/airflow/hierarchy.py +2 -2
- regscale/core/app/api.py +5 -2
- regscale/core/app/application.py +36 -6
- regscale/core/app/internal/control_editor.py +73 -21
- regscale/core/app/internal/evidence.py +727 -204
- regscale/core/app/internal/login.py +4 -2
- regscale/core/app/internal/model_editor.py +219 -64
- regscale/core/app/utils/app_utils.py +86 -12
- regscale/core/app/utils/catalog_utils/common.py +1 -1
- regscale/core/login.py +21 -4
- regscale/core/utils/async_graphql_client.py +363 -0
- regscale/core/utils/date.py +77 -1
- regscale/dev/cli.py +26 -0
- regscale/dev/code_gen.py +109 -24
- regscale/dev/version.py +72 -0
- regscale/integrations/commercial/__init__.py +30 -2
- regscale/integrations/commercial/aws/audit_manager_compliance.py +3908 -0
- regscale/integrations/commercial/aws/cli.py +3107 -54
- regscale/integrations/commercial/aws/cloudtrail_control_mappings.py +333 -0
- regscale/integrations/commercial/aws/cloudtrail_evidence.py +501 -0
- regscale/integrations/commercial/aws/cloudwatch_control_mappings.py +357 -0
- regscale/integrations/commercial/aws/cloudwatch_evidence.py +490 -0
- regscale/integrations/commercial/{amazon → aws}/common.py +71 -19
- regscale/integrations/commercial/aws/config_compliance.py +914 -0
- regscale/integrations/commercial/aws/conformance_pack_mappings.py +198 -0
- regscale/integrations/commercial/aws/control_compliance_analyzer.py +439 -0
- regscale/integrations/commercial/aws/evidence_generator.py +283 -0
- regscale/integrations/commercial/aws/guardduty_control_mappings.py +340 -0
- regscale/integrations/commercial/aws/guardduty_evidence.py +1053 -0
- regscale/integrations/commercial/aws/iam_control_mappings.py +368 -0
- regscale/integrations/commercial/aws/iam_evidence.py +574 -0
- regscale/integrations/commercial/aws/inventory/__init__.py +338 -22
- regscale/integrations/commercial/aws/inventory/base.py +107 -5
- regscale/integrations/commercial/aws/inventory/resources/analytics.py +390 -0
- regscale/integrations/commercial/aws/inventory/resources/applications.py +234 -0
- regscale/integrations/commercial/aws/inventory/resources/audit_manager.py +513 -0
- regscale/integrations/commercial/aws/inventory/resources/cloudtrail.py +315 -0
- regscale/integrations/commercial/aws/inventory/resources/cloudtrail_logs_metadata.py +476 -0
- regscale/integrations/commercial/aws/inventory/resources/cloudwatch.py +191 -0
- regscale/integrations/commercial/aws/inventory/resources/compute.py +328 -9
- regscale/integrations/commercial/aws/inventory/resources/config.py +464 -0
- regscale/integrations/commercial/aws/inventory/resources/containers.py +74 -9
- regscale/integrations/commercial/aws/inventory/resources/database.py +481 -31
- regscale/integrations/commercial/aws/inventory/resources/developer_tools.py +253 -0
- regscale/integrations/commercial/aws/inventory/resources/guardduty.py +286 -0
- regscale/integrations/commercial/aws/inventory/resources/iam.py +470 -0
- regscale/integrations/commercial/aws/inventory/resources/inspector.py +476 -0
- regscale/integrations/commercial/aws/inventory/resources/integration.py +175 -61
- regscale/integrations/commercial/aws/inventory/resources/kms.py +447 -0
- regscale/integrations/commercial/aws/inventory/resources/machine_learning.py +358 -0
- regscale/integrations/commercial/aws/inventory/resources/networking.py +390 -67
- regscale/integrations/commercial/aws/inventory/resources/s3.py +394 -0
- regscale/integrations/commercial/aws/inventory/resources/security.py +268 -72
- regscale/integrations/commercial/aws/inventory/resources/securityhub.py +473 -0
- regscale/integrations/commercial/aws/inventory/resources/storage.py +288 -29
- regscale/integrations/commercial/aws/inventory/resources/systems_manager.py +657 -0
- regscale/integrations/commercial/aws/inventory/resources/vpc.py +655 -0
- regscale/integrations/commercial/aws/kms_control_mappings.py +288 -0
- regscale/integrations/commercial/aws/kms_evidence.py +879 -0
- regscale/integrations/commercial/aws/ocsf/__init__.py +7 -0
- regscale/integrations/commercial/aws/ocsf/constants.py +115 -0
- regscale/integrations/commercial/aws/ocsf/mapper.py +435 -0
- regscale/integrations/commercial/aws/org_control_mappings.py +286 -0
- regscale/integrations/commercial/aws/org_evidence.py +666 -0
- regscale/integrations/commercial/aws/s3_control_mappings.py +356 -0
- regscale/integrations/commercial/aws/s3_evidence.py +632 -0
- regscale/integrations/commercial/aws/scanner.py +1072 -205
- regscale/integrations/commercial/aws/security_hub.py +319 -0
- regscale/integrations/commercial/aws/session_manager.py +282 -0
- regscale/integrations/commercial/aws/ssm_control_mappings.py +291 -0
- regscale/integrations/commercial/aws/ssm_evidence.py +492 -0
- regscale/integrations/commercial/jira.py +489 -153
- regscale/integrations/commercial/microsoft_defender/defender.py +326 -5
- regscale/integrations/commercial/microsoft_defender/defender_api.py +348 -14
- regscale/integrations/commercial/microsoft_defender/defender_constants.py +157 -0
- regscale/integrations/commercial/qualys/__init__.py +167 -68
- regscale/integrations/commercial/qualys/scanner.py +305 -39
- regscale/integrations/commercial/sarif/sairf_importer.py +432 -0
- regscale/integrations/commercial/sarif/sarif_converter.py +67 -0
- regscale/integrations/commercial/sicura/api.py +79 -42
- regscale/integrations/commercial/sicura/commands.py +8 -2
- regscale/integrations/commercial/sicura/scanner.py +83 -44
- regscale/integrations/commercial/stigv2/ckl_parser.py +5 -5
- regscale/integrations/commercial/synqly/assets.py +133 -16
- regscale/integrations/commercial/synqly/edr.py +2 -8
- regscale/integrations/commercial/synqly/query_builder.py +536 -0
- regscale/integrations/commercial/synqly/ticketing.py +27 -0
- regscale/integrations/commercial/synqly/vulnerabilities.py +165 -28
- regscale/integrations/commercial/tenablev2/cis_parsers.py +453 -0
- regscale/integrations/commercial/tenablev2/cis_scanner.py +447 -0
- regscale/integrations/commercial/tenablev2/commands.py +146 -5
- regscale/integrations/commercial/tenablev2/scanner.py +1 -3
- regscale/integrations/commercial/tenablev2/stig_parsers.py +113 -57
- regscale/integrations/commercial/wizv2/WizDataMixin.py +1 -1
- regscale/integrations/commercial/wizv2/click.py +191 -76
- regscale/integrations/commercial/wizv2/compliance/__init__.py +15 -0
- regscale/integrations/commercial/wizv2/{policy_compliance_helpers.py → compliance/helpers.py} +78 -60
- regscale/integrations/commercial/wizv2/compliance_report.py +1592 -0
- regscale/integrations/commercial/wizv2/core/__init__.py +133 -0
- regscale/integrations/commercial/wizv2/{async_client.py → core/client.py} +7 -3
- regscale/integrations/commercial/wizv2/{constants.py → core/constants.py} +92 -89
- regscale/integrations/commercial/wizv2/core/file_operations.py +237 -0
- regscale/integrations/commercial/wizv2/fetchers/__init__.py +11 -0
- regscale/integrations/commercial/wizv2/{data_fetcher.py → fetchers/policy_assessment.py} +66 -9
- regscale/integrations/commercial/wizv2/file_cleanup.py +104 -0
- regscale/integrations/commercial/wizv2/issue.py +776 -28
- regscale/integrations/commercial/wizv2/models/__init__.py +0 -0
- regscale/integrations/commercial/wizv2/parsers/__init__.py +34 -0
- regscale/integrations/commercial/wizv2/{parsers.py → parsers/main.py} +1 -1
- regscale/integrations/commercial/wizv2/processors/__init__.py +11 -0
- regscale/integrations/commercial/wizv2/{finding_processor.py → processors/finding.py} +1 -1
- regscale/integrations/commercial/wizv2/reports.py +243 -0
- regscale/integrations/commercial/wizv2/sbom.py +1 -1
- regscale/integrations/commercial/wizv2/scanner.py +1031 -441
- regscale/integrations/commercial/wizv2/utils/__init__.py +48 -0
- regscale/integrations/commercial/wizv2/{utils.py → utils/main.py} +116 -61
- regscale/integrations/commercial/wizv2/variables.py +89 -3
- regscale/integrations/compliance_integration.py +1036 -151
- regscale/integrations/control_matcher.py +432 -0
- regscale/integrations/due_date_handler.py +333 -0
- regscale/integrations/milestone_manager.py +291 -0
- regscale/integrations/public/__init__.py +14 -0
- regscale/integrations/public/cci_importer.py +834 -0
- regscale/integrations/public/csam/__init__.py +0 -0
- regscale/integrations/public/csam/csam.py +938 -0
- regscale/integrations/public/csam/csam_agency_defined.py +179 -0
- regscale/integrations/public/csam/csam_common.py +154 -0
- regscale/integrations/public/csam/csam_controls.py +432 -0
- regscale/integrations/public/csam/csam_poam.py +124 -0
- regscale/integrations/public/fedramp/click.py +77 -6
- regscale/integrations/public/fedramp/docx_parser.py +10 -1
- regscale/integrations/public/fedramp/fedramp_cis_crm.py +675 -289
- regscale/integrations/public/fedramp/fedramp_five.py +1 -1
- regscale/integrations/public/fedramp/poam/scanner.py +75 -7
- regscale/integrations/public/fedramp/poam_export_v5.py +888 -0
- regscale/integrations/scanner_integration.py +1961 -430
- regscale/models/integration_models/CCI_List.xml +1 -0
- regscale/models/integration_models/aqua.py +2 -2
- regscale/models/integration_models/cisa_kev_data.json +805 -11
- regscale/models/integration_models/flat_file_importer/__init__.py +5 -8
- regscale/models/integration_models/nexpose.py +36 -10
- regscale/models/integration_models/qualys.py +3 -4
- regscale/models/integration_models/synqly_models/capabilities.json +1 -1
- regscale/models/integration_models/synqly_models/connectors/vulnerabilities.py +87 -18
- regscale/models/integration_models/synqly_models/filter_parser.py +332 -0
- regscale/models/integration_models/synqly_models/ocsf_mapper.py +124 -25
- regscale/models/integration_models/synqly_models/synqly_model.py +89 -16
- regscale/models/locking.py +12 -8
- regscale/models/platform.py +4 -2
- regscale/models/regscale_models/__init__.py +7 -0
- regscale/models/regscale_models/assessment.py +2 -1
- regscale/models/regscale_models/catalog.py +1 -1
- regscale/models/regscale_models/compliance_settings.py +251 -1
- regscale/models/regscale_models/component.py +1 -0
- regscale/models/regscale_models/control_implementation.py +236 -41
- regscale/models/regscale_models/control_objective.py +74 -5
- regscale/models/regscale_models/file.py +2 -0
- regscale/models/regscale_models/form_field_value.py +5 -3
- regscale/models/regscale_models/inheritance.py +44 -0
- regscale/models/regscale_models/issue.py +301 -102
- regscale/models/regscale_models/milestone.py +33 -14
- regscale/models/regscale_models/organization.py +3 -0
- regscale/models/regscale_models/regscale_model.py +310 -73
- regscale/models/regscale_models/security_plan.py +4 -2
- regscale/models/regscale_models/vulnerability.py +3 -3
- regscale/regscale.py +25 -4
- regscale/templates/__init__.py +0 -0
- regscale/utils/threading/threadhandler.py +20 -15
- regscale/validation/record.py +23 -1
- {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/METADATA +17 -33
- {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/RECORD +310 -111
- tests/core/__init__.py +0 -0
- tests/core/utils/__init__.py +0 -0
- tests/core/utils/test_async_graphql_client.py +472 -0
- tests/fixtures/test_fixture.py +13 -8
- tests/regscale/core/test_login.py +171 -4
- tests/regscale/integrations/commercial/__init__.py +0 -0
- tests/regscale/integrations/commercial/aws/__init__.py +0 -0
- tests/regscale/integrations/commercial/aws/test_audit_manager_compliance.py +1304 -0
- tests/regscale/integrations/commercial/aws/test_audit_manager_evidence_aggregation.py +341 -0
- tests/regscale/integrations/commercial/aws/test_aws_analytics_collector.py +260 -0
- tests/regscale/integrations/commercial/aws/test_aws_applications_collector.py +242 -0
- tests/regscale/integrations/commercial/aws/test_aws_audit_manager_collector.py +1155 -0
- tests/regscale/integrations/commercial/aws/test_aws_cloudtrail_collector.py +534 -0
- tests/regscale/integrations/commercial/aws/test_aws_config_collector.py +400 -0
- tests/regscale/integrations/commercial/aws/test_aws_developer_tools_collector.py +203 -0
- tests/regscale/integrations/commercial/aws/test_aws_guardduty_collector.py +315 -0
- tests/regscale/integrations/commercial/aws/test_aws_iam_collector.py +458 -0
- tests/regscale/integrations/commercial/aws/test_aws_inspector_collector.py +353 -0
- tests/regscale/integrations/commercial/aws/test_aws_inventory_integration.py +530 -0
- tests/regscale/integrations/commercial/aws/test_aws_kms_collector.py +919 -0
- tests/regscale/integrations/commercial/aws/test_aws_machine_learning_collector.py +237 -0
- tests/regscale/integrations/commercial/aws/test_aws_s3_collector.py +722 -0
- tests/regscale/integrations/commercial/aws/test_aws_scanner_integration.py +722 -0
- tests/regscale/integrations/commercial/aws/test_aws_securityhub_collector.py +792 -0
- tests/regscale/integrations/commercial/aws/test_aws_systems_manager_collector.py +918 -0
- tests/regscale/integrations/commercial/aws/test_aws_vpc_collector.py +996 -0
- tests/regscale/integrations/commercial/aws/test_cli_evidence.py +431 -0
- tests/regscale/integrations/commercial/aws/test_cloudtrail_control_mappings.py +452 -0
- tests/regscale/integrations/commercial/aws/test_cloudtrail_evidence.py +788 -0
- tests/regscale/integrations/commercial/aws/test_config_compliance.py +298 -0
- tests/regscale/integrations/commercial/aws/test_conformance_pack_mappings.py +200 -0
- tests/regscale/integrations/commercial/aws/test_control_compliance_analyzer.py +375 -0
- tests/regscale/integrations/commercial/aws/test_datetime_parsing.py +223 -0
- tests/regscale/integrations/commercial/aws/test_evidence_generator.py +386 -0
- tests/regscale/integrations/commercial/aws/test_guardduty_control_mappings.py +564 -0
- tests/regscale/integrations/commercial/aws/test_guardduty_evidence.py +1041 -0
- tests/regscale/integrations/commercial/aws/test_iam_control_mappings.py +718 -0
- tests/regscale/integrations/commercial/aws/test_iam_evidence.py +1375 -0
- tests/regscale/integrations/commercial/aws/test_kms_control_mappings.py +656 -0
- tests/regscale/integrations/commercial/aws/test_kms_evidence.py +1163 -0
- tests/regscale/integrations/commercial/aws/test_ocsf_mapper.py +370 -0
- tests/regscale/integrations/commercial/aws/test_org_control_mappings.py +546 -0
- tests/regscale/integrations/commercial/aws/test_org_evidence.py +1240 -0
- tests/regscale/integrations/commercial/aws/test_s3_control_mappings.py +672 -0
- tests/regscale/integrations/commercial/aws/test_s3_evidence.py +987 -0
- tests/regscale/integrations/commercial/aws/test_scanner_evidence.py +373 -0
- tests/regscale/integrations/commercial/aws/test_security_hub_config_filtering.py +539 -0
- tests/regscale/integrations/commercial/aws/test_session_manager.py +516 -0
- tests/regscale/integrations/commercial/aws/test_ssm_control_mappings.py +588 -0
- tests/regscale/integrations/commercial/aws/test_ssm_evidence.py +735 -0
- tests/regscale/integrations/commercial/conftest.py +28 -0
- tests/regscale/integrations/commercial/microsoft_defender/__init__.py +1 -0
- tests/regscale/integrations/commercial/microsoft_defender/test_defender.py +1517 -0
- tests/regscale/integrations/commercial/microsoft_defender/test_defender_api.py +1748 -0
- tests/regscale/integrations/commercial/microsoft_defender/test_defender_constants.py +327 -0
- tests/regscale/integrations/commercial/microsoft_defender/test_defender_scanner.py +487 -0
- tests/regscale/integrations/commercial/test_aws.py +3742 -0
- tests/regscale/integrations/commercial/test_burp.py +48 -0
- tests/regscale/integrations/commercial/test_crowdstrike.py +49 -0
- tests/regscale/integrations/commercial/test_dependabot.py +341 -0
- tests/regscale/integrations/commercial/test_gcp.py +1543 -0
- tests/regscale/integrations/commercial/test_gitlab.py +549 -0
- tests/regscale/integrations/commercial/test_ip_mac_address_length.py +84 -0
- tests/regscale/integrations/commercial/test_jira.py +2204 -0
- tests/regscale/integrations/commercial/test_npm_audit.py +42 -0
- tests/regscale/integrations/commercial/test_okta.py +1228 -0
- tests/regscale/integrations/commercial/test_sarif_converter.py +251 -0
- tests/regscale/integrations/commercial/test_sicura.py +349 -0
- tests/regscale/integrations/commercial/test_snow.py +423 -0
- tests/regscale/integrations/commercial/test_sonarcloud.py +394 -0
- tests/regscale/integrations/commercial/test_sqlserver.py +186 -0
- tests/regscale/integrations/commercial/test_stig.py +33 -0
- tests/regscale/integrations/commercial/test_stig_mapper.py +153 -0
- tests/regscale/integrations/commercial/test_stigv2.py +406 -0
- tests/regscale/integrations/commercial/test_wiz.py +1365 -0
- tests/regscale/integrations/commercial/test_wiz_inventory.py +256 -0
- tests/regscale/integrations/commercial/wizv2/__init__.py +339 -0
- tests/regscale/integrations/commercial/wizv2/compliance/__init__.py +1 -0
- tests/regscale/integrations/commercial/wizv2/compliance/test_helpers.py +903 -0
- tests/regscale/integrations/commercial/wizv2/core/__init__.py +1 -0
- tests/regscale/integrations/commercial/wizv2/core/test_auth.py +701 -0
- tests/regscale/integrations/commercial/wizv2/core/test_client.py +1037 -0
- tests/regscale/integrations/commercial/wizv2/core/test_file_operations.py +989 -0
- tests/regscale/integrations/commercial/wizv2/fetchers/__init__.py +1 -0
- tests/regscale/integrations/commercial/wizv2/fetchers/test_policy_assessment.py +805 -0
- tests/regscale/integrations/commercial/wizv2/parsers/__init__.py +1 -0
- tests/regscale/integrations/commercial/wizv2/parsers/test_main.py +1153 -0
- tests/regscale/integrations/commercial/wizv2/processors/__init__.py +1 -0
- tests/regscale/integrations/commercial/wizv2/processors/test_finding.py +671 -0
- tests/regscale/integrations/commercial/wizv2/test_WizDataMixin.py +537 -0
- tests/regscale/integrations/commercial/wizv2/test_click_comprehensive.py +851 -0
- tests/regscale/integrations/commercial/wizv2/test_compliance_report_comprehensive.py +910 -0
- tests/regscale/integrations/commercial/wizv2/test_compliance_report_normalization.py +138 -0
- tests/regscale/integrations/commercial/wizv2/test_file_cleanup.py +283 -0
- tests/regscale/integrations/commercial/wizv2/test_file_operations.py +260 -0
- tests/regscale/integrations/commercial/wizv2/test_issue.py +343 -0
- tests/regscale/integrations/commercial/wizv2/test_issue_comprehensive.py +1203 -0
- tests/regscale/integrations/commercial/wizv2/test_reports.py +497 -0
- tests/regscale/integrations/commercial/wizv2/test_sbom.py +643 -0
- tests/regscale/integrations/commercial/wizv2/test_scanner_comprehensive.py +805 -0
- tests/regscale/integrations/commercial/wizv2/test_wiz_click_client_id.py +165 -0
- tests/regscale/integrations/commercial/wizv2/test_wiz_compliance_report.py +1394 -0
- tests/regscale/integrations/commercial/wizv2/test_wiz_compliance_unit.py +341 -0
- tests/regscale/integrations/commercial/wizv2/test_wiz_control_normalization.py +138 -0
- tests/regscale/integrations/commercial/wizv2/test_wiz_findings_comprehensive.py +364 -0
- tests/regscale/integrations/commercial/wizv2/test_wiz_inventory_comprehensive.py +644 -0
- tests/regscale/integrations/commercial/wizv2/test_wiz_status_mapping.py +149 -0
- tests/regscale/integrations/commercial/wizv2/test_wizv2.py +1218 -0
- tests/regscale/integrations/commercial/wizv2/test_wizv2_utils.py +519 -0
- tests/regscale/integrations/commercial/wizv2/utils/__init__.py +1 -0
- tests/regscale/integrations/commercial/wizv2/utils/test_main.py +1523 -0
- tests/regscale/integrations/public/__init__.py +0 -0
- tests/regscale/integrations/public/fedramp/__init__.py +1 -0
- tests/regscale/integrations/public/fedramp/test_gen_asset_list.py +150 -0
- tests/regscale/integrations/public/fedramp/test_poam_export_v5.py +1293 -0
- tests/regscale/integrations/public/test_alienvault.py +220 -0
- tests/regscale/integrations/public/test_cci.py +1053 -0
- tests/regscale/integrations/public/test_cisa.py +1021 -0
- tests/regscale/integrations/public/test_emass.py +518 -0
- tests/regscale/integrations/public/test_fedramp.py +1152 -0
- tests/regscale/integrations/public/test_fedramp_cis_crm.py +3661 -0
- tests/regscale/integrations/public/test_file_uploads.py +506 -0
- tests/regscale/integrations/public/test_oscal.py +453 -0
- tests/regscale/integrations/test_compliance_status_mapping.py +406 -0
- tests/regscale/integrations/test_control_matcher.py +1421 -0
- tests/regscale/integrations/test_control_matching.py +155 -0
- tests/regscale/integrations/test_milestone_manager.py +408 -0
- tests/regscale/models/test_control_implementation.py +118 -3
- tests/regscale/models/test_form_field_value_integration.py +304 -0
- tests/regscale/models/test_issue.py +378 -1
- tests/regscale/models/test_module_integration.py +582 -0
- tests/regscale/models/test_tenable_integrations.py +811 -105
- regscale/integrations/commercial/wizv2/policy_compliance.py +0 -3057
- regscale/integrations/public/fedramp/mappings/fedramp_r4_parts.json +0 -7388
- regscale/integrations/public/fedramp/mappings/fedramp_r5_parts.json +0 -9605
- regscale/integrations/public/fedramp/parts_mapper.py +0 -107
- /regscale/integrations/commercial/{amazon → sarif}/__init__.py +0 -0
- /regscale/integrations/commercial/wizv2/{wiz_auth.py → core/auth.py} +0 -0
- {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/LICENSE +0 -0
- {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/WHEEL +0 -0
- {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/entry_points.txt +0 -0
- {regscale_cli-6.21.2.0.dist-info → regscale_cli-6.28.2.1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,834 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
"""RegScale CLI command to normalize CCI data from XML files."""
|
|
3
|
+
import datetime
|
|
4
|
+
import logging
|
|
5
|
+
import xml.etree.ElementTree as ET
|
|
6
|
+
from typing import Dict, List, Optional, Tuple
|
|
7
|
+
|
|
8
|
+
import click
|
|
9
|
+
from rich.progress import Progress, TaskID
|
|
10
|
+
|
|
11
|
+
from regscale.core.app.application import Application
|
|
12
|
+
from regscale.core.app.utils.app_utils import create_progress_object, error_and_exit
|
|
13
|
+
from regscale.models.regscale_models import Catalog, SecurityControl, CCI, ControlObjective
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger("regscale")
|
|
16
|
+
|
|
17
|
+
# RegScale date format constant
|
|
18
|
+
REGSCALE_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class CCIImporter:
|
|
22
|
+
"""Imports CCI data from XML files and maps to security controls."""
|
|
23
|
+
|
|
24
|
+
def __init__(self, xml_data: ET.Element, version: str = "5", verbose: bool = False):
|
|
25
|
+
"""
|
|
26
|
+
Initialize the CCI importer.
|
|
27
|
+
|
|
28
|
+
:param ET.Element xml_data: The root element of the XML data
|
|
29
|
+
:param str version: NIST version to use for filtering
|
|
30
|
+
:param bool verbose: Whether to output verbose information
|
|
31
|
+
"""
|
|
32
|
+
self.xml_data = xml_data
|
|
33
|
+
self.normalized_cci: Dict[str, List[Dict]] = {}
|
|
34
|
+
self.cci_grouped_by_index: Dict[str, str] = {}
|
|
35
|
+
self.verbose = verbose
|
|
36
|
+
self.reference_version = version
|
|
37
|
+
self._user_context: Optional[Tuple[Optional[str], int]] = None
|
|
38
|
+
|
|
39
|
+
@staticmethod
|
|
40
|
+
def _parse_control_id(ref_index: str) -> str:
|
|
41
|
+
"""
|
|
42
|
+
Extract the main control_id from a reference index (e.g., 'AC-1 a 1 (b)' -> 'AC-1').
|
|
43
|
+
|
|
44
|
+
:param str ref_index: Reference index string to parse
|
|
45
|
+
:return: Main control ID
|
|
46
|
+
:rtype: str
|
|
47
|
+
"""
|
|
48
|
+
parts = ref_index.strip().split()
|
|
49
|
+
return parts[0] if parts else ""
|
|
50
|
+
|
|
51
|
+
@staticmethod
|
|
52
|
+
def format_index(index: str) -> str:
|
|
53
|
+
"""
|
|
54
|
+
Format index according to ControlObjective matching requirements.
|
|
55
|
+
|
|
56
|
+
Examples:
|
|
57
|
+
'AC-1 a 1' -> 'AC-1(a)(1)'
|
|
58
|
+
'IA-13 (03) (a)' -> 'IA-13(03)(a)'
|
|
59
|
+
'AC-1 a 1 (a)' -> 'AC-1(a)(1)(a)'
|
|
60
|
+
|
|
61
|
+
:param str index: Raw index string from XML
|
|
62
|
+
:return: Formatted index string
|
|
63
|
+
:rtype: str
|
|
64
|
+
"""
|
|
65
|
+
import re
|
|
66
|
+
|
|
67
|
+
index = index.strip()
|
|
68
|
+
|
|
69
|
+
# Pattern: match either (text) or non-whitespace text
|
|
70
|
+
pattern = r"\([^)]+\)|[^\s()]+"
|
|
71
|
+
parts = re.findall(pattern, index)
|
|
72
|
+
|
|
73
|
+
if len(parts) <= 1:
|
|
74
|
+
return parts[0] if parts else index
|
|
75
|
+
|
|
76
|
+
# First part is the base control (e.g., 'AC-1', 'IA-13')
|
|
77
|
+
result = parts[0]
|
|
78
|
+
|
|
79
|
+
# Process remaining parts
|
|
80
|
+
for part in parts[1:]:
|
|
81
|
+
if part.startswith("("):
|
|
82
|
+
# Already has parentheses, just append
|
|
83
|
+
result += part
|
|
84
|
+
else:
|
|
85
|
+
# Need to add parentheses
|
|
86
|
+
result += f"({part})"
|
|
87
|
+
|
|
88
|
+
return result
|
|
89
|
+
|
|
90
|
+
@staticmethod
|
|
91
|
+
def parse_objective_id(objective_id: str) -> Tuple[Optional[str], Optional[str]]:
|
|
92
|
+
"""
|
|
93
|
+
Parse an objective otherId to extract control base and part.
|
|
94
|
+
|
|
95
|
+
Supports both NIST 800-53 Revision 4 and 5 formats:
|
|
96
|
+
|
|
97
|
+
Revision 5 Examples:
|
|
98
|
+
"ac-1_smt.a" -> ("AC-1", "a")
|
|
99
|
+
"ac-2.3_smt.a" -> ("AC-2(3)", "a")
|
|
100
|
+
"au-10.1_smt.a" -> ("AU-10(1)", "a")
|
|
101
|
+
"ac-2.4_smt" -> ("AC-2(4)", None)
|
|
102
|
+
|
|
103
|
+
Revision 4 Examples:
|
|
104
|
+
"ac-1_smt.a.1" -> ("AC-1", "a")
|
|
105
|
+
"ac-1_smt.b.2" -> ("AC-1", "b")
|
|
106
|
+
"ac-2.3_smt.d" -> ("AC-2(3)", "d")
|
|
107
|
+
|
|
108
|
+
:param str objective_id: Objective otherId value
|
|
109
|
+
:return: Tuple of (control_base, part_letter or None)
|
|
110
|
+
:rtype: Tuple[Optional[str], Optional[str]]
|
|
111
|
+
"""
|
|
112
|
+
import re
|
|
113
|
+
|
|
114
|
+
# Pattern 1: xx-nn[.nn]_smt.x[.nn] (with part letter, optional subpart for rev 4)
|
|
115
|
+
# Matches: ac-1_smt.a, ac-1_smt.a.1, ac-2.3_smt.d
|
|
116
|
+
match = re.match(r"^([a-z]+)-(\d+)(?:\.(\d+))?_smt\.([a-z]+)(?:\.\d+)?$", objective_id.lower())
|
|
117
|
+
|
|
118
|
+
if match:
|
|
119
|
+
family = match.group(1).upper()
|
|
120
|
+
control_num = match.group(2)
|
|
121
|
+
enhancement = match.group(3)
|
|
122
|
+
part = match.group(4)
|
|
123
|
+
else:
|
|
124
|
+
# Pattern 2: xx-nn[.nn]_smt[.x][.nn] (without clear part letter, or just enhancement)
|
|
125
|
+
# Matches: ac-2.4_smt, ac-1_smt
|
|
126
|
+
match = re.match(r"^([a-z]+)-(\d+)(?:\.(\d+))?_smt(?:\.[a-z]+)?(?:\.\d+)?$", objective_id.lower())
|
|
127
|
+
if not match:
|
|
128
|
+
return None, None
|
|
129
|
+
|
|
130
|
+
family = match.group(1).upper()
|
|
131
|
+
control_num = match.group(2)
|
|
132
|
+
enhancement = match.group(3)
|
|
133
|
+
part = None
|
|
134
|
+
|
|
135
|
+
if enhancement:
|
|
136
|
+
# Enhancement like AC-2(3)
|
|
137
|
+
control_base = f"{family}-{control_num}({enhancement})"
|
|
138
|
+
else:
|
|
139
|
+
# Base control like AC-1
|
|
140
|
+
control_base = f"{family}-{control_num}"
|
|
141
|
+
|
|
142
|
+
return control_base, part
|
|
143
|
+
|
|
144
|
+
@staticmethod
|
|
145
|
+
def find_matching_ccis(control_base: str, part: Optional[str], cci_map: Dict[str, str]) -> List[str]:
|
|
146
|
+
"""
|
|
147
|
+
Find all CCI IDs that match the control base and part.
|
|
148
|
+
|
|
149
|
+
Examples:
|
|
150
|
+
control_base="AC-1", part="a" matches:
|
|
151
|
+
- AC-1(a)(1)(a)
|
|
152
|
+
- AC-1(a)(1)(b)
|
|
153
|
+
- AC-1(a)(2)
|
|
154
|
+
- AC-1(a)
|
|
155
|
+
|
|
156
|
+
:param str control_base: Control identifier (e.g., "AC-1", "AC-2(3)")
|
|
157
|
+
:param Optional[str] part: Part letter (e.g., "a", "b") or None for enhancements
|
|
158
|
+
:param Dict[str, str] cci_map: Map of formatted index to comma-separated CCI IDs
|
|
159
|
+
:return: List of CCI ID strings (comma-separated)
|
|
160
|
+
:rtype: List[str]
|
|
161
|
+
"""
|
|
162
|
+
matching_ccis = []
|
|
163
|
+
|
|
164
|
+
for index, cci_ids in cci_map.items():
|
|
165
|
+
# Check if index starts with control_base
|
|
166
|
+
if index.startswith(control_base):
|
|
167
|
+
# Extract the part after control_base
|
|
168
|
+
remainder = index[len(control_base) :]
|
|
169
|
+
|
|
170
|
+
# For enhancements without parts, only match exact control (no remainder)
|
|
171
|
+
# And Check if the remainder starts with (part)
|
|
172
|
+
if (part is None and remainder == "") or remainder.startswith(f"({part})") or remainder == f"({part})":
|
|
173
|
+
matching_ccis.append(cci_ids)
|
|
174
|
+
|
|
175
|
+
return matching_ccis
|
|
176
|
+
|
|
177
|
+
def find_matching_ccis_by_name(self, control_base: str, name: str, cci_map: Dict[str, str]) -> List[str]:
|
|
178
|
+
"""
|
|
179
|
+
Find CCIs by matching control base and objective name field.
|
|
180
|
+
Fallback method when otherId matching fails.
|
|
181
|
+
|
|
182
|
+
Supports both NIST 800-53 Revision 4 and 5 label formats:
|
|
183
|
+
|
|
184
|
+
Revision 5 Examples:
|
|
185
|
+
control_base="AC-2(4)", name="AC-2(4)" -> matches AC-2(4)
|
|
186
|
+
control_base="AC-1", name="a" -> matches AC-1(a), AC-1(a)(1), etc.
|
|
187
|
+
|
|
188
|
+
Revision 4 Examples:
|
|
189
|
+
control_base="AC-1", name="a.1." -> matches AC-1(a), AC-1(a)(1), etc.
|
|
190
|
+
control_base="AC-1", name="b.2." -> matches AC-1(b), AC-1(b)(2), etc.
|
|
191
|
+
|
|
192
|
+
:param str control_base: Control identifier
|
|
193
|
+
:param str name: Objective name field
|
|
194
|
+
:param Dict[str, str] cci_map: Map of formatted index to comma-separated CCI IDs
|
|
195
|
+
:return: List of CCI ID strings (comma-separated)
|
|
196
|
+
:rtype: List[str]
|
|
197
|
+
"""
|
|
198
|
+
import re
|
|
199
|
+
|
|
200
|
+
matching_ccis = []
|
|
201
|
+
|
|
202
|
+
# Remove trailing period and whitespace from name for matching
|
|
203
|
+
clean_name = name.strip().rstrip(".").strip()
|
|
204
|
+
|
|
205
|
+
# If name exactly matches control_base, match that control
|
|
206
|
+
if clean_name == control_base:
|
|
207
|
+
if control_base in cci_map:
|
|
208
|
+
matching_ccis.append(cci_map[control_base])
|
|
209
|
+
return matching_ccis
|
|
210
|
+
|
|
211
|
+
# Try to extract part letter from different formats
|
|
212
|
+
part = None
|
|
213
|
+
|
|
214
|
+
# Check if name is a single letter (Revision 5 format: "a", "b")
|
|
215
|
+
if len(clean_name) == 1 and clean_name.isalpha():
|
|
216
|
+
part = clean_name.lower()
|
|
217
|
+
elif match := re.match(r"^([a-z])\.\d+$", clean_name.lower()):
|
|
218
|
+
part = match.group(1)
|
|
219
|
+
|
|
220
|
+
# If we extracted a part letter, try matching
|
|
221
|
+
if part:
|
|
222
|
+
matching_ccis = self._extract_part_letter(cci_map, control_base, part, matching_ccis)
|
|
223
|
+
|
|
224
|
+
return matching_ccis
|
|
225
|
+
|
|
226
|
+
@staticmethod
|
|
227
|
+
def _extract_part_letter(
|
|
228
|
+
cci_map: Dict[str, str], control_base: str, part: str, matching_ccis: List[str]
|
|
229
|
+
) -> List[str]:
|
|
230
|
+
"""
|
|
231
|
+
Extract the part letter from the name.
|
|
232
|
+
|
|
233
|
+
:param Dict[str, str] cci_map: The map of CCI IDs to their indices.
|
|
234
|
+
:param str control_base: The control base to match against.
|
|
235
|
+
:param str part: The part letter to match against.
|
|
236
|
+
:param List[str] matching_ccis: The list of matching CCI IDs.
|
|
237
|
+
:rtype: List[str]
|
|
238
|
+
"""
|
|
239
|
+
for index, cci_ids in cci_map.items():
|
|
240
|
+
if index.startswith(control_base):
|
|
241
|
+
remainder = index[len(control_base) :]
|
|
242
|
+
if remainder.startswith(f"({part})") or remainder == f"({part})":
|
|
243
|
+
matching_ccis.append(cci_ids)
|
|
244
|
+
return matching_ccis
|
|
245
|
+
|
|
246
|
+
@staticmethod
|
|
247
|
+
def ccis_already_present(current_other_id: str, new_cci_ids: str) -> bool:
|
|
248
|
+
"""
|
|
249
|
+
Check if any of the new CCI IDs are already present in current otherId.
|
|
250
|
+
Prevents duplicate CCI mappings.
|
|
251
|
+
|
|
252
|
+
:param str current_other_id: Current otherId value
|
|
253
|
+
:param str new_cci_ids: Comma-separated string of CCI IDs to add
|
|
254
|
+
:return: True if any CCIs are already present
|
|
255
|
+
:rtype: bool
|
|
256
|
+
"""
|
|
257
|
+
if not current_other_id:
|
|
258
|
+
return False
|
|
259
|
+
|
|
260
|
+
# Extract individual CCI IDs from both strings
|
|
261
|
+
existing_ccis: set[str] = {cci.strip() for cci in current_other_id.split(",") if cci.strip().startswith("CCI-")}
|
|
262
|
+
new_ccis: set[str] = {cci.strip() for cci in new_cci_ids.split(",") if cci.strip().startswith("CCI-")}
|
|
263
|
+
|
|
264
|
+
# Check if there's any overlap
|
|
265
|
+
return bool(existing_ccis & new_ccis)
|
|
266
|
+
|
|
267
|
+
@staticmethod
|
|
268
|
+
def _extract_cci_data(cci_item: ET.Element) -> Tuple[Optional[str], str]:
|
|
269
|
+
"""
|
|
270
|
+
Extract CCI ID and definition from CCI item.
|
|
271
|
+
|
|
272
|
+
:param ET.Element cci_item: XML element containing CCI data
|
|
273
|
+
:return: Tuple of (cci_id, definition)
|
|
274
|
+
:rtype: Tuple[Optional[str], str]
|
|
275
|
+
"""
|
|
276
|
+
cci_id = cci_item.get("id")
|
|
277
|
+
definition_elem = cci_item.find(".//{http://iase.disa.mil/cci}definition")
|
|
278
|
+
definition = definition_elem.text if definition_elem is not None and definition_elem.text else ""
|
|
279
|
+
return cci_id, definition
|
|
280
|
+
|
|
281
|
+
def _process_references(self, references: List[ET.Element], cci_id: str, definition: str) -> None:
|
|
282
|
+
"""
|
|
283
|
+
Process reference elements and add to normalized CCI data.
|
|
284
|
+
|
|
285
|
+
:param List[ET.Element] references: List of reference XML elements
|
|
286
|
+
:param str cci_id: CCI identifier
|
|
287
|
+
:param str definition: CCI definition text
|
|
288
|
+
:rtype: None
|
|
289
|
+
"""
|
|
290
|
+
for ref in references:
|
|
291
|
+
if not self._is_valid_reference(ref):
|
|
292
|
+
continue
|
|
293
|
+
|
|
294
|
+
ref_index = ref.get("index")
|
|
295
|
+
if ref_index:
|
|
296
|
+
main_control = self._parse_control_id(ref_index)
|
|
297
|
+
self._add_cci_to_control(main_control, cci_id, definition)
|
|
298
|
+
|
|
299
|
+
def _is_valid_reference(self, ref: ET.Element) -> bool:
|
|
300
|
+
"""
|
|
301
|
+
Check if reference matches the target version.
|
|
302
|
+
|
|
303
|
+
:param ET.Element ref: Reference XML element
|
|
304
|
+
:return: True if reference version matches target version
|
|
305
|
+
:rtype: bool
|
|
306
|
+
"""
|
|
307
|
+
ref_version = ref.get("version")
|
|
308
|
+
return ref_version is not None and ref_version == self.reference_version
|
|
309
|
+
|
|
310
|
+
def _add_cci_to_control(self, main_control: str, cci_id: str, definition: str) -> None:
|
|
311
|
+
"""
|
|
312
|
+
Add CCI data to the normalized structure for a control.
|
|
313
|
+
|
|
314
|
+
:param str main_control: Control identifier
|
|
315
|
+
:param str cci_id: CCI identifier
|
|
316
|
+
:param str definition: CCI definition
|
|
317
|
+
:rtype: None
|
|
318
|
+
"""
|
|
319
|
+
if main_control not in self.normalized_cci:
|
|
320
|
+
self.normalized_cci[main_control] = []
|
|
321
|
+
self.normalized_cci[main_control].append({"cci_id": cci_id, "definition": definition})
|
|
322
|
+
|
|
323
|
+
def parse_cci(self) -> None:
|
|
324
|
+
"""
|
|
325
|
+
Parse CCI items from XML and create both mapping structures.
|
|
326
|
+
|
|
327
|
+
Creates:
|
|
328
|
+
- normalized_cci: Dict[control_id, List[Dict]] - for SecurityControl mapping
|
|
329
|
+
- cci_grouped_by_index: Dict[formatted_index, str] - for ControlObjective mapping
|
|
330
|
+
|
|
331
|
+
:rtype: None
|
|
332
|
+
"""
|
|
333
|
+
if self.verbose:
|
|
334
|
+
logger.info("Parsing CCI items from XML...")
|
|
335
|
+
|
|
336
|
+
# Track all CCI items with formatted indices for objective mapping
|
|
337
|
+
from collections import defaultdict
|
|
338
|
+
|
|
339
|
+
temp_grouped: Dict[str, List[str]] = defaultdict(list)
|
|
340
|
+
|
|
341
|
+
for cci_item in self.xml_data.findall(".//{http://iase.disa.mil/cci}cci_item"):
|
|
342
|
+
cci_id, definition = self._extract_cci_data(cci_item)
|
|
343
|
+
if not cci_id:
|
|
344
|
+
continue
|
|
345
|
+
|
|
346
|
+
references = cci_item.findall(".//{http://iase.disa.mil/cci}reference")
|
|
347
|
+
|
|
348
|
+
for ref in references:
|
|
349
|
+
if not self._is_valid_reference(ref):
|
|
350
|
+
continue
|
|
351
|
+
|
|
352
|
+
ref_index = ref.get("index")
|
|
353
|
+
if ref_index:
|
|
354
|
+
# Existing: simple control ID for SecurityControl mapping
|
|
355
|
+
main_control = self._parse_control_id(ref_index)
|
|
356
|
+
self._add_cci_to_control(main_control, cci_id, definition)
|
|
357
|
+
|
|
358
|
+
# NEW: formatted index for ControlObjective mapping
|
|
359
|
+
formatted_index = self.format_index(ref_index)
|
|
360
|
+
temp_grouped[formatted_index].append(cci_id)
|
|
361
|
+
|
|
362
|
+
# Convert to comma-separated format
|
|
363
|
+
self.cci_grouped_by_index = {index: ", ".join(cci_list) for index, cci_list in temp_grouped.items()}
|
|
364
|
+
|
|
365
|
+
if self.verbose:
|
|
366
|
+
logger.info(f"Created {len(self.normalized_cci)} control mappings")
|
|
367
|
+
logger.info(f"Created {len(self.cci_grouped_by_index)} formatted index mappings")
|
|
368
|
+
|
|
369
|
+
@staticmethod
|
|
370
|
+
def _get_catalog(catalog_id: int) -> Catalog:
|
|
371
|
+
"""
|
|
372
|
+
Get the catalog with specified ID.
|
|
373
|
+
|
|
374
|
+
:param int catalog_id: ID of the catalog to retrieve
|
|
375
|
+
:return: Catalog instance
|
|
376
|
+
:rtype: Catalog
|
|
377
|
+
:raises SystemExit: If catalog not found
|
|
378
|
+
"""
|
|
379
|
+
try:
|
|
380
|
+
catalog = Catalog.get(id=catalog_id)
|
|
381
|
+
if catalog is None:
|
|
382
|
+
error_and_exit(f"Catalog with id {catalog_id} not found. Please ensure the catalog exists.")
|
|
383
|
+
return catalog
|
|
384
|
+
except Exception:
|
|
385
|
+
error_and_exit(f"Catalog with id {catalog_id} not found. Please ensure the catalog exists.")
|
|
386
|
+
|
|
387
|
+
def _get_user_context(self) -> Tuple[Optional[str], int]:
|
|
388
|
+
"""
|
|
389
|
+
Get user ID and tenant ID from application config.
|
|
390
|
+
|
|
391
|
+
:return: Tuple of (user_id, tenant_id)
|
|
392
|
+
:rtype: Tuple[Optional[str], int]
|
|
393
|
+
"""
|
|
394
|
+
if self._user_context is None:
|
|
395
|
+
app = Application()
|
|
396
|
+
user_id = app.config.get("userId")
|
|
397
|
+
tenant_id = app.config.get("tenantId", 1)
|
|
398
|
+
|
|
399
|
+
try:
|
|
400
|
+
user_id = str(user_id) if user_id else None
|
|
401
|
+
except (TypeError, ValueError):
|
|
402
|
+
user_id = None
|
|
403
|
+
if self.verbose:
|
|
404
|
+
logger.warning("userId in config is not set or invalid; created_by will be None.")
|
|
405
|
+
|
|
406
|
+
# Convert tenant_id to int if it's a string
|
|
407
|
+
try:
|
|
408
|
+
tenant_id = int(tenant_id)
|
|
409
|
+
except (TypeError, ValueError):
|
|
410
|
+
tenant_id = 1
|
|
411
|
+
if self.verbose:
|
|
412
|
+
logger.warning("tenantId in config is not valid; using default value 1.")
|
|
413
|
+
|
|
414
|
+
self._user_context = (user_id, tenant_id)
|
|
415
|
+
|
|
416
|
+
return self._user_context
|
|
417
|
+
|
|
418
|
+
@staticmethod
|
|
419
|
+
def _find_existing_cci(control_id: int, cci_id: str) -> Optional[CCI]:
|
|
420
|
+
"""
|
|
421
|
+
Find existing CCI by ID within a control.
|
|
422
|
+
|
|
423
|
+
:param int control_id: Security control ID
|
|
424
|
+
:param str cci_id: CCI identifier to search for
|
|
425
|
+
:return: Existing CCI instance or None
|
|
426
|
+
:rtype: Optional[CCI]
|
|
427
|
+
"""
|
|
428
|
+
try:
|
|
429
|
+
existing_ccis: List[CCI] = CCI.get_all_by_parent(parent_id=control_id)
|
|
430
|
+
for existing in existing_ccis:
|
|
431
|
+
if existing.uuid == cci_id:
|
|
432
|
+
return existing
|
|
433
|
+
except Exception:
|
|
434
|
+
pass
|
|
435
|
+
return None
|
|
436
|
+
|
|
437
|
+
@staticmethod
|
|
438
|
+
def _create_cci_data(
|
|
439
|
+
cci_id: str, definition: str, user_id: Optional[str], tenant_id: int, current_time: str
|
|
440
|
+
) -> Dict:
|
|
441
|
+
"""
|
|
442
|
+
Create common CCI data structure.
|
|
443
|
+
|
|
444
|
+
:param str cci_id: CCI identifier
|
|
445
|
+
:param str definition: CCI definition
|
|
446
|
+
:param Optional[str] user_id: User ID
|
|
447
|
+
:param int tenant_id: Tenant ID
|
|
448
|
+
:param str current_time: Current timestamp string
|
|
449
|
+
:return: Dictionary with common CCI attributes
|
|
450
|
+
:rtype: Dict
|
|
451
|
+
"""
|
|
452
|
+
return {
|
|
453
|
+
"name": cci_id,
|
|
454
|
+
"description": definition,
|
|
455
|
+
"controlType": "policy",
|
|
456
|
+
"publishDate": current_time,
|
|
457
|
+
"dateLastUpdated": current_time,
|
|
458
|
+
"lastUpdatedById": user_id,
|
|
459
|
+
"isPublic": True,
|
|
460
|
+
"tenantsId": tenant_id,
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
@staticmethod
|
|
464
|
+
def _update_existing_cci(existing_cci: CCI, cci_data: Dict) -> None:
|
|
465
|
+
"""
|
|
466
|
+
Update an existing CCI with new data.
|
|
467
|
+
|
|
468
|
+
:param CCI existing_cci: CCI instance to update
|
|
469
|
+
:param Dict cci_data: Dictionary with CCI attributes
|
|
470
|
+
:rtype: None
|
|
471
|
+
"""
|
|
472
|
+
for key, value in cci_data.items():
|
|
473
|
+
setattr(existing_cci, key, value)
|
|
474
|
+
existing_cci.create_or_update()
|
|
475
|
+
|
|
476
|
+
@staticmethod
|
|
477
|
+
def _create_new_cci(cci_id: str, cci_data: Dict, control_id: int, user_id: Optional[str], current_time: str) -> CCI:
|
|
478
|
+
"""
|
|
479
|
+
Create a new CCI instance.
|
|
480
|
+
|
|
481
|
+
:param str cci_id: CCI identifier
|
|
482
|
+
:param Dict cci_data: Dictionary with common CCI attributes
|
|
483
|
+
:param int control_id: Security control ID
|
|
484
|
+
:param Optional[str] user_id: User ID
|
|
485
|
+
:param str current_time: Current timestamp string
|
|
486
|
+
:return: Created CCI instance
|
|
487
|
+
:rtype: CCI
|
|
488
|
+
"""
|
|
489
|
+
new_cci = CCI(
|
|
490
|
+
uuid=cci_id,
|
|
491
|
+
securityControlId=control_id,
|
|
492
|
+
createdById=user_id,
|
|
493
|
+
dateCreated=current_time,
|
|
494
|
+
**cci_data,
|
|
495
|
+
)
|
|
496
|
+
new_cci.create()
|
|
497
|
+
return new_cci
|
|
498
|
+
|
|
499
|
+
def _process_cci_for_control(
|
|
500
|
+
self, control_id: int, cci_list: List[Dict], user_id: Optional[str], tenant_id: int
|
|
501
|
+
) -> Tuple[int, int]:
|
|
502
|
+
"""
|
|
503
|
+
Process all CCI items for a specific control.
|
|
504
|
+
|
|
505
|
+
:param int control_id: Security control ID
|
|
506
|
+
:param List[Dict] cci_list: List of CCI data dictionaries
|
|
507
|
+
:param Optional[str] user_id: User ID
|
|
508
|
+
:param int tenant_id: Tenant ID
|
|
509
|
+
:return: Tuple of (created_count, updated_count)
|
|
510
|
+
:rtype: Tuple[int, int]
|
|
511
|
+
"""
|
|
512
|
+
created_count = 0
|
|
513
|
+
updated_count = 0
|
|
514
|
+
current_time = datetime.datetime.now().strftime(REGSCALE_DATE_FORMAT)
|
|
515
|
+
|
|
516
|
+
for cci in cci_list:
|
|
517
|
+
cci_id = cci["cci_id"]
|
|
518
|
+
definition = cci["definition"]
|
|
519
|
+
|
|
520
|
+
existing_cci = self._find_existing_cci(control_id, cci_id)
|
|
521
|
+
cci_data = self._create_cci_data(cci_id, definition, user_id, tenant_id, current_time)
|
|
522
|
+
|
|
523
|
+
if existing_cci:
|
|
524
|
+
self._update_existing_cci(existing_cci, cci_data)
|
|
525
|
+
updated_count += 1
|
|
526
|
+
else:
|
|
527
|
+
self._create_new_cci(cci_id, cci_data, control_id, user_id, current_time)
|
|
528
|
+
created_count += 1
|
|
529
|
+
|
|
530
|
+
return created_count, updated_count
|
|
531
|
+
|
|
532
|
+
def map_to_security_controls(self, catalog_id: int = 1) -> Dict[str, int]:
|
|
533
|
+
"""
|
|
534
|
+
Map normalized CCI data to security controls in the database.
|
|
535
|
+
|
|
536
|
+
:param int catalog_id: ID of the catalog containing security controls (default: 1)
|
|
537
|
+
:return: Dictionary with operation statistics
|
|
538
|
+
:rtype: Dict[str, int]
|
|
539
|
+
"""
|
|
540
|
+
if self.verbose:
|
|
541
|
+
logger.info("Mapping CCI data to security controls...")
|
|
542
|
+
|
|
543
|
+
catalog = self._get_catalog(catalog_id)
|
|
544
|
+
security_controls: List[SecurityControl] = SecurityControl.get_all_by_parent(parent_id=catalog.id)
|
|
545
|
+
control_map = {sc.controlId: sc.id for sc in security_controls}
|
|
546
|
+
|
|
547
|
+
user_id, tenant_id = self._get_user_context()
|
|
548
|
+
|
|
549
|
+
created_count = 0
|
|
550
|
+
updated_count = 0
|
|
551
|
+
skipped_count = 0
|
|
552
|
+
|
|
553
|
+
with create_progress_object() as progress:
|
|
554
|
+
logger.info(f"Parsing and mapping {len(self.normalized_cci)} normalized CCI entries...")
|
|
555
|
+
main_task = progress.add_task("Parsing and mapping CCIs...", total=len(self.normalized_cci))
|
|
556
|
+
for main_control, cci_list in self.normalized_cci.items():
|
|
557
|
+
if main_control in control_map:
|
|
558
|
+
control_id = control_map[main_control]
|
|
559
|
+
control_created, control_updated = self._process_cci_for_control(
|
|
560
|
+
control_id, cci_list, user_id, tenant_id
|
|
561
|
+
)
|
|
562
|
+
created_count += control_created
|
|
563
|
+
updated_count += control_updated
|
|
564
|
+
else:
|
|
565
|
+
skipped_count += len(cci_list)
|
|
566
|
+
if self.verbose:
|
|
567
|
+
logger.warning(f"Warning: Control not found for key: {main_control}")
|
|
568
|
+
progress.update(main_task, advance=1)
|
|
569
|
+
|
|
570
|
+
return {
|
|
571
|
+
"created": created_count,
|
|
572
|
+
"updated": updated_count,
|
|
573
|
+
"skipped": skipped_count,
|
|
574
|
+
"total_processed": len(self.normalized_cci),
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
def map_to_control_objectives(self, catalog_id: int = 1) -> Dict[str, int]:
|
|
578
|
+
"""
|
|
579
|
+
Map grouped CCI data to control objectives in the database.
|
|
580
|
+
Updates the otherId field of existing ControlObjective records.
|
|
581
|
+
|
|
582
|
+
:param int catalog_id: ID of the catalog containing control objectives (default: 1)
|
|
583
|
+
:return: Dictionary with operation statistics
|
|
584
|
+
:rtype: Dict[str, int]
|
|
585
|
+
"""
|
|
586
|
+
if self.verbose:
|
|
587
|
+
logger.info("Mapping CCI data to control objectives...")
|
|
588
|
+
|
|
589
|
+
# Fetch all objectives for the catalog
|
|
590
|
+
objectives: List[ControlObjective] = ControlObjective.get_by_catalog(catalog_id=catalog_id)
|
|
591
|
+
|
|
592
|
+
if self.verbose:
|
|
593
|
+
logger.info(f"Found {len(objectives)} objectives in catalog {catalog_id}")
|
|
594
|
+
|
|
595
|
+
objectives_updated = 0
|
|
596
|
+
objectives_skipped = 0
|
|
597
|
+
objectives_not_found = 0
|
|
598
|
+
|
|
599
|
+
with create_progress_object() as progress:
|
|
600
|
+
logger.info(f"Processing {len(objectives)} objectives...")
|
|
601
|
+
task = progress.add_task("Mapping CCIs to objectives...", total=len(objectives))
|
|
602
|
+
|
|
603
|
+
for obj in objectives:
|
|
604
|
+
objective_id = obj.otherId
|
|
605
|
+
|
|
606
|
+
# Skip objectives without proper otherId
|
|
607
|
+
if not objective_id or "_smt" not in objective_id:
|
|
608
|
+
objectives_not_found += 1
|
|
609
|
+
progress.update(task, advance=1)
|
|
610
|
+
continue
|
|
611
|
+
|
|
612
|
+
# Extract just the objective ID part (before any CCIs)
|
|
613
|
+
# Format: "ac-1_smt.a" or "ac-1_smt.a, CCI-000001, CCI-000002"
|
|
614
|
+
objective_id_parts = objective_id.split(",")
|
|
615
|
+
base_objective_id = objective_id_parts[0].strip()
|
|
616
|
+
|
|
617
|
+
# Parse the objective ID
|
|
618
|
+
control_base, part = self.parse_objective_id(base_objective_id)
|
|
619
|
+
|
|
620
|
+
if not control_base:
|
|
621
|
+
objectives_not_found += 1
|
|
622
|
+
progress.update(task, advance=1)
|
|
623
|
+
continue
|
|
624
|
+
|
|
625
|
+
# Find matching CCIs by otherId
|
|
626
|
+
matching_ccis = self.find_matching_ccis(control_base, part, self.cci_grouped_by_index)
|
|
627
|
+
|
|
628
|
+
# Fallback: try matching by name
|
|
629
|
+
if not matching_ccis and obj.name:
|
|
630
|
+
matching_ccis = self.find_matching_ccis_by_name(control_base, obj.name, self.cci_grouped_by_index)
|
|
631
|
+
|
|
632
|
+
if matching_ccis:
|
|
633
|
+
skipped_count, updated_count = self._handle_matching_ccis(
|
|
634
|
+
control_objective=obj,
|
|
635
|
+
matching_ccis=matching_ccis,
|
|
636
|
+
base_objective_id=base_objective_id,
|
|
637
|
+
)
|
|
638
|
+
objectives_skipped += skipped_count
|
|
639
|
+
objectives_updated += updated_count
|
|
640
|
+
else:
|
|
641
|
+
objectives_not_found += 1
|
|
642
|
+
|
|
643
|
+
progress.update(task, advance=1)
|
|
644
|
+
|
|
645
|
+
return {
|
|
646
|
+
"updated": objectives_updated,
|
|
647
|
+
"skipped": objectives_skipped,
|
|
648
|
+
"not_found": objectives_not_found,
|
|
649
|
+
"total_processed": len(objectives),
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
def _handle_matching_ccis(
|
|
653
|
+
self,
|
|
654
|
+
control_objective: ControlObjective,
|
|
655
|
+
matching_ccis: List[str],
|
|
656
|
+
base_objective_id: str,
|
|
657
|
+
) -> Tuple[int, int]:
|
|
658
|
+
"""
|
|
659
|
+
Handle matching CCIs.
|
|
660
|
+
|
|
661
|
+
:param ControlObjective control_objective: ControlObjective instance
|
|
662
|
+
:param List[str] matching_ccis: List of matching CCI IDs
|
|
663
|
+
:param str base_objective_id: Base objective ID
|
|
664
|
+
:return: Tuple of (number of objectives skipped, number of objectives updated)
|
|
665
|
+
:rtype: Tuple[int, int]
|
|
666
|
+
"""
|
|
667
|
+
# Combine all CCI IDs
|
|
668
|
+
all_cci_ids = ", ".join(matching_ccis)
|
|
669
|
+
|
|
670
|
+
# Check for duplicates
|
|
671
|
+
if self.ccis_already_present(control_objective.otherId, all_cci_ids):
|
|
672
|
+
if self.verbose:
|
|
673
|
+
logger.info(f"Skipping {base_objective_id} - CCIs already present")
|
|
674
|
+
return 1, 0
|
|
675
|
+
|
|
676
|
+
# Update the otherId field
|
|
677
|
+
control_objective.otherId = f"{control_objective.otherId}, {all_cci_ids}"
|
|
678
|
+
control_objective.save()
|
|
679
|
+
|
|
680
|
+
if self.verbose:
|
|
681
|
+
logger.info(f"Updated {base_objective_id} with {all_cci_ids}")
|
|
682
|
+
return 0, 1
|
|
683
|
+
|
|
684
|
+
def get_normalized_cci(self) -> Dict[str, List[Dict]]:
|
|
685
|
+
"""
|
|
686
|
+
Get the normalized CCI data.
|
|
687
|
+
|
|
688
|
+
:return: Dictionary of normalized CCI data
|
|
689
|
+
:rtype: Dict[str, List[Dict]]
|
|
690
|
+
"""
|
|
691
|
+
return self.normalized_cci
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
def _load_xml_file(xml_file: str) -> ET.Element:
|
|
695
|
+
"""
|
|
696
|
+
Load and parse XML file.
|
|
697
|
+
|
|
698
|
+
:param str xml_file: Path to XML file
|
|
699
|
+
:return: Root element of parsed XML
|
|
700
|
+
:rtype: ET.Element
|
|
701
|
+
:raises click.ClickException: If XML parsing fails
|
|
702
|
+
"""
|
|
703
|
+
try:
|
|
704
|
+
logger.info(f"Loading XML file: {xml_file}")
|
|
705
|
+
tree = ET.parse(xml_file)
|
|
706
|
+
return tree.getroot()
|
|
707
|
+
except ET.ParseError as e:
|
|
708
|
+
error_and_exit(f"Failed to parse XML file: {e}")
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
def _display_verbose_output(normalized_data: Dict[str, List[Dict]]) -> None:
|
|
712
|
+
"""
|
|
713
|
+
Display detailed normalized CCI data.
|
|
714
|
+
|
|
715
|
+
:param Dict[str, List[Dict]] normalized_data: Dictionary of normalized CCI data
|
|
716
|
+
:rtype: None
|
|
717
|
+
"""
|
|
718
|
+
logger.info("\nNormalized CCI Data:")
|
|
719
|
+
for key, value in normalized_data.items():
|
|
720
|
+
logger.info(f" {key}: {len(value)} CCI items")
|
|
721
|
+
for cci in value:
|
|
722
|
+
definition_preview = cci["definition"][:100] + "..." if len(cci["definition"]) > 100 else cci["definition"]
|
|
723
|
+
logger.info(f" - {cci['cci_id']}: {definition_preview}")
|
|
724
|
+
|
|
725
|
+
|
|
726
|
+
def _display_results(stats: Dict[str, int]) -> None:
|
|
727
|
+
"""
|
|
728
|
+
Display database operation results.
|
|
729
|
+
|
|
730
|
+
:param Dict[str, int] stats: Dictionary with operation statistics
|
|
731
|
+
:rtype: None
|
|
732
|
+
"""
|
|
733
|
+
logger.info(
|
|
734
|
+
f"[green]\nDatabase operations completed:"
|
|
735
|
+
f"[green]\n - Created: {stats['created']}"
|
|
736
|
+
f"[green]\n - Updated: {stats['updated']}"
|
|
737
|
+
f"[green]\n - Skipped: {stats['skipped']}"
|
|
738
|
+
f"[green]\n - Total processed: {stats['total_processed']}",
|
|
739
|
+
)
|
|
740
|
+
|
|
741
|
+
|
|
742
|
+
def _display_objective_results(stats: Dict[str, int]) -> None:
|
|
743
|
+
"""
|
|
744
|
+
Display control objective mapping results.
|
|
745
|
+
|
|
746
|
+
:param Dict[str, int] stats: Dictionary with operation statistics
|
|
747
|
+
:rtype: None
|
|
748
|
+
"""
|
|
749
|
+
logger.info(
|
|
750
|
+
f"[green]\nControl objective operations completed:"
|
|
751
|
+
f"[green]\n - Updated: {stats['updated']}"
|
|
752
|
+
f"[green]\n - Skipped: {stats['skipped']}"
|
|
753
|
+
f"[green]\n - Not found: {stats['not_found']}"
|
|
754
|
+
f"[green]\n - Total processed: {stats['total_processed']}",
|
|
755
|
+
)
|
|
756
|
+
|
|
757
|
+
|
|
758
|
+
def _process_cci_import(
|
|
759
|
+
importer: CCIImporter, dry_run: bool, verbose: bool, catalog_id: int, disable_objectives: bool = False
|
|
760
|
+
) -> None:
|
|
761
|
+
"""
|
|
762
|
+
Process CCI import with optional database operations.
|
|
763
|
+
|
|
764
|
+
:param CCIImporter importer: CCIImporter instance
|
|
765
|
+
:param bool dry_run: Whether to skip database operations
|
|
766
|
+
:param bool verbose: Whether to display verbose output
|
|
767
|
+
:param int catalog_id: ID of the catalog containing security controls
|
|
768
|
+
:param bool disable_objectives: Whether to disable mapping to control objectives
|
|
769
|
+
:rtype: None
|
|
770
|
+
"""
|
|
771
|
+
importer.parse_cci()
|
|
772
|
+
normalized_data = importer.get_normalized_cci()
|
|
773
|
+
|
|
774
|
+
logger.info(f"[green]Successfully parsed {len(normalized_data)} normalized CCI entries[/green]")
|
|
775
|
+
|
|
776
|
+
if verbose:
|
|
777
|
+
_display_verbose_output(normalized_data)
|
|
778
|
+
|
|
779
|
+
if not dry_run:
|
|
780
|
+
# Map to SecurityControl (existing functionality)
|
|
781
|
+
stats = importer.map_to_security_controls(catalog_id)
|
|
782
|
+
_display_results(stats)
|
|
783
|
+
|
|
784
|
+
# Map to ControlObjective (new functionality)
|
|
785
|
+
if not disable_objectives:
|
|
786
|
+
logger.info("\n[cyan]Mapping CCIs to control objectives...[/cyan]")
|
|
787
|
+
obj_stats = importer.map_to_control_objectives(catalog_id)
|
|
788
|
+
_display_objective_results(obj_stats)
|
|
789
|
+
else:
|
|
790
|
+
logger.info("\n[yellow]DRY RUN MODE: No database changes were made[/yellow]")
|
|
791
|
+
|
|
792
|
+
|
|
793
|
+
@click.command(name="cci_importer")
|
|
794
|
+
@click.option(
|
|
795
|
+
"--xml_file", "-f", type=click.Path(exists=True), default=None, required=False, help="Path to the CCI XML file."
|
|
796
|
+
)
|
|
797
|
+
@click.option("--dry-run", "-d", is_flag=True, help="Parse and display normalized data without saving to database")
|
|
798
|
+
@click.option("--verbose", "-v", is_flag=True, help="Display detailed output including all normalized CCI data")
|
|
799
|
+
@click.option(
|
|
800
|
+
"--nist-version", "-n", type=click.Choice(["4", "5"]), default="5", help="NIST 800-53 Revision version (default: 5)"
|
|
801
|
+
)
|
|
802
|
+
@click.option(
|
|
803
|
+
"--catalog-id", "-c", type=click.INT, default=1, help="ID of the catalog containing security controls (default: 1)"
|
|
804
|
+
)
|
|
805
|
+
@click.option(
|
|
806
|
+
"--disable-objectives",
|
|
807
|
+
"-o",
|
|
808
|
+
is_flag=True,
|
|
809
|
+
help="Disable mapping CCIs to control objectives (updates otherId field)",
|
|
810
|
+
)
|
|
811
|
+
def cci_importer(
|
|
812
|
+
xml_file: str, dry_run: bool, verbose: bool, nist_version: str, catalog_id: int, disable_objectives: bool
|
|
813
|
+
) -> None:
|
|
814
|
+
"""Import CCI data from XML files and map to security controls and/or objectives.
|
|
815
|
+
|
|
816
|
+
By default, maps CCIs to SecurityControl entities. Use --disable-objectives flag
|
|
817
|
+
to also update ControlObjective.otherId fields with CCI mappings.
|
|
818
|
+
|
|
819
|
+
If no XML file is specified, defaults to packaged CCI_List.xml.
|
|
820
|
+
"""
|
|
821
|
+
|
|
822
|
+
try:
|
|
823
|
+
if not xml_file:
|
|
824
|
+
import importlib.resources as pkg_resources
|
|
825
|
+
from regscale.models import integration_models
|
|
826
|
+
|
|
827
|
+
files = pkg_resources.files(integration_models)
|
|
828
|
+
cci_path = files / "CCI_List.xml"
|
|
829
|
+
xml_file = str(cci_path)
|
|
830
|
+
root = _load_xml_file(xml_file)
|
|
831
|
+
importer = CCIImporter(root, version=nist_version, verbose=verbose)
|
|
832
|
+
_process_cci_import(importer, dry_run, verbose, catalog_id, disable_objectives)
|
|
833
|
+
except Exception as e:
|
|
834
|
+
error_and_exit(f"Unexpected error: {e}")
|