regscale-cli 6.16.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of regscale-cli might be problematic. Click here for more details.
- regscale/__init__.py +1 -0
- regscale/airflow/__init__.py +9 -0
- regscale/airflow/azure/__init__.py +9 -0
- regscale/airflow/azure/cli.py +89 -0
- regscale/airflow/azure/upload_dags.py +116 -0
- regscale/airflow/click_dags.py +127 -0
- regscale/airflow/click_mixins.py +82 -0
- regscale/airflow/config.py +25 -0
- regscale/airflow/factories/__init__.py +0 -0
- regscale/airflow/factories/connections.py +58 -0
- regscale/airflow/factories/workflows.py +78 -0
- regscale/airflow/hierarchy.py +88 -0
- regscale/airflow/operators/__init__.py +0 -0
- regscale/airflow/operators/click.py +36 -0
- regscale/airflow/sensors/__init__.py +0 -0
- regscale/airflow/sensors/sql.py +107 -0
- regscale/airflow/sessions/__init__.py +0 -0
- regscale/airflow/sessions/sql/__init__.py +3 -0
- regscale/airflow/sessions/sql/queries.py +64 -0
- regscale/airflow/sessions/sql/sql_server_queries.py +248 -0
- regscale/airflow/tasks/__init__.py +0 -0
- regscale/airflow/tasks/branches.py +22 -0
- regscale/airflow/tasks/cli.py +116 -0
- regscale/airflow/tasks/click.py +73 -0
- regscale/airflow/tasks/debugging.py +9 -0
- regscale/airflow/tasks/groups.py +116 -0
- regscale/airflow/tasks/init.py +60 -0
- regscale/airflow/tasks/states.py +47 -0
- regscale/airflow/tasks/workflows.py +36 -0
- regscale/ansible/__init__.py +9 -0
- regscale/core/__init__.py +0 -0
- regscale/core/app/__init__.py +3 -0
- regscale/core/app/api.py +571 -0
- regscale/core/app/application.py +665 -0
- regscale/core/app/internal/__init__.py +136 -0
- regscale/core/app/internal/admin_actions.py +230 -0
- regscale/core/app/internal/assessments_editor.py +873 -0
- regscale/core/app/internal/catalog.py +316 -0
- regscale/core/app/internal/comparison.py +459 -0
- regscale/core/app/internal/control_editor.py +571 -0
- regscale/core/app/internal/encrypt.py +79 -0
- regscale/core/app/internal/evidence.py +1240 -0
- regscale/core/app/internal/file_uploads.py +151 -0
- regscale/core/app/internal/healthcheck.py +66 -0
- regscale/core/app/internal/login.py +305 -0
- regscale/core/app/internal/migrations.py +240 -0
- regscale/core/app/internal/model_editor.py +1701 -0
- regscale/core/app/internal/poam_editor.py +632 -0
- regscale/core/app/internal/workflow.py +105 -0
- regscale/core/app/logz.py +74 -0
- regscale/core/app/utils/XMLIR.py +258 -0
- regscale/core/app/utils/__init__.py +0 -0
- regscale/core/app/utils/api_handler.py +358 -0
- regscale/core/app/utils/app_utils.py +1110 -0
- regscale/core/app/utils/catalog_utils/__init__.py +0 -0
- regscale/core/app/utils/catalog_utils/common.py +91 -0
- regscale/core/app/utils/catalog_utils/compare_catalog.py +193 -0
- regscale/core/app/utils/catalog_utils/diagnostic_catalog.py +97 -0
- regscale/core/app/utils/catalog_utils/download_catalog.py +103 -0
- regscale/core/app/utils/catalog_utils/update_catalog.py +718 -0
- regscale/core/app/utils/catalog_utils/update_catalog_v2.py +1378 -0
- regscale/core/app/utils/catalog_utils/update_catalog_v3.py +1272 -0
- regscale/core/app/utils/catalog_utils/update_plans.py +334 -0
- regscale/core/app/utils/file_utils.py +238 -0
- regscale/core/app/utils/parser_utils.py +81 -0
- regscale/core/app/utils/pickle_file_handler.py +57 -0
- regscale/core/app/utils/regscale_utils.py +319 -0
- regscale/core/app/utils/report_utils.py +119 -0
- regscale/core/app/utils/variables.py +226 -0
- regscale/core/decorators.py +31 -0
- regscale/core/lazy_group.py +65 -0
- regscale/core/login.py +63 -0
- regscale/core/server/__init__.py +0 -0
- regscale/core/server/flask_api.py +473 -0
- regscale/core/server/helpers.py +373 -0
- regscale/core/server/rest.py +64 -0
- regscale/core/server/static/css/bootstrap.css +6030 -0
- regscale/core/server/static/css/bootstrap.min.css +6 -0
- regscale/core/server/static/css/main.css +176 -0
- regscale/core/server/static/images/regscale-cli.svg +49 -0
- regscale/core/server/static/images/regscale.svg +38 -0
- regscale/core/server/templates/base.html +74 -0
- regscale/core/server/templates/index.html +43 -0
- regscale/core/server/templates/login.html +28 -0
- regscale/core/server/templates/make_base64.html +22 -0
- regscale/core/server/templates/upload_STIG.html +109 -0
- regscale/core/server/templates/upload_STIG_result.html +26 -0
- regscale/core/server/templates/upload_ssp.html +144 -0
- regscale/core/server/templates/upload_ssp_result.html +128 -0
- regscale/core/static/__init__.py +0 -0
- regscale/core/static/regex.py +14 -0
- regscale/core/utils/__init__.py +117 -0
- regscale/core/utils/click_utils.py +13 -0
- regscale/core/utils/date.py +238 -0
- regscale/core/utils/graphql.py +254 -0
- regscale/core/utils/urls.py +23 -0
- regscale/dev/__init__.py +6 -0
- regscale/dev/analysis.py +454 -0
- regscale/dev/cli.py +235 -0
- regscale/dev/code_gen.py +492 -0
- regscale/dev/dirs.py +69 -0
- regscale/dev/docs.py +384 -0
- regscale/dev/monitoring.py +26 -0
- regscale/dev/profiling.py +216 -0
- regscale/exceptions/__init__.py +4 -0
- regscale/exceptions/license_exception.py +7 -0
- regscale/exceptions/validation_exception.py +9 -0
- regscale/integrations/__init__.py +1 -0
- regscale/integrations/commercial/__init__.py +486 -0
- regscale/integrations/commercial/ad.py +433 -0
- regscale/integrations/commercial/amazon/__init__.py +0 -0
- regscale/integrations/commercial/amazon/common.py +106 -0
- regscale/integrations/commercial/aqua/__init__.py +0 -0
- regscale/integrations/commercial/aqua/aqua.py +91 -0
- regscale/integrations/commercial/aws/__init__.py +6 -0
- regscale/integrations/commercial/aws/cli.py +322 -0
- regscale/integrations/commercial/aws/inventory/__init__.py +110 -0
- regscale/integrations/commercial/aws/inventory/base.py +64 -0
- regscale/integrations/commercial/aws/inventory/resources/__init__.py +19 -0
- regscale/integrations/commercial/aws/inventory/resources/compute.py +234 -0
- regscale/integrations/commercial/aws/inventory/resources/containers.py +113 -0
- regscale/integrations/commercial/aws/inventory/resources/database.py +101 -0
- regscale/integrations/commercial/aws/inventory/resources/integration.py +237 -0
- regscale/integrations/commercial/aws/inventory/resources/networking.py +253 -0
- regscale/integrations/commercial/aws/inventory/resources/security.py +240 -0
- regscale/integrations/commercial/aws/inventory/resources/storage.py +91 -0
- regscale/integrations/commercial/aws/scanner.py +823 -0
- regscale/integrations/commercial/azure/__init__.py +0 -0
- regscale/integrations/commercial/azure/common.py +32 -0
- regscale/integrations/commercial/azure/intune.py +488 -0
- regscale/integrations/commercial/azure/scanner.py +49 -0
- regscale/integrations/commercial/burp.py +78 -0
- regscale/integrations/commercial/cpe.py +144 -0
- regscale/integrations/commercial/crowdstrike.py +1117 -0
- regscale/integrations/commercial/defender.py +1511 -0
- regscale/integrations/commercial/dependabot.py +210 -0
- regscale/integrations/commercial/durosuite/__init__.py +0 -0
- regscale/integrations/commercial/durosuite/api.py +1546 -0
- regscale/integrations/commercial/durosuite/process_devices.py +101 -0
- regscale/integrations/commercial/durosuite/scanner.py +637 -0
- regscale/integrations/commercial/durosuite/variables.py +21 -0
- regscale/integrations/commercial/ecr.py +90 -0
- regscale/integrations/commercial/gcp/__init__.py +237 -0
- regscale/integrations/commercial/gcp/auth.py +96 -0
- regscale/integrations/commercial/gcp/control_tests.py +238 -0
- regscale/integrations/commercial/gcp/variables.py +18 -0
- regscale/integrations/commercial/gitlab.py +332 -0
- regscale/integrations/commercial/grype.py +165 -0
- regscale/integrations/commercial/ibm.py +90 -0
- regscale/integrations/commercial/import_all/__init__.py +0 -0
- regscale/integrations/commercial/import_all/import_all_cmd.py +467 -0
- regscale/integrations/commercial/import_all/scan_file_fingerprints.json +27 -0
- regscale/integrations/commercial/jira.py +1046 -0
- regscale/integrations/commercial/mappings/__init__.py +0 -0
- regscale/integrations/commercial/mappings/csf_controls.json +713 -0
- regscale/integrations/commercial/mappings/nist_800_53_r5_controls.json +1516 -0
- regscale/integrations/commercial/nessus/__init__.py +0 -0
- regscale/integrations/commercial/nessus/nessus_utils.py +429 -0
- regscale/integrations/commercial/nessus/scanner.py +416 -0
- regscale/integrations/commercial/nexpose.py +90 -0
- regscale/integrations/commercial/okta.py +798 -0
- regscale/integrations/commercial/opentext/__init__.py +0 -0
- regscale/integrations/commercial/opentext/click.py +99 -0
- regscale/integrations/commercial/opentext/scanner.py +143 -0
- regscale/integrations/commercial/prisma.py +91 -0
- regscale/integrations/commercial/qualys.py +1462 -0
- regscale/integrations/commercial/salesforce.py +980 -0
- regscale/integrations/commercial/sap/__init__.py +0 -0
- regscale/integrations/commercial/sap/click.py +31 -0
- regscale/integrations/commercial/sap/sysdig/__init__.py +0 -0
- regscale/integrations/commercial/sap/sysdig/click.py +57 -0
- regscale/integrations/commercial/sap/sysdig/sysdig_scanner.py +190 -0
- regscale/integrations/commercial/sap/tenable/__init__.py +0 -0
- regscale/integrations/commercial/sap/tenable/click.py +49 -0
- regscale/integrations/commercial/sap/tenable/scanner.py +196 -0
- regscale/integrations/commercial/servicenow.py +1756 -0
- regscale/integrations/commercial/sicura/__init__.py +0 -0
- regscale/integrations/commercial/sicura/api.py +855 -0
- regscale/integrations/commercial/sicura/commands.py +73 -0
- regscale/integrations/commercial/sicura/scanner.py +481 -0
- regscale/integrations/commercial/sicura/variables.py +16 -0
- regscale/integrations/commercial/snyk.py +90 -0
- regscale/integrations/commercial/sonarcloud.py +260 -0
- regscale/integrations/commercial/sqlserver.py +369 -0
- regscale/integrations/commercial/stig_mapper_integration/__init__.py +0 -0
- regscale/integrations/commercial/stig_mapper_integration/click_commands.py +38 -0
- regscale/integrations/commercial/stig_mapper_integration/mapping_engine.py +353 -0
- regscale/integrations/commercial/stigv2/__init__.py +0 -0
- regscale/integrations/commercial/stigv2/ckl_parser.py +349 -0
- regscale/integrations/commercial/stigv2/click_commands.py +95 -0
- regscale/integrations/commercial/stigv2/stig_integration.py +202 -0
- regscale/integrations/commercial/synqly/__init__.py +0 -0
- regscale/integrations/commercial/synqly/assets.py +46 -0
- regscale/integrations/commercial/synqly/ticketing.py +132 -0
- regscale/integrations/commercial/synqly/vulnerabilities.py +223 -0
- regscale/integrations/commercial/synqly_jira.py +840 -0
- regscale/integrations/commercial/tenablev2/__init__.py +0 -0
- regscale/integrations/commercial/tenablev2/authenticate.py +31 -0
- regscale/integrations/commercial/tenablev2/click.py +1584 -0
- regscale/integrations/commercial/tenablev2/scanner.py +504 -0
- regscale/integrations/commercial/tenablev2/stig_parsers.py +140 -0
- regscale/integrations/commercial/tenablev2/utils.py +78 -0
- regscale/integrations/commercial/tenablev2/variables.py +17 -0
- regscale/integrations/commercial/trivy.py +162 -0
- regscale/integrations/commercial/veracode.py +96 -0
- regscale/integrations/commercial/wizv2/WizDataMixin.py +97 -0
- regscale/integrations/commercial/wizv2/__init__.py +0 -0
- regscale/integrations/commercial/wizv2/click.py +429 -0
- regscale/integrations/commercial/wizv2/constants.py +1001 -0
- regscale/integrations/commercial/wizv2/issue.py +361 -0
- regscale/integrations/commercial/wizv2/models.py +112 -0
- regscale/integrations/commercial/wizv2/parsers.py +339 -0
- regscale/integrations/commercial/wizv2/sbom.py +115 -0
- regscale/integrations/commercial/wizv2/scanner.py +416 -0
- regscale/integrations/commercial/wizv2/utils.py +796 -0
- regscale/integrations/commercial/wizv2/variables.py +39 -0
- regscale/integrations/commercial/wizv2/wiz_auth.py +159 -0
- regscale/integrations/commercial/xray.py +91 -0
- regscale/integrations/integration/__init__.py +2 -0
- regscale/integrations/integration/integration.py +26 -0
- regscale/integrations/integration/inventory.py +17 -0
- regscale/integrations/integration/issue.py +100 -0
- regscale/integrations/integration_override.py +149 -0
- regscale/integrations/public/__init__.py +103 -0
- regscale/integrations/public/cisa.py +641 -0
- regscale/integrations/public/criticality_updater.py +70 -0
- regscale/integrations/public/emass.py +411 -0
- regscale/integrations/public/emass_slcm_import.py +697 -0
- regscale/integrations/public/fedramp/__init__.py +0 -0
- regscale/integrations/public/fedramp/appendix_parser.py +548 -0
- regscale/integrations/public/fedramp/click.py +479 -0
- regscale/integrations/public/fedramp/components.py +714 -0
- regscale/integrations/public/fedramp/docx_parser.py +259 -0
- regscale/integrations/public/fedramp/fedramp_cis_crm.py +1124 -0
- regscale/integrations/public/fedramp/fedramp_common.py +3181 -0
- regscale/integrations/public/fedramp/fedramp_docx.py +388 -0
- regscale/integrations/public/fedramp/fedramp_five.py +2343 -0
- regscale/integrations/public/fedramp/fedramp_traversal.py +138 -0
- regscale/integrations/public/fedramp/import_fedramp_r4_ssp.py +279 -0
- regscale/integrations/public/fedramp/import_workbook.py +495 -0
- regscale/integrations/public/fedramp/inventory_items.py +244 -0
- regscale/integrations/public/fedramp/mappings/__init__.py +0 -0
- regscale/integrations/public/fedramp/mappings/fedramp_r4_parts.json +7388 -0
- regscale/integrations/public/fedramp/mappings/fedramp_r5_params.json +8636 -0
- regscale/integrations/public/fedramp/mappings/fedramp_r5_parts.json +9605 -0
- regscale/integrations/public/fedramp/mappings/system_roles.py +34 -0
- regscale/integrations/public/fedramp/mappings/user.py +175 -0
- regscale/integrations/public/fedramp/mappings/values.py +141 -0
- regscale/integrations/public/fedramp/markdown_parser.py +150 -0
- regscale/integrations/public/fedramp/metadata.py +689 -0
- regscale/integrations/public/fedramp/models/__init__.py +59 -0
- regscale/integrations/public/fedramp/models/leveraged_auth_new.py +168 -0
- regscale/integrations/public/fedramp/models/poam_importer.py +522 -0
- regscale/integrations/public/fedramp/parts_mapper.py +107 -0
- regscale/integrations/public/fedramp/poam/__init__.py +0 -0
- regscale/integrations/public/fedramp/poam/scanner.py +851 -0
- regscale/integrations/public/fedramp/properties.py +201 -0
- regscale/integrations/public/fedramp/reporting.py +84 -0
- regscale/integrations/public/fedramp/resources.py +496 -0
- regscale/integrations/public/fedramp/rosetta.py +110 -0
- regscale/integrations/public/fedramp/ssp_logger.py +87 -0
- regscale/integrations/public/fedramp/system_characteristics.py +922 -0
- regscale/integrations/public/fedramp/system_control_implementations.py +582 -0
- regscale/integrations/public/fedramp/system_implementation.py +190 -0
- regscale/integrations/public/fedramp/xml_utils.py +87 -0
- regscale/integrations/public/nist_catalog.py +275 -0
- regscale/integrations/public/oscal.py +1946 -0
- regscale/integrations/public/otx.py +169 -0
- regscale/integrations/scanner_integration.py +2692 -0
- regscale/integrations/variables.py +25 -0
- regscale/models/__init__.py +7 -0
- regscale/models/app_models/__init__.py +5 -0
- regscale/models/app_models/catalog_compare.py +213 -0
- regscale/models/app_models/click.py +252 -0
- regscale/models/app_models/datetime_encoder.py +21 -0
- regscale/models/app_models/import_validater.py +321 -0
- regscale/models/app_models/mapping.py +260 -0
- regscale/models/app_models/pipeline.py +37 -0
- regscale/models/click_models.py +413 -0
- regscale/models/config.py +154 -0
- regscale/models/email_style.css +67 -0
- regscale/models/hierarchy.py +8 -0
- regscale/models/inspect_models.py +79 -0
- regscale/models/integration_models/__init__.py +0 -0
- regscale/models/integration_models/amazon_models/__init__.py +0 -0
- regscale/models/integration_models/amazon_models/inspector.py +262 -0
- regscale/models/integration_models/amazon_models/inspector_scan.py +206 -0
- regscale/models/integration_models/aqua.py +247 -0
- regscale/models/integration_models/azure_alerts.py +255 -0
- regscale/models/integration_models/base64.py +23 -0
- regscale/models/integration_models/burp.py +433 -0
- regscale/models/integration_models/burp_models.py +128 -0
- regscale/models/integration_models/cisa_kev_data.json +19333 -0
- regscale/models/integration_models/defender_data.py +93 -0
- regscale/models/integration_models/defenderimport.py +143 -0
- regscale/models/integration_models/drf.py +443 -0
- regscale/models/integration_models/ecr_models/__init__.py +0 -0
- regscale/models/integration_models/ecr_models/data.py +69 -0
- regscale/models/integration_models/ecr_models/ecr.py +239 -0
- regscale/models/integration_models/flat_file_importer.py +1079 -0
- regscale/models/integration_models/grype_import.py +247 -0
- regscale/models/integration_models/ibm.py +126 -0
- regscale/models/integration_models/implementation_results.py +85 -0
- regscale/models/integration_models/nexpose.py +140 -0
- regscale/models/integration_models/prisma.py +202 -0
- regscale/models/integration_models/qualys.py +720 -0
- regscale/models/integration_models/qualys_scanner.py +160 -0
- regscale/models/integration_models/sbom/__init__.py +0 -0
- regscale/models/integration_models/sbom/cyclone_dx.py +139 -0
- regscale/models/integration_models/send_reminders.py +620 -0
- regscale/models/integration_models/snyk.py +155 -0
- regscale/models/integration_models/synqly_models/__init__.py +0 -0
- regscale/models/integration_models/synqly_models/capabilities.json +1 -0
- regscale/models/integration_models/synqly_models/connector_types.py +22 -0
- regscale/models/integration_models/synqly_models/connectors/__init__.py +7 -0
- regscale/models/integration_models/synqly_models/connectors/assets.py +97 -0
- regscale/models/integration_models/synqly_models/connectors/ticketing.py +583 -0
- regscale/models/integration_models/synqly_models/connectors/vulnerabilities.py +169 -0
- regscale/models/integration_models/synqly_models/ocsf_mapper.py +331 -0
- regscale/models/integration_models/synqly_models/param.py +72 -0
- regscale/models/integration_models/synqly_models/synqly_model.py +733 -0
- regscale/models/integration_models/synqly_models/tenants.py +39 -0
- regscale/models/integration_models/tenable_models/__init__.py +0 -0
- regscale/models/integration_models/tenable_models/integration.py +187 -0
- regscale/models/integration_models/tenable_models/models.py +513 -0
- regscale/models/integration_models/trivy_import.py +231 -0
- regscale/models/integration_models/veracode.py +217 -0
- regscale/models/integration_models/xray.py +135 -0
- regscale/models/locking.py +100 -0
- regscale/models/platform.py +110 -0
- regscale/models/regscale_models/__init__.py +67 -0
- regscale/models/regscale_models/assessment.py +570 -0
- regscale/models/regscale_models/assessment_plan.py +52 -0
- regscale/models/regscale_models/asset.py +567 -0
- regscale/models/regscale_models/asset_mapping.py +190 -0
- regscale/models/regscale_models/case.py +42 -0
- regscale/models/regscale_models/catalog.py +261 -0
- regscale/models/regscale_models/cci.py +46 -0
- regscale/models/regscale_models/change.py +167 -0
- regscale/models/regscale_models/checklist.py +372 -0
- regscale/models/regscale_models/comment.py +49 -0
- regscale/models/regscale_models/compliance_settings.py +112 -0
- regscale/models/regscale_models/component.py +412 -0
- regscale/models/regscale_models/component_mapping.py +65 -0
- regscale/models/regscale_models/control.py +38 -0
- regscale/models/regscale_models/control_implementation.py +1128 -0
- regscale/models/regscale_models/control_objective.py +261 -0
- regscale/models/regscale_models/control_parameter.py +100 -0
- regscale/models/regscale_models/control_test.py +34 -0
- regscale/models/regscale_models/control_test_plan.py +75 -0
- regscale/models/regscale_models/control_test_result.py +52 -0
- regscale/models/regscale_models/custom_field.py +245 -0
- regscale/models/regscale_models/data.py +109 -0
- regscale/models/regscale_models/data_center.py +40 -0
- regscale/models/regscale_models/deviation.py +203 -0
- regscale/models/regscale_models/email.py +97 -0
- regscale/models/regscale_models/evidence.py +47 -0
- regscale/models/regscale_models/evidence_mapping.py +40 -0
- regscale/models/regscale_models/facility.py +59 -0
- regscale/models/regscale_models/file.py +382 -0
- regscale/models/regscale_models/filetag.py +37 -0
- regscale/models/regscale_models/form_field_value.py +94 -0
- regscale/models/regscale_models/group.py +169 -0
- regscale/models/regscale_models/implementation_objective.py +335 -0
- regscale/models/regscale_models/implementation_option.py +275 -0
- regscale/models/regscale_models/implementation_role.py +33 -0
- regscale/models/regscale_models/incident.py +177 -0
- regscale/models/regscale_models/interconnection.py +43 -0
- regscale/models/regscale_models/issue.py +1176 -0
- regscale/models/regscale_models/leveraged_authorization.py +125 -0
- regscale/models/regscale_models/line_of_inquiry.py +52 -0
- regscale/models/regscale_models/link.py +205 -0
- regscale/models/regscale_models/meta_data.py +64 -0
- regscale/models/regscale_models/mixins/__init__.py +0 -0
- regscale/models/regscale_models/mixins/parent_cache.py +124 -0
- regscale/models/regscale_models/module.py +224 -0
- regscale/models/regscale_models/modules.py +191 -0
- regscale/models/regscale_models/objective.py +14 -0
- regscale/models/regscale_models/parameter.py +87 -0
- regscale/models/regscale_models/ports_protocol.py +81 -0
- regscale/models/regscale_models/privacy.py +89 -0
- regscale/models/regscale_models/profile.py +50 -0
- regscale/models/regscale_models/profile_link.py +68 -0
- regscale/models/regscale_models/profile_mapping.py +124 -0
- regscale/models/regscale_models/project.py +63 -0
- regscale/models/regscale_models/property.py +278 -0
- regscale/models/regscale_models/question.py +85 -0
- regscale/models/regscale_models/questionnaire.py +87 -0
- regscale/models/regscale_models/questionnaire_instance.py +177 -0
- regscale/models/regscale_models/rbac.py +132 -0
- regscale/models/regscale_models/reference.py +86 -0
- regscale/models/regscale_models/regscale_model.py +1643 -0
- regscale/models/regscale_models/requirement.py +29 -0
- regscale/models/regscale_models/risk.py +274 -0
- regscale/models/regscale_models/sbom.py +54 -0
- regscale/models/regscale_models/scan_history.py +436 -0
- regscale/models/regscale_models/search.py +53 -0
- regscale/models/regscale_models/security_control.py +132 -0
- regscale/models/regscale_models/security_plan.py +204 -0
- regscale/models/regscale_models/software_inventory.py +159 -0
- regscale/models/regscale_models/stake_holder.py +64 -0
- regscale/models/regscale_models/stig.py +647 -0
- regscale/models/regscale_models/supply_chain.py +152 -0
- regscale/models/regscale_models/system_role.py +188 -0
- regscale/models/regscale_models/system_role_external_assignment.py +40 -0
- regscale/models/regscale_models/tag.py +37 -0
- regscale/models/regscale_models/tag_mapping.py +19 -0
- regscale/models/regscale_models/task.py +133 -0
- regscale/models/regscale_models/threat.py +196 -0
- regscale/models/regscale_models/user.py +175 -0
- regscale/models/regscale_models/user_group.py +55 -0
- regscale/models/regscale_models/vulnerability.py +242 -0
- regscale/models/regscale_models/vulnerability_mapping.py +162 -0
- regscale/models/regscale_models/workflow.py +55 -0
- regscale/models/regscale_models/workflow_action.py +34 -0
- regscale/models/regscale_models/workflow_instance.py +269 -0
- regscale/models/regscale_models/workflow_instance_step.py +114 -0
- regscale/models/regscale_models/workflow_template.py +58 -0
- regscale/models/regscale_models/workflow_template_step.py +45 -0
- regscale/regscale.py +815 -0
- regscale/utils/__init__.py +7 -0
- regscale/utils/b64conversion.py +14 -0
- regscale/utils/click_utils.py +118 -0
- regscale/utils/decorators.py +48 -0
- regscale/utils/dict_utils.py +59 -0
- regscale/utils/files.py +79 -0
- regscale/utils/fxns.py +30 -0
- regscale/utils/graphql_client.py +113 -0
- regscale/utils/lists.py +16 -0
- regscale/utils/numbers.py +12 -0
- regscale/utils/shell.py +148 -0
- regscale/utils/string.py +121 -0
- regscale/utils/synqly_utils.py +165 -0
- regscale/utils/threading/__init__.py +8 -0
- regscale/utils/threading/threadhandler.py +131 -0
- regscale/utils/threading/threadsafe_counter.py +47 -0
- regscale/utils/threading/threadsafe_dict.py +242 -0
- regscale/utils/threading/threadsafe_list.py +83 -0
- regscale/utils/version.py +104 -0
- regscale/validation/__init__.py +0 -0
- regscale/validation/address.py +37 -0
- regscale/validation/record.py +48 -0
- regscale/visualization/__init__.py +5 -0
- regscale/visualization/click.py +34 -0
- regscale_cli-6.16.0.0.dist-info/LICENSE +21 -0
- regscale_cli-6.16.0.0.dist-info/METADATA +659 -0
- regscale_cli-6.16.0.0.dist-info/RECORD +481 -0
- regscale_cli-6.16.0.0.dist-info/WHEEL +5 -0
- regscale_cli-6.16.0.0.dist-info/entry_points.txt +6 -0
- regscale_cli-6.16.0.0.dist-info/top_level.txt +2 -0
- tests/fixtures/__init__.py +2 -0
- tests/fixtures/api.py +87 -0
- tests/fixtures/models.py +91 -0
- tests/fixtures/test_fixture.py +144 -0
- tests/mocks/__init__.py +0 -0
- tests/mocks/objects.py +3 -0
- tests/mocks/response.py +32 -0
- tests/mocks/xml.py +13 -0
- tests/regscale/__init__.py +0 -0
- tests/regscale/core/__init__.py +0 -0
- tests/regscale/core/test_api.py +232 -0
- tests/regscale/core/test_app.py +406 -0
- tests/regscale/core/test_login.py +37 -0
- tests/regscale/core/test_logz.py +66 -0
- tests/regscale/core/test_sbom_generator.py +87 -0
- tests/regscale/core/test_validation_utils.py +163 -0
- tests/regscale/core/test_version.py +78 -0
- tests/regscale/models/__init__.py +0 -0
- tests/regscale/models/test_asset.py +71 -0
- tests/regscale/models/test_config.py +26 -0
- tests/regscale/models/test_control_implementation.py +27 -0
- tests/regscale/models/test_import.py +97 -0
- tests/regscale/models/test_issue.py +36 -0
- tests/regscale/models/test_mapping.py +52 -0
- tests/regscale/models/test_platform.py +31 -0
- tests/regscale/models/test_regscale_model.py +346 -0
- tests/regscale/models/test_report.py +32 -0
- tests/regscale/models/test_tenable_integrations.py +118 -0
- tests/regscale/models/test_user_model.py +121 -0
- tests/regscale/test_about.py +19 -0
- tests/regscale/test_authorization.py +65 -0
|
@@ -0,0 +1,1462 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""Integrates Qualys assets and vulnerabilities into RegScale CLI"""
|
|
4
|
+
import os
|
|
5
|
+
import pprint
|
|
6
|
+
import traceback
|
|
7
|
+
from asyncio import sleep
|
|
8
|
+
from datetime import datetime, timedelta, timezone
|
|
9
|
+
from json import JSONDecodeError
|
|
10
|
+
from typing import Any, Optional, Tuple, Union
|
|
11
|
+
from urllib.parse import urljoin
|
|
12
|
+
|
|
13
|
+
import click
|
|
14
|
+
import requests
|
|
15
|
+
import xmltodict
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from requests import Session
|
|
18
|
+
from rich.progress import TaskID
|
|
19
|
+
|
|
20
|
+
from regscale.core.app.logz import create_logger
|
|
21
|
+
from regscale.core.app.utils.app_utils import (
|
|
22
|
+
check_file_path,
|
|
23
|
+
check_license,
|
|
24
|
+
create_progress_object,
|
|
25
|
+
error_and_exit,
|
|
26
|
+
get_current_datetime,
|
|
27
|
+
save_data_to,
|
|
28
|
+
)
|
|
29
|
+
from regscale.core.app.utils.file_utils import download_from_s3
|
|
30
|
+
from regscale.models.app_models.click import regscale_ssp_id
|
|
31
|
+
from regscale.models import Asset, Issue, Search, regscale_models
|
|
32
|
+
from regscale.models.app_models.click import NotRequiredIf, save_output_to
|
|
33
|
+
from regscale.models.integration_models.flat_file_importer import FlatFileImporter
|
|
34
|
+
from regscale.models.integration_models.qualys import (
|
|
35
|
+
Qualys,
|
|
36
|
+
QualysContainerScansImporter,
|
|
37
|
+
QualysWasScansImporter,
|
|
38
|
+
QualysPolicyScansImporter,
|
|
39
|
+
)
|
|
40
|
+
from regscale.models.integration_models.qualys_scanner import QualysTotalCloudIntegration
|
|
41
|
+
|
|
42
|
+
####################################################################################################
|
|
43
|
+
#
|
|
44
|
+
# Qualys API Documentation:
|
|
45
|
+
# https://qualysguard.qg2.apps.qualys.com/qwebhelp/fo_portal/api_doc/index.htm
|
|
46
|
+
#
|
|
47
|
+
####################################################################################################
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
# create global variables for the entire module
|
|
51
|
+
logger = create_logger()
|
|
52
|
+
|
|
53
|
+
# create progress object to add tasks to for real time updates
|
|
54
|
+
job_progress = create_progress_object()
|
|
55
|
+
HEADERS = {"X-Requested-With": "RegScale CLI"}
|
|
56
|
+
QUALYS_API = Session()
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
# Create group to handle Qualys commands
|
|
60
|
+
@click.group()
|
|
61
|
+
def qualys():
|
|
62
|
+
"""Performs actions from the Qualys API"""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@qualys.command(name="export_scans")
|
|
66
|
+
@save_output_to()
|
|
67
|
+
@click.option(
|
|
68
|
+
"--days",
|
|
69
|
+
type=int,
|
|
70
|
+
default=30,
|
|
71
|
+
help="The number of days to go back for completed scans, default is 30.",
|
|
72
|
+
)
|
|
73
|
+
@click.option(
|
|
74
|
+
"--export",
|
|
75
|
+
type=click.BOOL,
|
|
76
|
+
help="To disable saving the scans as a .json file, use False. Defaults to True.",
|
|
77
|
+
default=True,
|
|
78
|
+
prompt=False,
|
|
79
|
+
required=False,
|
|
80
|
+
)
|
|
81
|
+
def export_past_scans(save_output_to: Path, days: int, export: bool = True):
|
|
82
|
+
"""Export scans from Qualys Host that were completed
|
|
83
|
+
in the last x days, defaults to last 30 days
|
|
84
|
+
and defaults to save it as a .json file"""
|
|
85
|
+
export_scans(
|
|
86
|
+
save_path=save_output_to,
|
|
87
|
+
days=days,
|
|
88
|
+
export=export,
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@qualys.command(name="import_scans")
|
|
93
|
+
@FlatFileImporter.common_scanner_options(
|
|
94
|
+
message="File path to the folder containing Aqua .csv files to process to RegScale.",
|
|
95
|
+
prompt="File path for Qualys files",
|
|
96
|
+
import_name="qualys",
|
|
97
|
+
)
|
|
98
|
+
@click.option(
|
|
99
|
+
"--skip_rows",
|
|
100
|
+
type=click.INT,
|
|
101
|
+
help="The number of rows in the file to skip to get to the column headers, defaults to 129.",
|
|
102
|
+
default=129,
|
|
103
|
+
)
|
|
104
|
+
def import_scans(
|
|
105
|
+
folder_path: os.PathLike[str],
|
|
106
|
+
regscale_ssp_id: int,
|
|
107
|
+
scan_date: datetime,
|
|
108
|
+
mappings_path: os.PathLike[str],
|
|
109
|
+
disable_mapping: bool,
|
|
110
|
+
skip_rows: int,
|
|
111
|
+
s3_bucket: str,
|
|
112
|
+
s3_prefix: str,
|
|
113
|
+
aws_profile: str,
|
|
114
|
+
upload_file: bool,
|
|
115
|
+
):
|
|
116
|
+
"""Import scans from Qualys"""
|
|
117
|
+
import_qualys_scans(
|
|
118
|
+
folder_path=folder_path,
|
|
119
|
+
regscale_ssp_id=regscale_ssp_id,
|
|
120
|
+
scan_date=scan_date,
|
|
121
|
+
mappings_path=mappings_path,
|
|
122
|
+
disable_mapping=disable_mapping,
|
|
123
|
+
skip_rows=skip_rows,
|
|
124
|
+
s3_bucket=s3_bucket,
|
|
125
|
+
s3_prefix=s3_prefix,
|
|
126
|
+
aws_profile=aws_profile,
|
|
127
|
+
upload_file=upload_file,
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def import_qualys_scans(
|
|
132
|
+
folder_path: os.PathLike[str],
|
|
133
|
+
regscale_ssp_id: int,
|
|
134
|
+
scan_date: datetime,
|
|
135
|
+
mappings_path: os.PathLike[str],
|
|
136
|
+
disable_mapping: bool,
|
|
137
|
+
skip_rows: int,
|
|
138
|
+
s3_bucket: str,
|
|
139
|
+
s3_prefix: str,
|
|
140
|
+
aws_profile: str,
|
|
141
|
+
upload_file: Optional[bool] = True,
|
|
142
|
+
) -> None:
|
|
143
|
+
"""
|
|
144
|
+
Import scans from Qualys
|
|
145
|
+
|
|
146
|
+
:param os.PathLike[str] folder_path: File path to the folder containing Qualys .csv files to process to RegScale
|
|
147
|
+
:param int regscale_ssp_id: The RegScale SSP ID
|
|
148
|
+
:param datetime scan_date: The date of the scan
|
|
149
|
+
:param os.PathLike[str] mappings_path: The path to the mappings file
|
|
150
|
+
:param bool disable_mapping: Whether to disable custom mappings
|
|
151
|
+
:param int skip_rows: The number of rows in the file to skip to get to the column headers
|
|
152
|
+
:param str s3_bucket: The S3 bucket to download the files from
|
|
153
|
+
:param str s3_prefix: The S3 prefix to download the files from
|
|
154
|
+
:param str aws_profile: The AWS profile to use for S3 access
|
|
155
|
+
:param Optional[bool] upload_file: Whether to upload the file to RegScale after processing, defaults to True
|
|
156
|
+
:rtype: None
|
|
157
|
+
"""
|
|
158
|
+
FlatFileImporter.import_files(
|
|
159
|
+
import_type=Qualys,
|
|
160
|
+
import_name="Qualys",
|
|
161
|
+
file_types=".csv",
|
|
162
|
+
folder_path=folder_path,
|
|
163
|
+
regscale_ssp_id=regscale_ssp_id,
|
|
164
|
+
scan_date=scan_date,
|
|
165
|
+
mappings_path=mappings_path,
|
|
166
|
+
disable_mapping=disable_mapping,
|
|
167
|
+
s3_bucket=s3_bucket,
|
|
168
|
+
s3_prefix=s3_prefix,
|
|
169
|
+
aws_profile=aws_profile,
|
|
170
|
+
upload_file=upload_file,
|
|
171
|
+
skip_rows=skip_rows,
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
@qualys.command(name="save_results")
|
|
176
|
+
@save_output_to()
|
|
177
|
+
@click.option(
|
|
178
|
+
"--scan_id",
|
|
179
|
+
type=click.STRING,
|
|
180
|
+
help="Qualys scan reference ID to get results, defaults to all.",
|
|
181
|
+
default="all",
|
|
182
|
+
)
|
|
183
|
+
def save_results(save_output_to: Path, scan_id: str):
|
|
184
|
+
"""Get scan results from Qualys using a scan ID or all scans and save them to a .json file."""
|
|
185
|
+
save_scan_results_by_id(save_path=save_output_to, scan_id=scan_id)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
@qualys.command(name="sync_qualys")
|
|
189
|
+
@click.option(
|
|
190
|
+
"--regscale_ssp_id",
|
|
191
|
+
type=click.INT,
|
|
192
|
+
required=True,
|
|
193
|
+
prompt="Enter RegScale System Security Plan ID",
|
|
194
|
+
help="The ID number from RegScale of the System Security Plan",
|
|
195
|
+
)
|
|
196
|
+
@click.option(
|
|
197
|
+
"--create_issue",
|
|
198
|
+
type=click.BOOL,
|
|
199
|
+
required=False,
|
|
200
|
+
help="Create Issue in RegScale from vulnerabilities in Qualys.",
|
|
201
|
+
default=False,
|
|
202
|
+
)
|
|
203
|
+
@click.option(
|
|
204
|
+
"--asset_group_id",
|
|
205
|
+
type=click.INT,
|
|
206
|
+
help="Filter assets from Qualys with an asset group ID.",
|
|
207
|
+
default=None,
|
|
208
|
+
cls=NotRequiredIf,
|
|
209
|
+
not_required_if=["asset_group_name"],
|
|
210
|
+
)
|
|
211
|
+
@click.option(
|
|
212
|
+
"--asset_group_name",
|
|
213
|
+
type=click.STRING,
|
|
214
|
+
help="Filter assets from Qualys with an asset group name.",
|
|
215
|
+
default=None,
|
|
216
|
+
cls=NotRequiredIf,
|
|
217
|
+
not_required_if=["asset_group_id"],
|
|
218
|
+
)
|
|
219
|
+
def sync_qualys(
|
|
220
|
+
regscale_ssp_id: int,
|
|
221
|
+
create_issue: bool = False,
|
|
222
|
+
asset_group_id: int = None,
|
|
223
|
+
asset_group_name: str = None,
|
|
224
|
+
):
|
|
225
|
+
"""
|
|
226
|
+
Query Qualys and sync assets & their associated
|
|
227
|
+
vulnerabilities to a Security Plan in RegScale.
|
|
228
|
+
"""
|
|
229
|
+
sync_qualys_to_regscale(
|
|
230
|
+
regscale_ssp_id=regscale_ssp_id,
|
|
231
|
+
create_issue=create_issue,
|
|
232
|
+
asset_group_id=asset_group_id,
|
|
233
|
+
asset_group_name=asset_group_name,
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
@qualys.command(name="get_asset_groups")
|
|
238
|
+
@save_output_to()
|
|
239
|
+
def get_asset_groups(save_output_to: Path):
|
|
240
|
+
"""
|
|
241
|
+
Get all asset groups from Qualys via API and save them to a .json file.
|
|
242
|
+
"""
|
|
243
|
+
# see if user has enterprise license
|
|
244
|
+
check_license()
|
|
245
|
+
|
|
246
|
+
date = get_current_datetime("%Y%m%d")
|
|
247
|
+
check_file_path(save_output_to)
|
|
248
|
+
asset_groups = get_asset_groups_from_qualys()
|
|
249
|
+
save_data_to(
|
|
250
|
+
file=Path(f"{save_output_to}/qualys_asset_groups_{date}.json"),
|
|
251
|
+
data=asset_groups,
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
@qualys.command(name="import_container_scans")
|
|
256
|
+
@FlatFileImporter.common_scanner_options(
|
|
257
|
+
message="File path to the folder containing container .csv files to process to RegScale.",
|
|
258
|
+
prompt="File path for Qualys files",
|
|
259
|
+
import_name="qualys_container_scan",
|
|
260
|
+
)
|
|
261
|
+
@click.option(
|
|
262
|
+
"--skip_rows",
|
|
263
|
+
type=click.INT,
|
|
264
|
+
help="The number of rows in the file to skip to get to the column headers, defaults to 5.",
|
|
265
|
+
default=5,
|
|
266
|
+
)
|
|
267
|
+
def import_container_scans(
|
|
268
|
+
folder_path: os.PathLike[str],
|
|
269
|
+
regscale_ssp_id: int,
|
|
270
|
+
scan_date: datetime,
|
|
271
|
+
mappings_path: Path,
|
|
272
|
+
disable_mapping: bool,
|
|
273
|
+
s3_bucket: str,
|
|
274
|
+
s3_prefix: str,
|
|
275
|
+
aws_profile: str,
|
|
276
|
+
upload_file: bool,
|
|
277
|
+
skip_rows: int,
|
|
278
|
+
):
|
|
279
|
+
"""
|
|
280
|
+
Import Qualys container scans from a CSV file into a RegScale Security Plan as assets and vulnerabilities.
|
|
281
|
+
"""
|
|
282
|
+
process_files_with_importer(
|
|
283
|
+
folder_path=str(folder_path),
|
|
284
|
+
importer_class=QualysContainerScansImporter,
|
|
285
|
+
regscale_ssp_id=regscale_ssp_id,
|
|
286
|
+
importer_args={
|
|
287
|
+
"plan_id": regscale_ssp_id,
|
|
288
|
+
"name": "QualysContainerScan",
|
|
289
|
+
"parent_id": regscale_ssp_id,
|
|
290
|
+
"parent_module": "securityplans",
|
|
291
|
+
"scan_date": scan_date,
|
|
292
|
+
},
|
|
293
|
+
mappings_path=str(mappings_path),
|
|
294
|
+
disable_mapping=disable_mapping,
|
|
295
|
+
skip_rows=skip_rows,
|
|
296
|
+
s3_bucket=s3_bucket,
|
|
297
|
+
s3_prefix=s3_prefix,
|
|
298
|
+
aws_profile=aws_profile,
|
|
299
|
+
upload_file=upload_file,
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
@qualys.command(name="import_was_scans")
|
|
304
|
+
@FlatFileImporter.common_scanner_options(
|
|
305
|
+
message="File path to the folder containing was .csv files to process to RegScale.",
|
|
306
|
+
prompt="File path for Qualys files",
|
|
307
|
+
import_name="qualys_was_scan",
|
|
308
|
+
)
|
|
309
|
+
@click.option(
|
|
310
|
+
"--skip_rows",
|
|
311
|
+
type=click.INT,
|
|
312
|
+
help="The number of rows in the file to skip to get to the column headers, defaults to 5.",
|
|
313
|
+
default=5,
|
|
314
|
+
)
|
|
315
|
+
def import_was_scans(
|
|
316
|
+
folder_path: os.PathLike[str],
|
|
317
|
+
regscale_ssp_id: int,
|
|
318
|
+
scan_date: datetime,
|
|
319
|
+
mappings_path: Path,
|
|
320
|
+
disable_mapping: bool,
|
|
321
|
+
skip_rows: int,
|
|
322
|
+
s3_bucket: str,
|
|
323
|
+
s3_prefix: str,
|
|
324
|
+
aws_profile: str,
|
|
325
|
+
upload_file: bool,
|
|
326
|
+
):
|
|
327
|
+
"""
|
|
328
|
+
Import Qualys was scans from a CSV file into a RegScale Security Plan as assets and vulnerabilities.
|
|
329
|
+
"""
|
|
330
|
+
process_files_with_importer(
|
|
331
|
+
folder_path=str(folder_path),
|
|
332
|
+
importer_class=QualysWasScansImporter,
|
|
333
|
+
regscale_ssp_id=regscale_ssp_id,
|
|
334
|
+
importer_args={
|
|
335
|
+
"plan_id": regscale_ssp_id,
|
|
336
|
+
"name": "QualysWASScan",
|
|
337
|
+
"parent_id": regscale_ssp_id,
|
|
338
|
+
"parent_module": "securityplans",
|
|
339
|
+
"scan_date": scan_date,
|
|
340
|
+
},
|
|
341
|
+
mappings_path=str(mappings_path),
|
|
342
|
+
disable_mapping=disable_mapping,
|
|
343
|
+
skip_rows=skip_rows,
|
|
344
|
+
s3_bucket=s3_bucket,
|
|
345
|
+
s3_prefix=s3_prefix,
|
|
346
|
+
aws_profile=aws_profile,
|
|
347
|
+
upload_file=upload_file,
|
|
348
|
+
)
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
@qualys.command(name="import_total_cloud")
|
|
352
|
+
@regscale_ssp_id()
|
|
353
|
+
@click.option(
|
|
354
|
+
"--include_tags",
|
|
355
|
+
"-t",
|
|
356
|
+
type=click.STRING,
|
|
357
|
+
required=False,
|
|
358
|
+
default=None,
|
|
359
|
+
help="Include tags in the import comma seperated string of tag names or ids, defaults to None.",
|
|
360
|
+
)
|
|
361
|
+
@click.option(
|
|
362
|
+
"--exclude_tags",
|
|
363
|
+
"-e",
|
|
364
|
+
type=click.STRING,
|
|
365
|
+
required=False,
|
|
366
|
+
default=None,
|
|
367
|
+
help="Exclude tags in the import comma seperated string of tag names or ids, defaults to None.",
|
|
368
|
+
)
|
|
369
|
+
def import_total_cloud_assets_and_vulnerabilities(regscale_ssp_id: int, include_tags: str, exclude_tags: str):
|
|
370
|
+
"""
|
|
371
|
+
Import Qualys Total Cloud Assets and Vulnerabilities into RegScale via API."""
|
|
372
|
+
import_total_cloud_data_from_qualys_api(
|
|
373
|
+
security_plan_id=regscale_ssp_id, include_tags=include_tags, exclude_tags=exclude_tags
|
|
374
|
+
)
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
@qualys.command(name="import_policy_scans")
|
|
378
|
+
@FlatFileImporter.common_scanner_options(
|
|
379
|
+
message="File path to the folder containing policy .csv files to process to RegScale.",
|
|
380
|
+
prompt="File path for Qualys files",
|
|
381
|
+
import_name="qualys_policy_scan",
|
|
382
|
+
)
|
|
383
|
+
@click.option(
|
|
384
|
+
"--skip_rows",
|
|
385
|
+
type=click.INT,
|
|
386
|
+
help="The number of rows in the file to skip to get to the column headers, defaults to 5.",
|
|
387
|
+
default=5,
|
|
388
|
+
)
|
|
389
|
+
def import_policy_scans(
|
|
390
|
+
folder_path: os.PathLike[str],
|
|
391
|
+
regscale_ssp_id: int,
|
|
392
|
+
scan_date: datetime,
|
|
393
|
+
mappings_path: Path,
|
|
394
|
+
disable_mapping: bool,
|
|
395
|
+
skip_rows: int,
|
|
396
|
+
s3_bucket: str,
|
|
397
|
+
s3_prefix: str,
|
|
398
|
+
aws_profile: str,
|
|
399
|
+
upload_file: bool,
|
|
400
|
+
):
|
|
401
|
+
"""
|
|
402
|
+
Import Qualys policy scans from a CSV file into a RegScale Security Plan as assets and vulnerabilities.
|
|
403
|
+
"""
|
|
404
|
+
process_files_with_importer(
|
|
405
|
+
folder_path=str(folder_path),
|
|
406
|
+
importer_class=QualysPolicyScansImporter,
|
|
407
|
+
regscale_ssp_id=regscale_ssp_id,
|
|
408
|
+
importer_args={
|
|
409
|
+
"plan_id": regscale_ssp_id,
|
|
410
|
+
"name": "QualysPolicyScan",
|
|
411
|
+
"parent_id": regscale_ssp_id,
|
|
412
|
+
"parent_module": "securityplans",
|
|
413
|
+
"scan_date": scan_date,
|
|
414
|
+
},
|
|
415
|
+
mappings_path=str(mappings_path),
|
|
416
|
+
disable_mapping=disable_mapping,
|
|
417
|
+
skip_rows=skip_rows,
|
|
418
|
+
s3_bucket=s3_bucket,
|
|
419
|
+
s3_prefix=s3_prefix,
|
|
420
|
+
aws_profile=aws_profile,
|
|
421
|
+
upload_file=upload_file,
|
|
422
|
+
)
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def process_files_with_importer(
|
|
426
|
+
regscale_ssp_id: int,
|
|
427
|
+
folder_path: str,
|
|
428
|
+
importer_class,
|
|
429
|
+
importer_args: dict,
|
|
430
|
+
s3_bucket: str,
|
|
431
|
+
s3_prefix: str,
|
|
432
|
+
aws_profile: str,
|
|
433
|
+
mappings_path: str = None,
|
|
434
|
+
disable_mapping: bool = False,
|
|
435
|
+
skip_rows: int = 0,
|
|
436
|
+
scan_date: datetime = None,
|
|
437
|
+
upload_file: Optional[bool] = True,
|
|
438
|
+
):
|
|
439
|
+
"""
|
|
440
|
+
Process files in a folder using a specified importer class.
|
|
441
|
+
|
|
442
|
+
:param int regscale_ssp_id: ID of the RegScale Security Plan to import the data into.
|
|
443
|
+
:param str folder_path: Path to the folder containing files.
|
|
444
|
+
:param Any importer_class: The importer class to instantiate for processing.
|
|
445
|
+
:param dict importer_args: Additional arguments to pass to the importer class.
|
|
446
|
+
:param str s3_bucket: S3 bucket to download the files from.
|
|
447
|
+
:param str s3_prefix: S3 prefix to download the files from.
|
|
448
|
+
:param str aws_profile: AWS profile to use for S3 access.
|
|
449
|
+
:param str mappings_path: Path to mapping configurations.
|
|
450
|
+
:param bool disable_mapping: Flag to disable mappings.
|
|
451
|
+
:param int skip_rows: Number of rows to skip in files.
|
|
452
|
+
:param scan_date: Date of the scan. Defaults to current datetime if not provided.
|
|
453
|
+
:param Optional[bool] upload_file: Whether to upload the file to RegScale after processing, defaults to True.
|
|
454
|
+
"""
|
|
455
|
+
import csv
|
|
456
|
+
from openpyxl import Workbook
|
|
457
|
+
|
|
458
|
+
if s3_bucket:
|
|
459
|
+
download_from_s3(s3_bucket, s3_prefix, folder_path, aws_profile)
|
|
460
|
+
|
|
461
|
+
files_lst = list(Path(folder_path).glob("*.csv"))
|
|
462
|
+
|
|
463
|
+
# If no files are found in the folder, return a warning
|
|
464
|
+
if len(files_lst) == 0:
|
|
465
|
+
logger.warning("No Qualys files found in the folder path provided.")
|
|
466
|
+
return
|
|
467
|
+
|
|
468
|
+
if not scan_date:
|
|
469
|
+
scan_date = datetime.now(timezone.utc)
|
|
470
|
+
|
|
471
|
+
for file in files_lst:
|
|
472
|
+
try:
|
|
473
|
+
original_file_name = str(file)
|
|
474
|
+
xlsx_file = (
|
|
475
|
+
f"{file.name}.xlsx" if not file.name.endswith(".csv") else str(file.name).replace(".csv", ".xlsx")
|
|
476
|
+
)
|
|
477
|
+
|
|
478
|
+
# Convert CSV to XLSX
|
|
479
|
+
wb = Workbook()
|
|
480
|
+
ws = wb.active
|
|
481
|
+
with open(file, "r") as f:
|
|
482
|
+
for row in csv.reader(f):
|
|
483
|
+
ws.append(row)
|
|
484
|
+
|
|
485
|
+
# Save the Excel file
|
|
486
|
+
full_file_path = Path(f"{file.parent}/{xlsx_file}")
|
|
487
|
+
wb.save(full_file_path)
|
|
488
|
+
|
|
489
|
+
# Initialize and use the importer
|
|
490
|
+
importer = importer_class(
|
|
491
|
+
plan_id=regscale_ssp_id,
|
|
492
|
+
name=importer_args.get("name", "QualysFileScan"),
|
|
493
|
+
file_path=str(full_file_path),
|
|
494
|
+
parent_id=regscale_ssp_id,
|
|
495
|
+
parent_module=importer_args.get("parent_module", "securityplans"),
|
|
496
|
+
scan_date=scan_date,
|
|
497
|
+
mappings_path=mappings_path,
|
|
498
|
+
disable_mapping=disable_mapping,
|
|
499
|
+
skip_rows=skip_rows,
|
|
500
|
+
upload_file=upload_file,
|
|
501
|
+
)
|
|
502
|
+
importer.clean_up(file_path=original_file_name)
|
|
503
|
+
except Exception as e:
|
|
504
|
+
error_message = traceback.format_exc()
|
|
505
|
+
logger.error(f"Failed to process file {file}: {error_message}\n{e}")
|
|
506
|
+
continue
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def export_scans(
|
|
510
|
+
save_path: Path,
|
|
511
|
+
days: int = 30,
|
|
512
|
+
export: bool = True,
|
|
513
|
+
) -> None:
|
|
514
|
+
"""
|
|
515
|
+
Function to export scans from Qualys that were completed in the last x days, defaults to 30
|
|
516
|
+
|
|
517
|
+
:param Path save_path: Path to save the scans to as a .json file
|
|
518
|
+
:param int days: # of days of completed scans to export, defaults to 30 days
|
|
519
|
+
:param bool export: Whether to save the scan data as a .json, defaults to True
|
|
520
|
+
:rtype: None
|
|
521
|
+
"""
|
|
522
|
+
# see if user has enterprise license
|
|
523
|
+
check_license()
|
|
524
|
+
date = get_current_datetime("%Y%m%d")
|
|
525
|
+
results = get_detailed_scans(days)
|
|
526
|
+
if export:
|
|
527
|
+
check_file_path(save_path)
|
|
528
|
+
save_data_to(
|
|
529
|
+
file=Path(f"{save_path.name}/qualys_scans_{date}.json"),
|
|
530
|
+
data=results,
|
|
531
|
+
)
|
|
532
|
+
else:
|
|
533
|
+
pprint.pprint(results, indent=4)
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def save_scan_results_by_id(save_path: Path, scan_id: str) -> None:
|
|
537
|
+
"""
|
|
538
|
+
Function to save the queries from Qualys using an ID a .json file
|
|
539
|
+
|
|
540
|
+
:param Path save_path: Path to save the scan results to as a .json file
|
|
541
|
+
:param str scan_id: Qualys scan ID to get the results for
|
|
542
|
+
:rtype: None
|
|
543
|
+
"""
|
|
544
|
+
# see if user has enterprise license
|
|
545
|
+
check_license()
|
|
546
|
+
|
|
547
|
+
check_file_path(save_path)
|
|
548
|
+
with job_progress:
|
|
549
|
+
if scan_id.lower() == "all":
|
|
550
|
+
# get all the scan results from Qualys
|
|
551
|
+
scans = get_scans_summary("all")
|
|
552
|
+
|
|
553
|
+
# add task to job progress to let user know # of scans to fetch
|
|
554
|
+
task1 = job_progress.add_task(
|
|
555
|
+
f"[#f8b737]Getting scan results for {len(scans['SCAN'])} scan(s)...",
|
|
556
|
+
total=len(scans["SCAN"]),
|
|
557
|
+
)
|
|
558
|
+
# get the scan results from Qualys
|
|
559
|
+
scan_data = get_scan_results(scans, task1)
|
|
560
|
+
else:
|
|
561
|
+
task1 = job_progress.add_task(f"[#f8b737]Getting scan results for {scan_id}...", total=1)
|
|
562
|
+
# get the scan result for the provided scan id
|
|
563
|
+
scan_data = get_scan_results(scan_id, task1)
|
|
564
|
+
# save the scan_data as the provided file_path
|
|
565
|
+
save_data_to(file=save_path, data=scan_data)
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
def sync_qualys_to_regscale(
|
|
569
|
+
regscale_ssp_id: int,
|
|
570
|
+
create_issue: bool = False,
|
|
571
|
+
asset_group_id: int = None,
|
|
572
|
+
asset_group_name: str = None,
|
|
573
|
+
) -> None:
|
|
574
|
+
"""
|
|
575
|
+
Sync Qualys assets and vulnerabilities to a security plan in RegScale
|
|
576
|
+
|
|
577
|
+
:param int regscale_ssp_id: ID # of the SSP in RegScale
|
|
578
|
+
:param bool create_issue: Flag whether to create an issue in RegScale from Qualys vulnerabilities, defaults to False
|
|
579
|
+
:param int asset_group_id: Optional filter for assets in Qualys with an asset group ID, defaults to None
|
|
580
|
+
:param str asset_group_name: Optional filter for assets in Qualys with an asset group name, defaults to None
|
|
581
|
+
:rtype: None
|
|
582
|
+
"""
|
|
583
|
+
# see if user has enterprise license
|
|
584
|
+
check_license()
|
|
585
|
+
|
|
586
|
+
# check if the user provided an asset group id or name
|
|
587
|
+
if asset_group_id:
|
|
588
|
+
# get the assets from Qualys using the group name
|
|
589
|
+
sync_qualys_assets_and_vulns(
|
|
590
|
+
ssp_id=regscale_ssp_id,
|
|
591
|
+
create_issue=create_issue,
|
|
592
|
+
asset_group_filter=asset_group_name,
|
|
593
|
+
)
|
|
594
|
+
elif asset_group_name:
|
|
595
|
+
# get the assets from Qualys using the group name
|
|
596
|
+
sync_qualys_assets_and_vulns(
|
|
597
|
+
ssp_id=regscale_ssp_id,
|
|
598
|
+
create_issue=create_issue,
|
|
599
|
+
asset_group_filter=asset_group_id,
|
|
600
|
+
)
|
|
601
|
+
else:
|
|
602
|
+
sync_qualys_assets_and_vulns(ssp_id=regscale_ssp_id, create_issue=create_issue)
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
def get_scan_results(scans: Any, task: TaskID) -> dict:
|
|
606
|
+
"""
|
|
607
|
+
Function to retrieve scan results from Qualys using provided scan list and returns a dictionary
|
|
608
|
+
|
|
609
|
+
:param Any scans: list of scans to retrieve from Qualys
|
|
610
|
+
:param TaskID task: task to update in the progress object
|
|
611
|
+
:return: dictionary of detailed Qualys scans
|
|
612
|
+
:rtype: dict
|
|
613
|
+
"""
|
|
614
|
+
app = check_license()
|
|
615
|
+
config = app.config
|
|
616
|
+
|
|
617
|
+
# set the auth for the QUALYS_API session
|
|
618
|
+
QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
|
|
619
|
+
|
|
620
|
+
scan_data = {}
|
|
621
|
+
# check number of scans requested
|
|
622
|
+
if isinstance(scans, str):
|
|
623
|
+
# only one scan was requested, set up variable for the for loop
|
|
624
|
+
scans = {"SCAN": [{"REF": scans}]}
|
|
625
|
+
for scan in scans["SCAN"]:
|
|
626
|
+
# set up data and parameters for the scans query
|
|
627
|
+
try:
|
|
628
|
+
# try and get the scan id ref #
|
|
629
|
+
scan_id = scan["REF"]
|
|
630
|
+
# set the parameters for the Qualys API call
|
|
631
|
+
params = {
|
|
632
|
+
"action": "fetch",
|
|
633
|
+
"scan_ref": scan_id,
|
|
634
|
+
"mode": "extended",
|
|
635
|
+
"output_format": "json_extended",
|
|
636
|
+
}
|
|
637
|
+
# get the scan data via API
|
|
638
|
+
res = QUALYS_API.get(
|
|
639
|
+
url=urljoin(config["qualysUrl"], "/api/2.0/fo/scan/"),
|
|
640
|
+
headers=HEADERS,
|
|
641
|
+
params=params,
|
|
642
|
+
)
|
|
643
|
+
# convert response to json
|
|
644
|
+
if res.status_code == 200:
|
|
645
|
+
try:
|
|
646
|
+
res_data = res.json()
|
|
647
|
+
scan_data[scan_id] = res_data
|
|
648
|
+
except JSONDecodeError:
|
|
649
|
+
error_and_exit("Unable to convert response to JSON.")
|
|
650
|
+
else:
|
|
651
|
+
error_and_exit(f"Received unexpected response from Qualys API: {res.status_code}: {res.text}")
|
|
652
|
+
except KeyError:
|
|
653
|
+
# unable to get the scan id ref #
|
|
654
|
+
continue
|
|
655
|
+
job_progress.update(task, advance=1)
|
|
656
|
+
return scan_data
|
|
657
|
+
|
|
658
|
+
|
|
659
|
+
def get_detailed_scans(days: int) -> list:
|
|
660
|
+
"""
|
|
661
|
+
function to get the list of all scans from Qualys using QUALYS_API
|
|
662
|
+
|
|
663
|
+
:param int days: # of days before today to filter scans
|
|
664
|
+
:return: list of results from Qualys API
|
|
665
|
+
:rtype: list
|
|
666
|
+
"""
|
|
667
|
+
app = check_license()
|
|
668
|
+
config = app.config
|
|
669
|
+
|
|
670
|
+
# set the auth for the QUALYS_API session
|
|
671
|
+
QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
|
|
672
|
+
|
|
673
|
+
today = datetime.now()
|
|
674
|
+
scan_date = today - timedelta(days=days)
|
|
675
|
+
|
|
676
|
+
# set up data and parameters for the scans query
|
|
677
|
+
params = {
|
|
678
|
+
"action": "list",
|
|
679
|
+
"scan_date_since": scan_date.strftime("%Y-%m-%d"),
|
|
680
|
+
"output_format": "json",
|
|
681
|
+
}
|
|
682
|
+
params2 = {
|
|
683
|
+
"action": "list",
|
|
684
|
+
"scan_datetime_since": scan_date.strftime("%Y-%m-%dT%H:%I:%S%ZZ"),
|
|
685
|
+
}
|
|
686
|
+
res = QUALYS_API.get(
|
|
687
|
+
url=urljoin(config["qualysUrl"], "/api/2.0/fo/scan/summary/"),
|
|
688
|
+
headers=HEADERS,
|
|
689
|
+
params=params,
|
|
690
|
+
)
|
|
691
|
+
response = QUALYS_API.get(
|
|
692
|
+
url=urljoin(config["qualysUrl"], "/api/2.0/fo/scan/vm/summary/"),
|
|
693
|
+
headers=HEADERS,
|
|
694
|
+
params=params2,
|
|
695
|
+
)
|
|
696
|
+
# convert response to json
|
|
697
|
+
res_data = res.json()
|
|
698
|
+
try:
|
|
699
|
+
response_data = xmltodict.parse(response.text)["SCAN_SUMMARY_OUTPUT"]["RESPONSE"]["SCAN_SUMMARY_LIST"][
|
|
700
|
+
"SCAN_SUMMARY"
|
|
701
|
+
]
|
|
702
|
+
if len(res_data) < 1:
|
|
703
|
+
res_data = response_data
|
|
704
|
+
else:
|
|
705
|
+
res_data.extend(response_data)
|
|
706
|
+
except JSONDecodeError:
|
|
707
|
+
logger.error("ERROR: Unable to convert to JSON.")
|
|
708
|
+
return res_data
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
def import_total_cloud_data_from_qualys_api(security_plan_id: int, include_tags: str, exclude_tags: str):
|
|
712
|
+
"""
|
|
713
|
+
Function to get the total cloud data from Qualys API
|
|
714
|
+
:param int security_plan_id: The ID of the plan to get the data for
|
|
715
|
+
:param str include_tags: The tags to include in the data
|
|
716
|
+
:param str exclude_tags: The tags to exclude from the data
|
|
717
|
+
"""
|
|
718
|
+
try:
|
|
719
|
+
|
|
720
|
+
app = check_license()
|
|
721
|
+
config = app.config
|
|
722
|
+
# set the auth for the QUALYS_API session
|
|
723
|
+
QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
|
|
724
|
+
params = {
|
|
725
|
+
"action": "list",
|
|
726
|
+
"show_asset_id": "1",
|
|
727
|
+
"show_tags": "1",
|
|
728
|
+
}
|
|
729
|
+
if exclude_tags or include_tags:
|
|
730
|
+
params["use_tags"] = "1"
|
|
731
|
+
if exclude_tags:
|
|
732
|
+
params["tag_set_exclude"] = exclude_tags
|
|
733
|
+
if include_tags:
|
|
734
|
+
params["tag_set_include"] = include_tags
|
|
735
|
+
response = QUALYS_API.get(
|
|
736
|
+
url=urljoin(config["qualysUrl"], "/api/2.0/fo/asset/host/vm/detection/"),
|
|
737
|
+
headers=HEADERS,
|
|
738
|
+
params=params,
|
|
739
|
+
)
|
|
740
|
+
if response and response.ok:
|
|
741
|
+
response_data = xmltodict.parse(response.text)
|
|
742
|
+
qt = QualysTotalCloudIntegration(plan_id=security_plan_id, xml_data=response_data)
|
|
743
|
+
qt.fetch_assets()
|
|
744
|
+
qt.fetch_findings()
|
|
745
|
+
|
|
746
|
+
else:
|
|
747
|
+
logger.error(
|
|
748
|
+
f"Received unexpected response from Qualys API: {response.status_code}: {response.text if response.text else 'response is null'}"
|
|
749
|
+
)
|
|
750
|
+
except Exception:
|
|
751
|
+
error_message = traceback.format_exc()
|
|
752
|
+
logger.error("Error occurred while processing Qualys data")
|
|
753
|
+
logger.error(error_message)
|
|
754
|
+
|
|
755
|
+
|
|
756
|
+
def get_scans_summary(scan_choice: str) -> dict:
|
|
757
|
+
"""
|
|
758
|
+
Get all scans from Qualys Host
|
|
759
|
+
|
|
760
|
+
:param str scan_choice: The type of scan to retrieve from Qualys API
|
|
761
|
+
:return: Detailed summary of scans from Qualys API as a dictionary
|
|
762
|
+
:rtype: dict
|
|
763
|
+
"""
|
|
764
|
+
app = check_license()
|
|
765
|
+
config = app.config
|
|
766
|
+
urls = []
|
|
767
|
+
|
|
768
|
+
# set the auth for the QUALYS_API session
|
|
769
|
+
QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
|
|
770
|
+
|
|
771
|
+
# set up variables for function
|
|
772
|
+
scan_data = {}
|
|
773
|
+
responses = []
|
|
774
|
+
scan_url = urljoin(config["qualysUrl"], "/api/2.0/fo/scan/")
|
|
775
|
+
|
|
776
|
+
# set up parameters for the scans query
|
|
777
|
+
params = {"action": "list"}
|
|
778
|
+
# check what scan list was requested and set urls list accordingly
|
|
779
|
+
if scan_choice.lower() == "all":
|
|
780
|
+
urls = [scan_url, scan_url + "compliance", scan_url + "scap"]
|
|
781
|
+
elif scan_choice.lower() == "vm":
|
|
782
|
+
urls = [scan_url]
|
|
783
|
+
elif scan_choice.lower() in ["compliance", "scap"]:
|
|
784
|
+
urls = [scan_url + scan_choice.lower()]
|
|
785
|
+
# get the list of vm scans
|
|
786
|
+
for url in urls:
|
|
787
|
+
# get the scan data
|
|
788
|
+
response = QUALYS_API.get(url=url, headers=HEADERS, params=params)
|
|
789
|
+
# store response into a list
|
|
790
|
+
responses.append(response)
|
|
791
|
+
# check the responses received for data
|
|
792
|
+
for response in responses:
|
|
793
|
+
# see if response was successful
|
|
794
|
+
if response.status_code == 200:
|
|
795
|
+
# parse the data
|
|
796
|
+
data = xmltodict.parse(response.text)["SCAN_LIST_OUTPUT"]["RESPONSE"]
|
|
797
|
+
# see if the scan has any data
|
|
798
|
+
try:
|
|
799
|
+
# add the data to the scan_data dictionary
|
|
800
|
+
scan_data.update(data["SCAN_LIST"])
|
|
801
|
+
except KeyError:
|
|
802
|
+
# no data found, continue the for loop
|
|
803
|
+
continue
|
|
804
|
+
return scan_data
|
|
805
|
+
|
|
806
|
+
|
|
807
|
+
def get_scan_details(days: int) -> list:
|
|
808
|
+
"""
|
|
809
|
+
Retrieve completed scans from last x days from Qualys Host
|
|
810
|
+
|
|
811
|
+
:param int days: # of days before today to filter scans
|
|
812
|
+
:return: Detailed summary of scans from Qualys API as a dictionary
|
|
813
|
+
:rtype: list
|
|
814
|
+
"""
|
|
815
|
+
app = check_license()
|
|
816
|
+
config = app.config
|
|
817
|
+
|
|
818
|
+
# set the auth for the QUALYS_API session
|
|
819
|
+
QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
|
|
820
|
+
# get since date for API call
|
|
821
|
+
since_date = datetime.now() - timedelta(days=days)
|
|
822
|
+
# set up data and parameters for the scans query
|
|
823
|
+
headers = {
|
|
824
|
+
"Content-Type": "application/json",
|
|
825
|
+
"Accept": "application/json",
|
|
826
|
+
"X-Requested-With": "RegScale CLI",
|
|
827
|
+
}
|
|
828
|
+
params = {
|
|
829
|
+
"action": "list",
|
|
830
|
+
"scan_date_since": since_date.strftime("%Y-%m-%d"),
|
|
831
|
+
"output_format": "json",
|
|
832
|
+
}
|
|
833
|
+
params2 = {
|
|
834
|
+
"action": "list",
|
|
835
|
+
"scan_datetime_since": since_date.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
836
|
+
}
|
|
837
|
+
res = QUALYS_API.get(
|
|
838
|
+
url=urljoin(config["qualysUrl"], "/api/2.0/fo/scan/summary/"),
|
|
839
|
+
headers=headers,
|
|
840
|
+
params=params,
|
|
841
|
+
)
|
|
842
|
+
response = QUALYS_API.get(
|
|
843
|
+
url=urljoin(config["qualysUrl"], "/api/2.0/fo/scan/vm/summary/"),
|
|
844
|
+
headers=headers,
|
|
845
|
+
params=params2,
|
|
846
|
+
)
|
|
847
|
+
# convert response to json
|
|
848
|
+
res_data = res.json()
|
|
849
|
+
try:
|
|
850
|
+
response_data = xmltodict.parse(response.text)["SCAN_SUMMARY_OUTPUT"]["RESPONSE"]["SCAN_SUMMARY_LIST"][
|
|
851
|
+
"SCAN_SUMMARY"
|
|
852
|
+
]
|
|
853
|
+
if len(res_data) < 1:
|
|
854
|
+
res_data = response_data
|
|
855
|
+
else:
|
|
856
|
+
res_data.update(response_data)
|
|
857
|
+
except JSONDecodeError as ex:
|
|
858
|
+
error_and_exit(f"Unable to convert to JSON.\n{ex}")
|
|
859
|
+
except KeyError:
|
|
860
|
+
error_and_exit(f"No data found.\n{response.text}")
|
|
861
|
+
return res_data
|
|
862
|
+
|
|
863
|
+
|
|
864
|
+
def sync_qualys_assets_and_vulns(
|
|
865
|
+
ssp_id: int,
|
|
866
|
+
create_issue: bool,
|
|
867
|
+
asset_group_filter: Optional[Union[int, str]] = None,
|
|
868
|
+
) -> None:
|
|
869
|
+
"""
|
|
870
|
+
Function to query Qualys and sync assets & associated vulnerabilities to RegScale
|
|
871
|
+
|
|
872
|
+
:param int ssp_id: RegScale System Security Plan ID
|
|
873
|
+
:param bool create_issue: Flag to create an issue in RegScale for each vulnerability from Qualys
|
|
874
|
+
:param Optional[Union[int, str]] asset_group_filter: Filter the Qualys assets by an asset group ID or name, if any
|
|
875
|
+
:rtype: None
|
|
876
|
+
"""
|
|
877
|
+
app = check_license()
|
|
878
|
+
config = app.config
|
|
879
|
+
|
|
880
|
+
# set the auth for the QUALYS_API session
|
|
881
|
+
QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
|
|
882
|
+
|
|
883
|
+
# Get the assets from RegScale with the provided SSP ID
|
|
884
|
+
logger.info("Getting assets from RegScale for SSP #%s...", ssp_id)
|
|
885
|
+
reg_assets = Asset.get_all_by_search(search=Search(parentID=ssp_id, module="securityplans"))
|
|
886
|
+
logger.info(
|
|
887
|
+
"Located %s asset(s) associated with SSP #%s in RegScale.",
|
|
888
|
+
len(reg_assets),
|
|
889
|
+
ssp_id,
|
|
890
|
+
)
|
|
891
|
+
logger.debug(reg_assets)
|
|
892
|
+
|
|
893
|
+
if qualys_assets := get_qualys_assets_and_scan_results(asset_group_filter):
|
|
894
|
+
logger.info("Received %s assets from Qualys.", len(qualys_assets))
|
|
895
|
+
logger.debug(qualys_assets)
|
|
896
|
+
else:
|
|
897
|
+
error_and_exit("No assets found in Qualys.")
|
|
898
|
+
sync_assets(
|
|
899
|
+
qualys_assets=qualys_assets,
|
|
900
|
+
reg_assets=reg_assets,
|
|
901
|
+
ssp_id=ssp_id,
|
|
902
|
+
config=config,
|
|
903
|
+
)
|
|
904
|
+
if create_issue:
|
|
905
|
+
# Get vulnerabilities from Qualys for the Qualys assets
|
|
906
|
+
logger.info("Getting vulnerabilities for %s asset(s) from Qualys...", len(qualys_assets))
|
|
907
|
+
qualys_assets_and_issues, total_vuln_count = get_issue_data_for_assets(qualys_assets)
|
|
908
|
+
logger.info("Received %s vulnerabilities from Qualys.", total_vuln_count)
|
|
909
|
+
logger.debug(qualys_assets_and_issues)
|
|
910
|
+
sync_issues(
|
|
911
|
+
ssp_id=ssp_id,
|
|
912
|
+
qualys_assets_and_issues=qualys_assets_and_issues,
|
|
913
|
+
)
|
|
914
|
+
|
|
915
|
+
|
|
916
|
+
def sync_assets(qualys_assets: list[dict], reg_assets: list[Asset], ssp_id: int, config: dict) -> None:
|
|
917
|
+
"""
|
|
918
|
+
Function to sync Qualys assets to RegScale
|
|
919
|
+
|
|
920
|
+
:param list[dict] qualys_assets: List of Qualys assets
|
|
921
|
+
:param list[Asset] reg_assets: List of RegScale assets
|
|
922
|
+
:param int ssp_id: RegScale System Security Plan ID
|
|
923
|
+
:param dict config: Configuration dictionary
|
|
924
|
+
:rtype: None
|
|
925
|
+
"""
|
|
926
|
+
update_assets = []
|
|
927
|
+
for qualys_asset in qualys_assets: # you can list as many input dicts as you want here
|
|
928
|
+
# Update parent id to SSP on insert
|
|
929
|
+
if lookup_assets := lookup_asset(reg_assets, qualys_asset["ASSET_ID"]):
|
|
930
|
+
for asset in set(lookup_assets):
|
|
931
|
+
asset.parentId = ssp_id
|
|
932
|
+
asset.parentModule = "securityplans"
|
|
933
|
+
asset.otherTrackingNumber = qualys_asset["ID"]
|
|
934
|
+
asset.ipAddress = qualys_asset["IP"]
|
|
935
|
+
asset.qualysId = qualys_asset["ASSET_ID"]
|
|
936
|
+
try:
|
|
937
|
+
assert asset.id
|
|
938
|
+
# avoid duplication
|
|
939
|
+
if asset.qualysId not in [v["qualysId"] for v in update_assets]:
|
|
940
|
+
update_assets.append(asset)
|
|
941
|
+
except AssertionError as aex:
|
|
942
|
+
logger.error("Asset does not have an id, unable to update!\n%s", aex)
|
|
943
|
+
update_and_insert_assets(
|
|
944
|
+
qualys_assets=qualys_assets, reg_assets=reg_assets, ssp_id=ssp_id, config=config, update_assets=update_assets
|
|
945
|
+
)
|
|
946
|
+
|
|
947
|
+
|
|
948
|
+
def update_and_insert_assets(
|
|
949
|
+
qualys_assets: list[dict], reg_assets: list[Asset], ssp_id: int, config: dict, update_assets: list[Asset]
|
|
950
|
+
) -> None:
|
|
951
|
+
"""
|
|
952
|
+
Function to update and insert Qualys assets into RegScale
|
|
953
|
+
|
|
954
|
+
:param list[dict] qualys_assets: List of Qualys assets as dictionaries
|
|
955
|
+
:param list[Asset] reg_assets: List of RegScale assets
|
|
956
|
+
:param int ssp_id: RegScale System Security Plan ID
|
|
957
|
+
:param dict config: RegScale CLI Configuration dictionary
|
|
958
|
+
:param list[Asset] update_assets: List of assets to update in RegScale
|
|
959
|
+
:rtype: None
|
|
960
|
+
"""
|
|
961
|
+
insert_assets = []
|
|
962
|
+
if assets_to_be_inserted := [
|
|
963
|
+
qualys_asset
|
|
964
|
+
for qualys_asset in qualys_assets
|
|
965
|
+
if qualys_asset["ASSET_ID"] not in [asset["ASSET_ID"] for asset in inner_join(reg_assets, qualys_assets)]
|
|
966
|
+
]:
|
|
967
|
+
for qualys_asset in assets_to_be_inserted:
|
|
968
|
+
# Do Insert
|
|
969
|
+
r_asset = Asset(
|
|
970
|
+
name=f'Qualys Asset #{qualys_asset["ASSET_ID"]} IP: {qualys_asset["IP"]}',
|
|
971
|
+
otherTrackingNumber=qualys_asset["ID"],
|
|
972
|
+
parentId=ssp_id,
|
|
973
|
+
parentModule="securityplans",
|
|
974
|
+
ipAddress=qualys_asset["IP"],
|
|
975
|
+
assetOwnerId=config["userId"],
|
|
976
|
+
assetType="Other",
|
|
977
|
+
assetCategory=regscale_models.AssetCategory.Hardware,
|
|
978
|
+
status="Off-Network",
|
|
979
|
+
qualysId=qualys_asset["ASSET_ID"],
|
|
980
|
+
)
|
|
981
|
+
# avoid duplication
|
|
982
|
+
if r_asset.qualysId not in set(v["qualysId"] for v in insert_assets):
|
|
983
|
+
insert_assets.append(r_asset)
|
|
984
|
+
try:
|
|
985
|
+
created_assets = Asset.batch_create(insert_assets, job_progress)
|
|
986
|
+
logger.info(
|
|
987
|
+
"RegScale Asset(s) successfully created: %i/%i",
|
|
988
|
+
len(created_assets),
|
|
989
|
+
len(insert_assets),
|
|
990
|
+
)
|
|
991
|
+
except requests.exceptions.RequestException as rex:
|
|
992
|
+
logger.error("Unable to create Qualys Assets in RegScale\n%s", rex)
|
|
993
|
+
if update_assets:
|
|
994
|
+
try:
|
|
995
|
+
updated_assets = Asset.batch_update(update_assets, job_progress)
|
|
996
|
+
logger.info(
|
|
997
|
+
"RegScale Asset(s) successfully updated: %i/%i",
|
|
998
|
+
len(updated_assets),
|
|
999
|
+
len(update_assets),
|
|
1000
|
+
)
|
|
1001
|
+
except requests.RequestException as rex:
|
|
1002
|
+
logger.error("Unable to Update Qualys Assets to RegScale\n%s", rex)
|
|
1003
|
+
|
|
1004
|
+
|
|
1005
|
+
def sync_issues(ssp_id: int, qualys_assets_and_issues: list[dict]) -> None:
|
|
1006
|
+
"""
|
|
1007
|
+
Function to sync Qualys issues to RegScale
|
|
1008
|
+
|
|
1009
|
+
:param int ssp_id: RegScale System Security Plan ID
|
|
1010
|
+
:param list[dict] qualys_assets_and_issues: List of Qualys assets and their issues
|
|
1011
|
+
:rtype: None
|
|
1012
|
+
"""
|
|
1013
|
+
update_issues = []
|
|
1014
|
+
insert_issues = []
|
|
1015
|
+
vuln_count = 0
|
|
1016
|
+
ssp_assets = Asset.get_all_by_parent(parent_id=ssp_id, parent_module="securityplans")
|
|
1017
|
+
for asset in qualys_assets_and_issues:
|
|
1018
|
+
# Create issues in RegScale from Qualys vulnerabilities
|
|
1019
|
+
regscale_issue_updates, regscale_new_issues = create_regscale_issue_from_vuln(
|
|
1020
|
+
regscale_ssp_id=ssp_id, qualys_asset=asset, regscale_assets=ssp_assets, vulns=asset["ISSUES"]
|
|
1021
|
+
)
|
|
1022
|
+
update_issues.extend(regscale_issue_updates)
|
|
1023
|
+
insert_issues.extend(regscale_new_issues)
|
|
1024
|
+
vuln_count += len(asset.get("ISSUES", []))
|
|
1025
|
+
if insert_issues:
|
|
1026
|
+
deduped_vulns = combine_duplicate_qualys_vulns(insert_issues)
|
|
1027
|
+
logger.info(
|
|
1028
|
+
"Creating %i new issue(s) in RegScale, condensed from %i Qualys vulnerabilities.",
|
|
1029
|
+
len(deduped_vulns),
|
|
1030
|
+
vuln_count,
|
|
1031
|
+
)
|
|
1032
|
+
created_issues = Issue.batch_create(deduped_vulns, job_progress)
|
|
1033
|
+
logger.info(
|
|
1034
|
+
"RegScale Issue(s) successfully created: %i/%i",
|
|
1035
|
+
len(created_issues),
|
|
1036
|
+
len(deduped_vulns),
|
|
1037
|
+
)
|
|
1038
|
+
if update_issues:
|
|
1039
|
+
deduped_vulns = combine_duplicate_qualys_vulns(update_issues)
|
|
1040
|
+
logger.info(
|
|
1041
|
+
"Updating %i existing issue(s) in RegScale, condensed from %i Qualys vulnerabilities.",
|
|
1042
|
+
len(deduped_vulns),
|
|
1043
|
+
vuln_count,
|
|
1044
|
+
)
|
|
1045
|
+
updated_issues = Issue.batch_update(deduped_vulns, job_progress)
|
|
1046
|
+
logger.info("RegScale Issue(s) successfully updated: %i/%i", len(updated_issues), len(deduped_vulns))
|
|
1047
|
+
|
|
1048
|
+
|
|
1049
|
+
def combine_duplicate_qualys_vulns(qualys_vulns: list[Issue]) -> list:
|
|
1050
|
+
"""
|
|
1051
|
+
Function to combine duplicate Qualys vulnerabilities
|
|
1052
|
+
|
|
1053
|
+
:param list[Issue] qualys_vulns: List of Qualys vulnerabilities as RegScale issues
|
|
1054
|
+
:return: List of Qualys vulnerabilities with duplicates combined
|
|
1055
|
+
:rtype: list
|
|
1056
|
+
"""
|
|
1057
|
+
with job_progress:
|
|
1058
|
+
logger.info("Combining duplicate Qualys vulnerabilities found across multiple assets...")
|
|
1059
|
+
deduping_task = job_progress.add_task(
|
|
1060
|
+
f"Combining {len(qualys_vulns)} Qualys vulnerabilities...",
|
|
1061
|
+
total=len(qualys_vulns),
|
|
1062
|
+
)
|
|
1063
|
+
combined_vulns: dict[str, Issue] = {}
|
|
1064
|
+
for vuln in qualys_vulns:
|
|
1065
|
+
if vuln.qualysId in combined_vulns:
|
|
1066
|
+
if current_identifier := combined_vulns[vuln.qualysId].assetIdentifier:
|
|
1067
|
+
combined_vulns[vuln.qualysId].assetIdentifier = update_asset_identifier(
|
|
1068
|
+
vuln.assetIdentifier, current_identifier
|
|
1069
|
+
)
|
|
1070
|
+
else:
|
|
1071
|
+
combined_vulns[vuln.qualysId].assetIdentifier = vuln.assetIdentifier
|
|
1072
|
+
else:
|
|
1073
|
+
combined_vulns[vuln.qualysId] = vuln
|
|
1074
|
+
job_progress.update(deduping_task, advance=1)
|
|
1075
|
+
return list(combined_vulns.values())
|
|
1076
|
+
|
|
1077
|
+
|
|
1078
|
+
def get_qualys_assets_and_scan_results(
|
|
1079
|
+
url: Optional[str] = None, asset_group_filter: Optional[Union[int, str]] = None
|
|
1080
|
+
) -> list:
|
|
1081
|
+
"""
|
|
1082
|
+
function to gather all assets from Qualys API host along with their scan results
|
|
1083
|
+
|
|
1084
|
+
:param Optional[str] url: URL to get the assets from, defaults to None, used for pagination
|
|
1085
|
+
:param Optional[Union[int, str]] asset_group_filter: Qualys asset group ID or name to filter by, if provided
|
|
1086
|
+
:return: list of dictionaries containing asset data
|
|
1087
|
+
:rtype: list
|
|
1088
|
+
"""
|
|
1089
|
+
app = check_license()
|
|
1090
|
+
config = app.config
|
|
1091
|
+
|
|
1092
|
+
# set the auth for the QUALYS_API session
|
|
1093
|
+
QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
|
|
1094
|
+
# set url
|
|
1095
|
+
if not url:
|
|
1096
|
+
url = urljoin(config["qualysUrl"], "api/2.0/fo/asset/host/vm/detection?action=list&show_asset_id=1")
|
|
1097
|
+
|
|
1098
|
+
# check if an asset group filter was provided and append it to the url
|
|
1099
|
+
if asset_group_filter:
|
|
1100
|
+
if isinstance(asset_group_filter, str):
|
|
1101
|
+
# Get the asset group ID from Qualys
|
|
1102
|
+
url += f"&ag_titles={asset_group_filter}"
|
|
1103
|
+
logger.info("Getting assets from Qualys by group name: %s...", asset_group_filter)
|
|
1104
|
+
else:
|
|
1105
|
+
url += f"&ag_ids={asset_group_filter}"
|
|
1106
|
+
logger.info(
|
|
1107
|
+
"Getting assets from from Qualys by group ID: #%s...",
|
|
1108
|
+
asset_group_filter,
|
|
1109
|
+
)
|
|
1110
|
+
else:
|
|
1111
|
+
# Get all assets from Qualys
|
|
1112
|
+
logger.info("Getting all assets from Qualys...")
|
|
1113
|
+
|
|
1114
|
+
# get the data via Qualys API host
|
|
1115
|
+
response = QUALYS_API.get(url=url, headers=HEADERS)
|
|
1116
|
+
res_data = xmltodict.parse(response.text)
|
|
1117
|
+
|
|
1118
|
+
try:
|
|
1119
|
+
# parse the xml data from response.text and convert it to a dictionary
|
|
1120
|
+
# and try to extract the data from the parsed XML dictionary
|
|
1121
|
+
asset_data = res_data["HOST_LIST_VM_DETECTION_OUTPUT"]["RESPONSE"]["HOST_LIST"]["HOST"]
|
|
1122
|
+
# check if we need to paginate the asset data
|
|
1123
|
+
if "WARNING" in res_data["HOST_LIST_VM_DETECTION_OUTPUT"]["RESPONSE"]:
|
|
1124
|
+
logger.warning("Not all assets were fetched, fetching more assets from Qualys...")
|
|
1125
|
+
asset_data.extend(
|
|
1126
|
+
get_qualys_assets_and_scan_results(
|
|
1127
|
+
url=res_data["HOST_LIST_VM_DETECTION_OUTPUT"]["RESPONSE"]["WARNING"]["URL"],
|
|
1128
|
+
asset_group_filter=asset_group_filter,
|
|
1129
|
+
)
|
|
1130
|
+
)
|
|
1131
|
+
except KeyError:
|
|
1132
|
+
# if there is a KeyError set the dictionary to nothing
|
|
1133
|
+
asset_data = []
|
|
1134
|
+
# return the asset_data variable
|
|
1135
|
+
return asset_data
|
|
1136
|
+
|
|
1137
|
+
|
|
1138
|
+
def get_issue_data_for_assets(asset_list: list) -> Tuple[list[dict], int]:
|
|
1139
|
+
"""
|
|
1140
|
+
Function to get issue data from Qualys via API for assets in Qualys
|
|
1141
|
+
|
|
1142
|
+
:param list asset_list: Assets and their scan results from Qualys
|
|
1143
|
+
:return: Updated asset list of Qualys assets and their associated vulnerabilities, total number of vulnerabilities
|
|
1144
|
+
:rtype: Tuple[list[dict], int]
|
|
1145
|
+
"""
|
|
1146
|
+
app = check_license()
|
|
1147
|
+
config = app.config
|
|
1148
|
+
|
|
1149
|
+
# set the auth for the QUALYS_API session
|
|
1150
|
+
QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
|
|
1151
|
+
|
|
1152
|
+
with job_progress:
|
|
1153
|
+
issues = {}
|
|
1154
|
+
for asset in asset_list:
|
|
1155
|
+
# check if the asset has any vulnerabilities
|
|
1156
|
+
if vulns := asset.get("DETECTION_LIST", {}).get("DETECTION", {}):
|
|
1157
|
+
asset_vulns = {}
|
|
1158
|
+
analyzing_vulns = job_progress.add_task(
|
|
1159
|
+
f"Analyzing {len(vulns)} vulnerabilities for asset #{asset['ASSET_ID']} from Qualys..."
|
|
1160
|
+
)
|
|
1161
|
+
# iterate through the vulnerabilities & verify they have a confirmed status
|
|
1162
|
+
for vuln in vulns:
|
|
1163
|
+
if vuln["TYPE"] == "Confirmed":
|
|
1164
|
+
issues[vuln["QID"]] = vuln
|
|
1165
|
+
asset_vulns[vuln["QID"]] = vuln
|
|
1166
|
+
job_progress.update(analyzing_vulns, advance=1)
|
|
1167
|
+
job_progress.update(analyzing_vulns, completed=len(vulns))
|
|
1168
|
+
# add the issues to the asset's dictionary
|
|
1169
|
+
asset["ISSUES"] = asset_vulns
|
|
1170
|
+
job_progress.remove_task(analyzing_vulns)
|
|
1171
|
+
asset_list = fetch_vulns_from_qualys(issue_ids=list(issues.keys()), asset_list=asset_list, config=config)
|
|
1172
|
+
return asset_list, len(issues)
|
|
1173
|
+
|
|
1174
|
+
|
|
1175
|
+
def parse_and_map_vuln_data(xml_data: str) -> dict:
|
|
1176
|
+
"""
|
|
1177
|
+
Function to parse Qualys vulnerability data from XML and map it to a dictionary using the Qualys ID as the key
|
|
1178
|
+
|
|
1179
|
+
:param str xml_data: XML data from Qualys API
|
|
1180
|
+
:return: Dictionary of Qualys vulnerability data
|
|
1181
|
+
:rtype: dict
|
|
1182
|
+
"""
|
|
1183
|
+
issue_data = (
|
|
1184
|
+
xmltodict.parse(xml_data)
|
|
1185
|
+
.get("KNOWLEDGE_BASE_VULN_LIST_OUTPUT", {})
|
|
1186
|
+
.get("RESPONSE", {})
|
|
1187
|
+
.get("VULN_LIST", {})
|
|
1188
|
+
.get("VULN", {})
|
|
1189
|
+
)
|
|
1190
|
+
# change the key for the fetched issues to be the qualys ID
|
|
1191
|
+
return {issue["QID"]: issue for issue in issue_data}
|
|
1192
|
+
|
|
1193
|
+
|
|
1194
|
+
def fetch_vulns_from_qualys(issue_ids: list[str], asset_list: list[dict], config: dict, retries: int = 0) -> list[dict]:
|
|
1195
|
+
"""
|
|
1196
|
+
Function to fetch vulnerability data from Qualys for a list of issues and assets
|
|
1197
|
+
|
|
1198
|
+
:param list[str] issue_ids: List of Qualys issue IDs to fetch data for
|
|
1199
|
+
:param list[dict] asset_list: List of Qualys assets to update with vulnerability data
|
|
1200
|
+
:param dict config: CLI Configuration dictionary
|
|
1201
|
+
:param int retries: Number of retries for fetching data, defaults to 0
|
|
1202
|
+
:return: Updated asset list with vulnerability data
|
|
1203
|
+
:rtype: list[dict]
|
|
1204
|
+
"""
|
|
1205
|
+
logger.info(
|
|
1206
|
+
f"Getting vulnerability data for {len(issue_ids)} issue(s) from Qualys for {len(asset_list)} asset(s)..."
|
|
1207
|
+
)
|
|
1208
|
+
base_url = urljoin(config["qualysUrl"], "api/2.0/fo/knowledge_base/vuln?action=list&details=All")
|
|
1209
|
+
if len(issue_ids) > 100:
|
|
1210
|
+
logger.warning(
|
|
1211
|
+
"Too many issues to fetch from Qualys. Downloading the Qualys database to prevent rate limits..."
|
|
1212
|
+
)
|
|
1213
|
+
# since there are a lot of vulnerabilities, download the database and reference it locally
|
|
1214
|
+
chunk_size_calc = 20 * 1024
|
|
1215
|
+
with QUALYS_API.post(
|
|
1216
|
+
url=base_url,
|
|
1217
|
+
headers=HEADERS,
|
|
1218
|
+
stream=True,
|
|
1219
|
+
) as response:
|
|
1220
|
+
check_file_path("artifacts")
|
|
1221
|
+
with open("./artifacts/qualys_vuln_db.xml", "wb") as f:
|
|
1222
|
+
for chunk in response.iter_content(chunk_size=chunk_size_calc):
|
|
1223
|
+
f.write(chunk)
|
|
1224
|
+
with open("./artifacts/qualys_vuln_db.xml", "r") as f:
|
|
1225
|
+
qualys_issue_data = parse_and_map_vuln_data(f.read())
|
|
1226
|
+
else:
|
|
1227
|
+
response = QUALYS_API.get(
|
|
1228
|
+
url=f"{base_url}&ids={','.join(issue_ids)}",
|
|
1229
|
+
headers=HEADERS,
|
|
1230
|
+
)
|
|
1231
|
+
if response.ok:
|
|
1232
|
+
qualys_issue_data = parse_and_map_vuln_data(response.text)
|
|
1233
|
+
logger.info("Received vulnerability data for %s issues from Qualys.", len(qualys_issue_data))
|
|
1234
|
+
elif response.status_code == 409:
|
|
1235
|
+
response_data = xmltodict.parse(response.text)["SIMPLE_RETURN"]["RESPONSE"]
|
|
1236
|
+
logger.warning(
|
|
1237
|
+
"Received timeout error from Qualys API: %s. Waiting %s seconds...",
|
|
1238
|
+
response_data["TEXT"],
|
|
1239
|
+
response_data["ITEM_LIST"]["ITEM"]["VALUE"],
|
|
1240
|
+
)
|
|
1241
|
+
sleep(int(response_data["ITEM_LIST"]["ITEM"]["VALUE"]))
|
|
1242
|
+
if retries < 3:
|
|
1243
|
+
fetch_vulns_from_qualys(issue_ids, asset_list, config, retries + 1)
|
|
1244
|
+
else:
|
|
1245
|
+
error_and_exit(
|
|
1246
|
+
"Unable to fetch vulnerability data from Qualys after 3 attempts. Please try again later."
|
|
1247
|
+
)
|
|
1248
|
+
else:
|
|
1249
|
+
error_and_exit(
|
|
1250
|
+
f"Received unexpected response from Qualys: {response.status_code}: {response.text}: {response.reason}"
|
|
1251
|
+
)
|
|
1252
|
+
return map_issue_data_to_assets(asset_list, qualys_issue_data)
|
|
1253
|
+
|
|
1254
|
+
|
|
1255
|
+
def map_issue_data_to_assets(assets: list[dict], qualys_issue_data: dict) -> list[dict]:
|
|
1256
|
+
"""
|
|
1257
|
+
Function to map Qualys issue data to Qualys assets
|
|
1258
|
+
|
|
1259
|
+
:param list[dict] assets: List of Qualys assets to map issue data to
|
|
1260
|
+
:param dict qualys_issue_data: List of Qualys issues to map to assets
|
|
1261
|
+
:return: Updated asset list with Qualys issue data
|
|
1262
|
+
:rtype: list[dict]
|
|
1263
|
+
"""
|
|
1264
|
+
for asset in assets:
|
|
1265
|
+
if issues := asset.get("ISSUES"):
|
|
1266
|
+
mapping_vulns = job_progress.add_task(
|
|
1267
|
+
f"Mapping {len(issues)} vulnerabilities to Asset #{asset['ASSET_ID']} from Qualys...",
|
|
1268
|
+
total=len(issues),
|
|
1269
|
+
)
|
|
1270
|
+
for issue in issues:
|
|
1271
|
+
if issue in qualys_issue_data:
|
|
1272
|
+
issues[issue]["ISSUE_DATA"] = qualys_issue_data[issue]
|
|
1273
|
+
job_progress.update(mapping_vulns, advance=1)
|
|
1274
|
+
job_progress.remove_task(mapping_vulns)
|
|
1275
|
+
return assets
|
|
1276
|
+
|
|
1277
|
+
|
|
1278
|
+
def lookup_asset(asset_list: list, asset_id: str = None) -> list[Asset]:
|
|
1279
|
+
"""
|
|
1280
|
+
Function to look up an asset in the asset list and returns an Asset object
|
|
1281
|
+
|
|
1282
|
+
:param list asset_list: List of assets from RegScale
|
|
1283
|
+
:param str asset_id: Qualys asset ID to search for, defaults to None
|
|
1284
|
+
:return: list of Asset objects
|
|
1285
|
+
:rtype: list[Asset]
|
|
1286
|
+
"""
|
|
1287
|
+
if asset_id:
|
|
1288
|
+
results = [Asset(**asset) for asset in asset_list if getattr(asset, "qualysId", None) == asset_id]
|
|
1289
|
+
else:
|
|
1290
|
+
results = [Asset(**asset) for asset in asset_list]
|
|
1291
|
+
# Return unique list
|
|
1292
|
+
return list(set(results)) or []
|
|
1293
|
+
|
|
1294
|
+
|
|
1295
|
+
def map_qualys_severity_to_regscale(severity: int) -> tuple[str, str]:
|
|
1296
|
+
"""
|
|
1297
|
+
Map Qualys vulnerability severity to RegScale Issue severity
|
|
1298
|
+
|
|
1299
|
+
:param int severity: Qualys vulnerability severity
|
|
1300
|
+
:return: RegScale Issue severity and key for init.yaml
|
|
1301
|
+
:rtype: tuple[str, str]
|
|
1302
|
+
"""
|
|
1303
|
+
if severity <= 2:
|
|
1304
|
+
return "III - Low - Other Weakness", "low"
|
|
1305
|
+
if severity == 3:
|
|
1306
|
+
return "II - Moderate - Reportable Condition", "moderate"
|
|
1307
|
+
if severity > 3:
|
|
1308
|
+
return "I - High - Significant Deficiency", "high"
|
|
1309
|
+
return "IV - Not Assigned", "low"
|
|
1310
|
+
|
|
1311
|
+
|
|
1312
|
+
def create_regscale_issue_from_vuln(
|
|
1313
|
+
regscale_ssp_id: int, qualys_asset: dict, regscale_assets: list[Asset], vulns: dict
|
|
1314
|
+
) -> Tuple[list[Issue], list[Issue]]:
|
|
1315
|
+
"""
|
|
1316
|
+
Sync Qualys vulnerabilities to RegScale issues.
|
|
1317
|
+
|
|
1318
|
+
:param int regscale_ssp_id: RegScale SSP ID
|
|
1319
|
+
:param dict qualys_asset: Qualys asset as a dictionary
|
|
1320
|
+
:param list[Asset] regscale_assets: list of RegScale assets
|
|
1321
|
+
:param dict vulns: dictionary of Qualys vulnerabilities associated with the provided asset
|
|
1322
|
+
:return: list of RegScale issues to update, and a list of issues to be created
|
|
1323
|
+
:rtype: Tuple[list[Issue], list[Issue]]
|
|
1324
|
+
"""
|
|
1325
|
+
app = check_license()
|
|
1326
|
+
config = app.config
|
|
1327
|
+
|
|
1328
|
+
# set the auth for the QUALYS_API session
|
|
1329
|
+
QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
|
|
1330
|
+
default_status = config["issues"]["qualys"]["status"]
|
|
1331
|
+
regscale_issues = []
|
|
1332
|
+
regscale_existing_issues = Issue.get_all_by_parent(parent_id=regscale_ssp_id, parent_module="securityplans")
|
|
1333
|
+
for vuln in vulns.values():
|
|
1334
|
+
asset_identifier = None
|
|
1335
|
+
severity, key = map_qualys_severity_to_regscale(int(vuln["SEVERITY"]))
|
|
1336
|
+
|
|
1337
|
+
default_due_delta = config["issues"]["qualys"][key]
|
|
1338
|
+
logger.debug("Processing vulnerability# %s", vuln["QID"])
|
|
1339
|
+
fmt = "%Y-%m-%dT%H:%M:%SZ"
|
|
1340
|
+
due_date = datetime.strptime(vuln["LAST_FOUND_DATETIME"], fmt) + timedelta(days=default_due_delta)
|
|
1341
|
+
regscale_asset = [asset for asset in regscale_assets if asset.qualysId == qualys_asset["ASSET_ID"]]
|
|
1342
|
+
if "DNS" not in qualys_asset.keys() or "IP" not in qualys_asset.keys():
|
|
1343
|
+
if regscale_asset:
|
|
1344
|
+
asset_identifier = f"RegScale Asset #{regscale_asset[0].id}: {regscale_asset[0].name}"
|
|
1345
|
+
else:
|
|
1346
|
+
if regscale_asset:
|
|
1347
|
+
asset_identifier = (
|
|
1348
|
+
f'RegScale Asset #{regscale_asset[0].id}: {regscale_asset[0].name} Qualys DNS: "'
|
|
1349
|
+
f'{qualys_asset["DNS"]} - IP: {qualys_asset["IP"]}'
|
|
1350
|
+
)
|
|
1351
|
+
else:
|
|
1352
|
+
asset_identifier = f'DNS: {qualys_asset["DNS"]} - IP: {qualys_asset["IP"]}'
|
|
1353
|
+
issue = Issue(
|
|
1354
|
+
title=vuln["ISSUE_DATA"]["TITLE"],
|
|
1355
|
+
description=vuln["ISSUE_DATA"]["CONSEQUENCE"] + "</br>" + vuln["ISSUE_DATA"]["DIAGNOSIS"],
|
|
1356
|
+
issueOwnerId=config["userId"],
|
|
1357
|
+
status=default_status,
|
|
1358
|
+
severityLevel=severity,
|
|
1359
|
+
qualysId=vuln["QID"],
|
|
1360
|
+
dueDate=due_date.strftime(fmt),
|
|
1361
|
+
identification="Vulnerability Assessment",
|
|
1362
|
+
parentId=regscale_ssp_id,
|
|
1363
|
+
parentModule="securityplans",
|
|
1364
|
+
recommendedActions=vuln["ISSUE_DATA"]["SOLUTION"],
|
|
1365
|
+
assetIdentifier=asset_identifier,
|
|
1366
|
+
)
|
|
1367
|
+
regscale_issues.append(issue)
|
|
1368
|
+
regscale_new_issues, regscale_update_issues = determine_issue_update_or_create(
|
|
1369
|
+
regscale_issues, regscale_existing_issues
|
|
1370
|
+
)
|
|
1371
|
+
return regscale_update_issues, regscale_new_issues
|
|
1372
|
+
|
|
1373
|
+
|
|
1374
|
+
def update_asset_identifier(new_identifier: Optional[str], current_identifier: Optional[str]) -> Optional[str]:
|
|
1375
|
+
"""
|
|
1376
|
+
Function to update the asset identifier for a RegScale issue
|
|
1377
|
+
|
|
1378
|
+
:param Optional[str] new_identifier: New asset identifier to add
|
|
1379
|
+
:param Optional[str] current_identifier: Current asset identifier
|
|
1380
|
+
:return: Updated asset identifier
|
|
1381
|
+
:rtype: str
|
|
1382
|
+
"""
|
|
1383
|
+
if not current_identifier and new_identifier:
|
|
1384
|
+
return new_identifier
|
|
1385
|
+
if current_identifier and new_identifier:
|
|
1386
|
+
if new_identifier not in current_identifier:
|
|
1387
|
+
return f"{current_identifier}<br>{new_identifier}"
|
|
1388
|
+
if new_identifier in current_identifier:
|
|
1389
|
+
return current_identifier
|
|
1390
|
+
if new_identifier == current_identifier:
|
|
1391
|
+
return current_identifier
|
|
1392
|
+
|
|
1393
|
+
|
|
1394
|
+
def determine_issue_update_or_create(
|
|
1395
|
+
qualys_issues: list[Issue], regscale_issues: list[Issue]
|
|
1396
|
+
) -> Tuple[list[Issue], list[Issue]]:
|
|
1397
|
+
"""
|
|
1398
|
+
Function to determine if Qualys issues needs to be updated or created in RegScale
|
|
1399
|
+
|
|
1400
|
+
:param list[Issue] qualys_issues: List of Qualys issues
|
|
1401
|
+
:param list[Issue] regscale_issues: List of existing RegScale issues
|
|
1402
|
+
:return: List of new issues and list of issues to update
|
|
1403
|
+
:rtype: Tuple[list[Issue], list[Issue]]
|
|
1404
|
+
"""
|
|
1405
|
+
new_issues = []
|
|
1406
|
+
update_issues = []
|
|
1407
|
+
for issue in qualys_issues:
|
|
1408
|
+
if issue.qualysId in [iss.qualysId for iss in regscale_issues]:
|
|
1409
|
+
update_issue = [iss for iss in regscale_issues if iss.qualysId == issue.qualysId][0]
|
|
1410
|
+
# Check if we need to concatenate the asset identifier
|
|
1411
|
+
update_issue.assetIdentifier = update_asset_identifier(issue.assetIdentifier, update_issue.assetIdentifier)
|
|
1412
|
+
update_issues.append(update_issue)
|
|
1413
|
+
else:
|
|
1414
|
+
new_issues.append(issue)
|
|
1415
|
+
return new_issues, update_issues
|
|
1416
|
+
|
|
1417
|
+
|
|
1418
|
+
def inner_join(reg_list: list, qualys_list: list) -> list:
|
|
1419
|
+
"""
|
|
1420
|
+
Function to compare assets from Qualys and assets from RegScale
|
|
1421
|
+
|
|
1422
|
+
:param list reg_list: list of assets from RegScale
|
|
1423
|
+
:param list qualys_list: list of assets from Qualys
|
|
1424
|
+
:return: list of assets that are in both RegScale and Qualys
|
|
1425
|
+
:rtype: list
|
|
1426
|
+
"""
|
|
1427
|
+
|
|
1428
|
+
set1 = set(getattr(lst, "qualysId", None) for lst in reg_list)
|
|
1429
|
+
data = []
|
|
1430
|
+
try:
|
|
1431
|
+
data = [list_qualys for list_qualys in qualys_list if getattr(list_qualys, "ASSET_ID", None) in set1]
|
|
1432
|
+
except KeyError as ex:
|
|
1433
|
+
logger.error(ex)
|
|
1434
|
+
return data
|
|
1435
|
+
|
|
1436
|
+
|
|
1437
|
+
def get_asset_groups_from_qualys() -> list:
|
|
1438
|
+
"""
|
|
1439
|
+
Get all asset groups from Qualys via API
|
|
1440
|
+
|
|
1441
|
+
:return: list of assets from Qualys
|
|
1442
|
+
:rtype: list
|
|
1443
|
+
"""
|
|
1444
|
+
app = check_license()
|
|
1445
|
+
config = app.config
|
|
1446
|
+
asset_groups = []
|
|
1447
|
+
|
|
1448
|
+
# set the auth for the QUALYS_API session
|
|
1449
|
+
QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
|
|
1450
|
+
response = QUALYS_API.get(url=urljoin(config["qualysUrl"], "api/2.0/fo/asset/group?action=list"), headers=HEADERS)
|
|
1451
|
+
if response.ok:
|
|
1452
|
+
logger.debug(response.text)
|
|
1453
|
+
try:
|
|
1454
|
+
asset_groups = xmltodict.parse(response.text)["ASSET_GROUP_LIST_OUTPUT"]["RESPONSE"]["ASSET_GROUP_LIST"][
|
|
1455
|
+
"ASSET_GROUP"
|
|
1456
|
+
]
|
|
1457
|
+
except KeyError:
|
|
1458
|
+
logger.debug(response.text)
|
|
1459
|
+
error_and_exit(
|
|
1460
|
+
f"Unable to retrieve asset groups from Qualys.\nReceived: #{response.status_code}: {response.text}"
|
|
1461
|
+
)
|
|
1462
|
+
return asset_groups
|