regscale-cli 6.16.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of regscale-cli might be problematic. Click here for more details.

Files changed (481) hide show
  1. regscale/__init__.py +1 -0
  2. regscale/airflow/__init__.py +9 -0
  3. regscale/airflow/azure/__init__.py +9 -0
  4. regscale/airflow/azure/cli.py +89 -0
  5. regscale/airflow/azure/upload_dags.py +116 -0
  6. regscale/airflow/click_dags.py +127 -0
  7. regscale/airflow/click_mixins.py +82 -0
  8. regscale/airflow/config.py +25 -0
  9. regscale/airflow/factories/__init__.py +0 -0
  10. regscale/airflow/factories/connections.py +58 -0
  11. regscale/airflow/factories/workflows.py +78 -0
  12. regscale/airflow/hierarchy.py +88 -0
  13. regscale/airflow/operators/__init__.py +0 -0
  14. regscale/airflow/operators/click.py +36 -0
  15. regscale/airflow/sensors/__init__.py +0 -0
  16. regscale/airflow/sensors/sql.py +107 -0
  17. regscale/airflow/sessions/__init__.py +0 -0
  18. regscale/airflow/sessions/sql/__init__.py +3 -0
  19. regscale/airflow/sessions/sql/queries.py +64 -0
  20. regscale/airflow/sessions/sql/sql_server_queries.py +248 -0
  21. regscale/airflow/tasks/__init__.py +0 -0
  22. regscale/airflow/tasks/branches.py +22 -0
  23. regscale/airflow/tasks/cli.py +116 -0
  24. regscale/airflow/tasks/click.py +73 -0
  25. regscale/airflow/tasks/debugging.py +9 -0
  26. regscale/airflow/tasks/groups.py +116 -0
  27. regscale/airflow/tasks/init.py +60 -0
  28. regscale/airflow/tasks/states.py +47 -0
  29. regscale/airflow/tasks/workflows.py +36 -0
  30. regscale/ansible/__init__.py +9 -0
  31. regscale/core/__init__.py +0 -0
  32. regscale/core/app/__init__.py +3 -0
  33. regscale/core/app/api.py +571 -0
  34. regscale/core/app/application.py +665 -0
  35. regscale/core/app/internal/__init__.py +136 -0
  36. regscale/core/app/internal/admin_actions.py +230 -0
  37. regscale/core/app/internal/assessments_editor.py +873 -0
  38. regscale/core/app/internal/catalog.py +316 -0
  39. regscale/core/app/internal/comparison.py +459 -0
  40. regscale/core/app/internal/control_editor.py +571 -0
  41. regscale/core/app/internal/encrypt.py +79 -0
  42. regscale/core/app/internal/evidence.py +1240 -0
  43. regscale/core/app/internal/file_uploads.py +151 -0
  44. regscale/core/app/internal/healthcheck.py +66 -0
  45. regscale/core/app/internal/login.py +305 -0
  46. regscale/core/app/internal/migrations.py +240 -0
  47. regscale/core/app/internal/model_editor.py +1701 -0
  48. regscale/core/app/internal/poam_editor.py +632 -0
  49. regscale/core/app/internal/workflow.py +105 -0
  50. regscale/core/app/logz.py +74 -0
  51. regscale/core/app/utils/XMLIR.py +258 -0
  52. regscale/core/app/utils/__init__.py +0 -0
  53. regscale/core/app/utils/api_handler.py +358 -0
  54. regscale/core/app/utils/app_utils.py +1110 -0
  55. regscale/core/app/utils/catalog_utils/__init__.py +0 -0
  56. regscale/core/app/utils/catalog_utils/common.py +91 -0
  57. regscale/core/app/utils/catalog_utils/compare_catalog.py +193 -0
  58. regscale/core/app/utils/catalog_utils/diagnostic_catalog.py +97 -0
  59. regscale/core/app/utils/catalog_utils/download_catalog.py +103 -0
  60. regscale/core/app/utils/catalog_utils/update_catalog.py +718 -0
  61. regscale/core/app/utils/catalog_utils/update_catalog_v2.py +1378 -0
  62. regscale/core/app/utils/catalog_utils/update_catalog_v3.py +1272 -0
  63. regscale/core/app/utils/catalog_utils/update_plans.py +334 -0
  64. regscale/core/app/utils/file_utils.py +238 -0
  65. regscale/core/app/utils/parser_utils.py +81 -0
  66. regscale/core/app/utils/pickle_file_handler.py +57 -0
  67. regscale/core/app/utils/regscale_utils.py +319 -0
  68. regscale/core/app/utils/report_utils.py +119 -0
  69. regscale/core/app/utils/variables.py +226 -0
  70. regscale/core/decorators.py +31 -0
  71. regscale/core/lazy_group.py +65 -0
  72. regscale/core/login.py +63 -0
  73. regscale/core/server/__init__.py +0 -0
  74. regscale/core/server/flask_api.py +473 -0
  75. regscale/core/server/helpers.py +373 -0
  76. regscale/core/server/rest.py +64 -0
  77. regscale/core/server/static/css/bootstrap.css +6030 -0
  78. regscale/core/server/static/css/bootstrap.min.css +6 -0
  79. regscale/core/server/static/css/main.css +176 -0
  80. regscale/core/server/static/images/regscale-cli.svg +49 -0
  81. regscale/core/server/static/images/regscale.svg +38 -0
  82. regscale/core/server/templates/base.html +74 -0
  83. regscale/core/server/templates/index.html +43 -0
  84. regscale/core/server/templates/login.html +28 -0
  85. regscale/core/server/templates/make_base64.html +22 -0
  86. regscale/core/server/templates/upload_STIG.html +109 -0
  87. regscale/core/server/templates/upload_STIG_result.html +26 -0
  88. regscale/core/server/templates/upload_ssp.html +144 -0
  89. regscale/core/server/templates/upload_ssp_result.html +128 -0
  90. regscale/core/static/__init__.py +0 -0
  91. regscale/core/static/regex.py +14 -0
  92. regscale/core/utils/__init__.py +117 -0
  93. regscale/core/utils/click_utils.py +13 -0
  94. regscale/core/utils/date.py +238 -0
  95. regscale/core/utils/graphql.py +254 -0
  96. regscale/core/utils/urls.py +23 -0
  97. regscale/dev/__init__.py +6 -0
  98. regscale/dev/analysis.py +454 -0
  99. regscale/dev/cli.py +235 -0
  100. regscale/dev/code_gen.py +492 -0
  101. regscale/dev/dirs.py +69 -0
  102. regscale/dev/docs.py +384 -0
  103. regscale/dev/monitoring.py +26 -0
  104. regscale/dev/profiling.py +216 -0
  105. regscale/exceptions/__init__.py +4 -0
  106. regscale/exceptions/license_exception.py +7 -0
  107. regscale/exceptions/validation_exception.py +9 -0
  108. regscale/integrations/__init__.py +1 -0
  109. regscale/integrations/commercial/__init__.py +486 -0
  110. regscale/integrations/commercial/ad.py +433 -0
  111. regscale/integrations/commercial/amazon/__init__.py +0 -0
  112. regscale/integrations/commercial/amazon/common.py +106 -0
  113. regscale/integrations/commercial/aqua/__init__.py +0 -0
  114. regscale/integrations/commercial/aqua/aqua.py +91 -0
  115. regscale/integrations/commercial/aws/__init__.py +6 -0
  116. regscale/integrations/commercial/aws/cli.py +322 -0
  117. regscale/integrations/commercial/aws/inventory/__init__.py +110 -0
  118. regscale/integrations/commercial/aws/inventory/base.py +64 -0
  119. regscale/integrations/commercial/aws/inventory/resources/__init__.py +19 -0
  120. regscale/integrations/commercial/aws/inventory/resources/compute.py +234 -0
  121. regscale/integrations/commercial/aws/inventory/resources/containers.py +113 -0
  122. regscale/integrations/commercial/aws/inventory/resources/database.py +101 -0
  123. regscale/integrations/commercial/aws/inventory/resources/integration.py +237 -0
  124. regscale/integrations/commercial/aws/inventory/resources/networking.py +253 -0
  125. regscale/integrations/commercial/aws/inventory/resources/security.py +240 -0
  126. regscale/integrations/commercial/aws/inventory/resources/storage.py +91 -0
  127. regscale/integrations/commercial/aws/scanner.py +823 -0
  128. regscale/integrations/commercial/azure/__init__.py +0 -0
  129. regscale/integrations/commercial/azure/common.py +32 -0
  130. regscale/integrations/commercial/azure/intune.py +488 -0
  131. regscale/integrations/commercial/azure/scanner.py +49 -0
  132. regscale/integrations/commercial/burp.py +78 -0
  133. regscale/integrations/commercial/cpe.py +144 -0
  134. regscale/integrations/commercial/crowdstrike.py +1117 -0
  135. regscale/integrations/commercial/defender.py +1511 -0
  136. regscale/integrations/commercial/dependabot.py +210 -0
  137. regscale/integrations/commercial/durosuite/__init__.py +0 -0
  138. regscale/integrations/commercial/durosuite/api.py +1546 -0
  139. regscale/integrations/commercial/durosuite/process_devices.py +101 -0
  140. regscale/integrations/commercial/durosuite/scanner.py +637 -0
  141. regscale/integrations/commercial/durosuite/variables.py +21 -0
  142. regscale/integrations/commercial/ecr.py +90 -0
  143. regscale/integrations/commercial/gcp/__init__.py +237 -0
  144. regscale/integrations/commercial/gcp/auth.py +96 -0
  145. regscale/integrations/commercial/gcp/control_tests.py +238 -0
  146. regscale/integrations/commercial/gcp/variables.py +18 -0
  147. regscale/integrations/commercial/gitlab.py +332 -0
  148. regscale/integrations/commercial/grype.py +165 -0
  149. regscale/integrations/commercial/ibm.py +90 -0
  150. regscale/integrations/commercial/import_all/__init__.py +0 -0
  151. regscale/integrations/commercial/import_all/import_all_cmd.py +467 -0
  152. regscale/integrations/commercial/import_all/scan_file_fingerprints.json +27 -0
  153. regscale/integrations/commercial/jira.py +1046 -0
  154. regscale/integrations/commercial/mappings/__init__.py +0 -0
  155. regscale/integrations/commercial/mappings/csf_controls.json +713 -0
  156. regscale/integrations/commercial/mappings/nist_800_53_r5_controls.json +1516 -0
  157. regscale/integrations/commercial/nessus/__init__.py +0 -0
  158. regscale/integrations/commercial/nessus/nessus_utils.py +429 -0
  159. regscale/integrations/commercial/nessus/scanner.py +416 -0
  160. regscale/integrations/commercial/nexpose.py +90 -0
  161. regscale/integrations/commercial/okta.py +798 -0
  162. regscale/integrations/commercial/opentext/__init__.py +0 -0
  163. regscale/integrations/commercial/opentext/click.py +99 -0
  164. regscale/integrations/commercial/opentext/scanner.py +143 -0
  165. regscale/integrations/commercial/prisma.py +91 -0
  166. regscale/integrations/commercial/qualys.py +1462 -0
  167. regscale/integrations/commercial/salesforce.py +980 -0
  168. regscale/integrations/commercial/sap/__init__.py +0 -0
  169. regscale/integrations/commercial/sap/click.py +31 -0
  170. regscale/integrations/commercial/sap/sysdig/__init__.py +0 -0
  171. regscale/integrations/commercial/sap/sysdig/click.py +57 -0
  172. regscale/integrations/commercial/sap/sysdig/sysdig_scanner.py +190 -0
  173. regscale/integrations/commercial/sap/tenable/__init__.py +0 -0
  174. regscale/integrations/commercial/sap/tenable/click.py +49 -0
  175. regscale/integrations/commercial/sap/tenable/scanner.py +196 -0
  176. regscale/integrations/commercial/servicenow.py +1756 -0
  177. regscale/integrations/commercial/sicura/__init__.py +0 -0
  178. regscale/integrations/commercial/sicura/api.py +855 -0
  179. regscale/integrations/commercial/sicura/commands.py +73 -0
  180. regscale/integrations/commercial/sicura/scanner.py +481 -0
  181. regscale/integrations/commercial/sicura/variables.py +16 -0
  182. regscale/integrations/commercial/snyk.py +90 -0
  183. regscale/integrations/commercial/sonarcloud.py +260 -0
  184. regscale/integrations/commercial/sqlserver.py +369 -0
  185. regscale/integrations/commercial/stig_mapper_integration/__init__.py +0 -0
  186. regscale/integrations/commercial/stig_mapper_integration/click_commands.py +38 -0
  187. regscale/integrations/commercial/stig_mapper_integration/mapping_engine.py +353 -0
  188. regscale/integrations/commercial/stigv2/__init__.py +0 -0
  189. regscale/integrations/commercial/stigv2/ckl_parser.py +349 -0
  190. regscale/integrations/commercial/stigv2/click_commands.py +95 -0
  191. regscale/integrations/commercial/stigv2/stig_integration.py +202 -0
  192. regscale/integrations/commercial/synqly/__init__.py +0 -0
  193. regscale/integrations/commercial/synqly/assets.py +46 -0
  194. regscale/integrations/commercial/synqly/ticketing.py +132 -0
  195. regscale/integrations/commercial/synqly/vulnerabilities.py +223 -0
  196. regscale/integrations/commercial/synqly_jira.py +840 -0
  197. regscale/integrations/commercial/tenablev2/__init__.py +0 -0
  198. regscale/integrations/commercial/tenablev2/authenticate.py +31 -0
  199. regscale/integrations/commercial/tenablev2/click.py +1584 -0
  200. regscale/integrations/commercial/tenablev2/scanner.py +504 -0
  201. regscale/integrations/commercial/tenablev2/stig_parsers.py +140 -0
  202. regscale/integrations/commercial/tenablev2/utils.py +78 -0
  203. regscale/integrations/commercial/tenablev2/variables.py +17 -0
  204. regscale/integrations/commercial/trivy.py +162 -0
  205. regscale/integrations/commercial/veracode.py +96 -0
  206. regscale/integrations/commercial/wizv2/WizDataMixin.py +97 -0
  207. regscale/integrations/commercial/wizv2/__init__.py +0 -0
  208. regscale/integrations/commercial/wizv2/click.py +429 -0
  209. regscale/integrations/commercial/wizv2/constants.py +1001 -0
  210. regscale/integrations/commercial/wizv2/issue.py +361 -0
  211. regscale/integrations/commercial/wizv2/models.py +112 -0
  212. regscale/integrations/commercial/wizv2/parsers.py +339 -0
  213. regscale/integrations/commercial/wizv2/sbom.py +115 -0
  214. regscale/integrations/commercial/wizv2/scanner.py +416 -0
  215. regscale/integrations/commercial/wizv2/utils.py +796 -0
  216. regscale/integrations/commercial/wizv2/variables.py +39 -0
  217. regscale/integrations/commercial/wizv2/wiz_auth.py +159 -0
  218. regscale/integrations/commercial/xray.py +91 -0
  219. regscale/integrations/integration/__init__.py +2 -0
  220. regscale/integrations/integration/integration.py +26 -0
  221. regscale/integrations/integration/inventory.py +17 -0
  222. regscale/integrations/integration/issue.py +100 -0
  223. regscale/integrations/integration_override.py +149 -0
  224. regscale/integrations/public/__init__.py +103 -0
  225. regscale/integrations/public/cisa.py +641 -0
  226. regscale/integrations/public/criticality_updater.py +70 -0
  227. regscale/integrations/public/emass.py +411 -0
  228. regscale/integrations/public/emass_slcm_import.py +697 -0
  229. regscale/integrations/public/fedramp/__init__.py +0 -0
  230. regscale/integrations/public/fedramp/appendix_parser.py +548 -0
  231. regscale/integrations/public/fedramp/click.py +479 -0
  232. regscale/integrations/public/fedramp/components.py +714 -0
  233. regscale/integrations/public/fedramp/docx_parser.py +259 -0
  234. regscale/integrations/public/fedramp/fedramp_cis_crm.py +1124 -0
  235. regscale/integrations/public/fedramp/fedramp_common.py +3181 -0
  236. regscale/integrations/public/fedramp/fedramp_docx.py +388 -0
  237. regscale/integrations/public/fedramp/fedramp_five.py +2343 -0
  238. regscale/integrations/public/fedramp/fedramp_traversal.py +138 -0
  239. regscale/integrations/public/fedramp/import_fedramp_r4_ssp.py +279 -0
  240. regscale/integrations/public/fedramp/import_workbook.py +495 -0
  241. regscale/integrations/public/fedramp/inventory_items.py +244 -0
  242. regscale/integrations/public/fedramp/mappings/__init__.py +0 -0
  243. regscale/integrations/public/fedramp/mappings/fedramp_r4_parts.json +7388 -0
  244. regscale/integrations/public/fedramp/mappings/fedramp_r5_params.json +8636 -0
  245. regscale/integrations/public/fedramp/mappings/fedramp_r5_parts.json +9605 -0
  246. regscale/integrations/public/fedramp/mappings/system_roles.py +34 -0
  247. regscale/integrations/public/fedramp/mappings/user.py +175 -0
  248. regscale/integrations/public/fedramp/mappings/values.py +141 -0
  249. regscale/integrations/public/fedramp/markdown_parser.py +150 -0
  250. regscale/integrations/public/fedramp/metadata.py +689 -0
  251. regscale/integrations/public/fedramp/models/__init__.py +59 -0
  252. regscale/integrations/public/fedramp/models/leveraged_auth_new.py +168 -0
  253. regscale/integrations/public/fedramp/models/poam_importer.py +522 -0
  254. regscale/integrations/public/fedramp/parts_mapper.py +107 -0
  255. regscale/integrations/public/fedramp/poam/__init__.py +0 -0
  256. regscale/integrations/public/fedramp/poam/scanner.py +851 -0
  257. regscale/integrations/public/fedramp/properties.py +201 -0
  258. regscale/integrations/public/fedramp/reporting.py +84 -0
  259. regscale/integrations/public/fedramp/resources.py +496 -0
  260. regscale/integrations/public/fedramp/rosetta.py +110 -0
  261. regscale/integrations/public/fedramp/ssp_logger.py +87 -0
  262. regscale/integrations/public/fedramp/system_characteristics.py +922 -0
  263. regscale/integrations/public/fedramp/system_control_implementations.py +582 -0
  264. regscale/integrations/public/fedramp/system_implementation.py +190 -0
  265. regscale/integrations/public/fedramp/xml_utils.py +87 -0
  266. regscale/integrations/public/nist_catalog.py +275 -0
  267. regscale/integrations/public/oscal.py +1946 -0
  268. regscale/integrations/public/otx.py +169 -0
  269. regscale/integrations/scanner_integration.py +2692 -0
  270. regscale/integrations/variables.py +25 -0
  271. regscale/models/__init__.py +7 -0
  272. regscale/models/app_models/__init__.py +5 -0
  273. regscale/models/app_models/catalog_compare.py +213 -0
  274. regscale/models/app_models/click.py +252 -0
  275. regscale/models/app_models/datetime_encoder.py +21 -0
  276. regscale/models/app_models/import_validater.py +321 -0
  277. regscale/models/app_models/mapping.py +260 -0
  278. regscale/models/app_models/pipeline.py +37 -0
  279. regscale/models/click_models.py +413 -0
  280. regscale/models/config.py +154 -0
  281. regscale/models/email_style.css +67 -0
  282. regscale/models/hierarchy.py +8 -0
  283. regscale/models/inspect_models.py +79 -0
  284. regscale/models/integration_models/__init__.py +0 -0
  285. regscale/models/integration_models/amazon_models/__init__.py +0 -0
  286. regscale/models/integration_models/amazon_models/inspector.py +262 -0
  287. regscale/models/integration_models/amazon_models/inspector_scan.py +206 -0
  288. regscale/models/integration_models/aqua.py +247 -0
  289. regscale/models/integration_models/azure_alerts.py +255 -0
  290. regscale/models/integration_models/base64.py +23 -0
  291. regscale/models/integration_models/burp.py +433 -0
  292. regscale/models/integration_models/burp_models.py +128 -0
  293. regscale/models/integration_models/cisa_kev_data.json +19333 -0
  294. regscale/models/integration_models/defender_data.py +93 -0
  295. regscale/models/integration_models/defenderimport.py +143 -0
  296. regscale/models/integration_models/drf.py +443 -0
  297. regscale/models/integration_models/ecr_models/__init__.py +0 -0
  298. regscale/models/integration_models/ecr_models/data.py +69 -0
  299. regscale/models/integration_models/ecr_models/ecr.py +239 -0
  300. regscale/models/integration_models/flat_file_importer.py +1079 -0
  301. regscale/models/integration_models/grype_import.py +247 -0
  302. regscale/models/integration_models/ibm.py +126 -0
  303. regscale/models/integration_models/implementation_results.py +85 -0
  304. regscale/models/integration_models/nexpose.py +140 -0
  305. regscale/models/integration_models/prisma.py +202 -0
  306. regscale/models/integration_models/qualys.py +720 -0
  307. regscale/models/integration_models/qualys_scanner.py +160 -0
  308. regscale/models/integration_models/sbom/__init__.py +0 -0
  309. regscale/models/integration_models/sbom/cyclone_dx.py +139 -0
  310. regscale/models/integration_models/send_reminders.py +620 -0
  311. regscale/models/integration_models/snyk.py +155 -0
  312. regscale/models/integration_models/synqly_models/__init__.py +0 -0
  313. regscale/models/integration_models/synqly_models/capabilities.json +1 -0
  314. regscale/models/integration_models/synqly_models/connector_types.py +22 -0
  315. regscale/models/integration_models/synqly_models/connectors/__init__.py +7 -0
  316. regscale/models/integration_models/synqly_models/connectors/assets.py +97 -0
  317. regscale/models/integration_models/synqly_models/connectors/ticketing.py +583 -0
  318. regscale/models/integration_models/synqly_models/connectors/vulnerabilities.py +169 -0
  319. regscale/models/integration_models/synqly_models/ocsf_mapper.py +331 -0
  320. regscale/models/integration_models/synqly_models/param.py +72 -0
  321. regscale/models/integration_models/synqly_models/synqly_model.py +733 -0
  322. regscale/models/integration_models/synqly_models/tenants.py +39 -0
  323. regscale/models/integration_models/tenable_models/__init__.py +0 -0
  324. regscale/models/integration_models/tenable_models/integration.py +187 -0
  325. regscale/models/integration_models/tenable_models/models.py +513 -0
  326. regscale/models/integration_models/trivy_import.py +231 -0
  327. regscale/models/integration_models/veracode.py +217 -0
  328. regscale/models/integration_models/xray.py +135 -0
  329. regscale/models/locking.py +100 -0
  330. regscale/models/platform.py +110 -0
  331. regscale/models/regscale_models/__init__.py +67 -0
  332. regscale/models/regscale_models/assessment.py +570 -0
  333. regscale/models/regscale_models/assessment_plan.py +52 -0
  334. regscale/models/regscale_models/asset.py +567 -0
  335. regscale/models/regscale_models/asset_mapping.py +190 -0
  336. regscale/models/regscale_models/case.py +42 -0
  337. regscale/models/regscale_models/catalog.py +261 -0
  338. regscale/models/regscale_models/cci.py +46 -0
  339. regscale/models/regscale_models/change.py +167 -0
  340. regscale/models/regscale_models/checklist.py +372 -0
  341. regscale/models/regscale_models/comment.py +49 -0
  342. regscale/models/regscale_models/compliance_settings.py +112 -0
  343. regscale/models/regscale_models/component.py +412 -0
  344. regscale/models/regscale_models/component_mapping.py +65 -0
  345. regscale/models/regscale_models/control.py +38 -0
  346. regscale/models/regscale_models/control_implementation.py +1128 -0
  347. regscale/models/regscale_models/control_objective.py +261 -0
  348. regscale/models/regscale_models/control_parameter.py +100 -0
  349. regscale/models/regscale_models/control_test.py +34 -0
  350. regscale/models/regscale_models/control_test_plan.py +75 -0
  351. regscale/models/regscale_models/control_test_result.py +52 -0
  352. regscale/models/regscale_models/custom_field.py +245 -0
  353. regscale/models/regscale_models/data.py +109 -0
  354. regscale/models/regscale_models/data_center.py +40 -0
  355. regscale/models/regscale_models/deviation.py +203 -0
  356. regscale/models/regscale_models/email.py +97 -0
  357. regscale/models/regscale_models/evidence.py +47 -0
  358. regscale/models/regscale_models/evidence_mapping.py +40 -0
  359. regscale/models/regscale_models/facility.py +59 -0
  360. regscale/models/regscale_models/file.py +382 -0
  361. regscale/models/regscale_models/filetag.py +37 -0
  362. regscale/models/regscale_models/form_field_value.py +94 -0
  363. regscale/models/regscale_models/group.py +169 -0
  364. regscale/models/regscale_models/implementation_objective.py +335 -0
  365. regscale/models/regscale_models/implementation_option.py +275 -0
  366. regscale/models/regscale_models/implementation_role.py +33 -0
  367. regscale/models/regscale_models/incident.py +177 -0
  368. regscale/models/regscale_models/interconnection.py +43 -0
  369. regscale/models/regscale_models/issue.py +1176 -0
  370. regscale/models/regscale_models/leveraged_authorization.py +125 -0
  371. regscale/models/regscale_models/line_of_inquiry.py +52 -0
  372. regscale/models/regscale_models/link.py +205 -0
  373. regscale/models/regscale_models/meta_data.py +64 -0
  374. regscale/models/regscale_models/mixins/__init__.py +0 -0
  375. regscale/models/regscale_models/mixins/parent_cache.py +124 -0
  376. regscale/models/regscale_models/module.py +224 -0
  377. regscale/models/regscale_models/modules.py +191 -0
  378. regscale/models/regscale_models/objective.py +14 -0
  379. regscale/models/regscale_models/parameter.py +87 -0
  380. regscale/models/regscale_models/ports_protocol.py +81 -0
  381. regscale/models/regscale_models/privacy.py +89 -0
  382. regscale/models/regscale_models/profile.py +50 -0
  383. regscale/models/regscale_models/profile_link.py +68 -0
  384. regscale/models/regscale_models/profile_mapping.py +124 -0
  385. regscale/models/regscale_models/project.py +63 -0
  386. regscale/models/regscale_models/property.py +278 -0
  387. regscale/models/regscale_models/question.py +85 -0
  388. regscale/models/regscale_models/questionnaire.py +87 -0
  389. regscale/models/regscale_models/questionnaire_instance.py +177 -0
  390. regscale/models/regscale_models/rbac.py +132 -0
  391. regscale/models/regscale_models/reference.py +86 -0
  392. regscale/models/regscale_models/regscale_model.py +1643 -0
  393. regscale/models/regscale_models/requirement.py +29 -0
  394. regscale/models/regscale_models/risk.py +274 -0
  395. regscale/models/regscale_models/sbom.py +54 -0
  396. regscale/models/regscale_models/scan_history.py +436 -0
  397. regscale/models/regscale_models/search.py +53 -0
  398. regscale/models/regscale_models/security_control.py +132 -0
  399. regscale/models/regscale_models/security_plan.py +204 -0
  400. regscale/models/regscale_models/software_inventory.py +159 -0
  401. regscale/models/regscale_models/stake_holder.py +64 -0
  402. regscale/models/regscale_models/stig.py +647 -0
  403. regscale/models/regscale_models/supply_chain.py +152 -0
  404. regscale/models/regscale_models/system_role.py +188 -0
  405. regscale/models/regscale_models/system_role_external_assignment.py +40 -0
  406. regscale/models/regscale_models/tag.py +37 -0
  407. regscale/models/regscale_models/tag_mapping.py +19 -0
  408. regscale/models/regscale_models/task.py +133 -0
  409. regscale/models/regscale_models/threat.py +196 -0
  410. regscale/models/regscale_models/user.py +175 -0
  411. regscale/models/regscale_models/user_group.py +55 -0
  412. regscale/models/regscale_models/vulnerability.py +242 -0
  413. regscale/models/regscale_models/vulnerability_mapping.py +162 -0
  414. regscale/models/regscale_models/workflow.py +55 -0
  415. regscale/models/regscale_models/workflow_action.py +34 -0
  416. regscale/models/regscale_models/workflow_instance.py +269 -0
  417. regscale/models/regscale_models/workflow_instance_step.py +114 -0
  418. regscale/models/regscale_models/workflow_template.py +58 -0
  419. regscale/models/regscale_models/workflow_template_step.py +45 -0
  420. regscale/regscale.py +815 -0
  421. regscale/utils/__init__.py +7 -0
  422. regscale/utils/b64conversion.py +14 -0
  423. regscale/utils/click_utils.py +118 -0
  424. regscale/utils/decorators.py +48 -0
  425. regscale/utils/dict_utils.py +59 -0
  426. regscale/utils/files.py +79 -0
  427. regscale/utils/fxns.py +30 -0
  428. regscale/utils/graphql_client.py +113 -0
  429. regscale/utils/lists.py +16 -0
  430. regscale/utils/numbers.py +12 -0
  431. regscale/utils/shell.py +148 -0
  432. regscale/utils/string.py +121 -0
  433. regscale/utils/synqly_utils.py +165 -0
  434. regscale/utils/threading/__init__.py +8 -0
  435. regscale/utils/threading/threadhandler.py +131 -0
  436. regscale/utils/threading/threadsafe_counter.py +47 -0
  437. regscale/utils/threading/threadsafe_dict.py +242 -0
  438. regscale/utils/threading/threadsafe_list.py +83 -0
  439. regscale/utils/version.py +104 -0
  440. regscale/validation/__init__.py +0 -0
  441. regscale/validation/address.py +37 -0
  442. regscale/validation/record.py +48 -0
  443. regscale/visualization/__init__.py +5 -0
  444. regscale/visualization/click.py +34 -0
  445. regscale_cli-6.16.0.0.dist-info/LICENSE +21 -0
  446. regscale_cli-6.16.0.0.dist-info/METADATA +659 -0
  447. regscale_cli-6.16.0.0.dist-info/RECORD +481 -0
  448. regscale_cli-6.16.0.0.dist-info/WHEEL +5 -0
  449. regscale_cli-6.16.0.0.dist-info/entry_points.txt +6 -0
  450. regscale_cli-6.16.0.0.dist-info/top_level.txt +2 -0
  451. tests/fixtures/__init__.py +2 -0
  452. tests/fixtures/api.py +87 -0
  453. tests/fixtures/models.py +91 -0
  454. tests/fixtures/test_fixture.py +144 -0
  455. tests/mocks/__init__.py +0 -0
  456. tests/mocks/objects.py +3 -0
  457. tests/mocks/response.py +32 -0
  458. tests/mocks/xml.py +13 -0
  459. tests/regscale/__init__.py +0 -0
  460. tests/regscale/core/__init__.py +0 -0
  461. tests/regscale/core/test_api.py +232 -0
  462. tests/regscale/core/test_app.py +406 -0
  463. tests/regscale/core/test_login.py +37 -0
  464. tests/regscale/core/test_logz.py +66 -0
  465. tests/regscale/core/test_sbom_generator.py +87 -0
  466. tests/regscale/core/test_validation_utils.py +163 -0
  467. tests/regscale/core/test_version.py +78 -0
  468. tests/regscale/models/__init__.py +0 -0
  469. tests/regscale/models/test_asset.py +71 -0
  470. tests/regscale/models/test_config.py +26 -0
  471. tests/regscale/models/test_control_implementation.py +27 -0
  472. tests/regscale/models/test_import.py +97 -0
  473. tests/regscale/models/test_issue.py +36 -0
  474. tests/regscale/models/test_mapping.py +52 -0
  475. tests/regscale/models/test_platform.py +31 -0
  476. tests/regscale/models/test_regscale_model.py +346 -0
  477. tests/regscale/models/test_report.py +32 -0
  478. tests/regscale/models/test_tenable_integrations.py +118 -0
  479. tests/regscale/models/test_user_model.py +121 -0
  480. tests/regscale/test_about.py +19 -0
  481. tests/regscale/test_authorization.py +65 -0
@@ -0,0 +1,1462 @@
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ """Integrates Qualys assets and vulnerabilities into RegScale CLI"""
4
+ import os
5
+ import pprint
6
+ import traceback
7
+ from asyncio import sleep
8
+ from datetime import datetime, timedelta, timezone
9
+ from json import JSONDecodeError
10
+ from typing import Any, Optional, Tuple, Union
11
+ from urllib.parse import urljoin
12
+
13
+ import click
14
+ import requests
15
+ import xmltodict
16
+ from pathlib import Path
17
+ from requests import Session
18
+ from rich.progress import TaskID
19
+
20
+ from regscale.core.app.logz import create_logger
21
+ from regscale.core.app.utils.app_utils import (
22
+ check_file_path,
23
+ check_license,
24
+ create_progress_object,
25
+ error_and_exit,
26
+ get_current_datetime,
27
+ save_data_to,
28
+ )
29
+ from regscale.core.app.utils.file_utils import download_from_s3
30
+ from regscale.models.app_models.click import regscale_ssp_id
31
+ from regscale.models import Asset, Issue, Search, regscale_models
32
+ from regscale.models.app_models.click import NotRequiredIf, save_output_to
33
+ from regscale.models.integration_models.flat_file_importer import FlatFileImporter
34
+ from regscale.models.integration_models.qualys import (
35
+ Qualys,
36
+ QualysContainerScansImporter,
37
+ QualysWasScansImporter,
38
+ QualysPolicyScansImporter,
39
+ )
40
+ from regscale.models.integration_models.qualys_scanner import QualysTotalCloudIntegration
41
+
42
+ ####################################################################################################
43
+ #
44
+ # Qualys API Documentation:
45
+ # https://qualysguard.qg2.apps.qualys.com/qwebhelp/fo_portal/api_doc/index.htm
46
+ #
47
+ ####################################################################################################
48
+
49
+
50
+ # create global variables for the entire module
51
+ logger = create_logger()
52
+
53
+ # create progress object to add tasks to for real time updates
54
+ job_progress = create_progress_object()
55
+ HEADERS = {"X-Requested-With": "RegScale CLI"}
56
+ QUALYS_API = Session()
57
+
58
+
59
+ # Create group to handle Qualys commands
60
+ @click.group()
61
+ def qualys():
62
+ """Performs actions from the Qualys API"""
63
+
64
+
65
+ @qualys.command(name="export_scans")
66
+ @save_output_to()
67
+ @click.option(
68
+ "--days",
69
+ type=int,
70
+ default=30,
71
+ help="The number of days to go back for completed scans, default is 30.",
72
+ )
73
+ @click.option(
74
+ "--export",
75
+ type=click.BOOL,
76
+ help="To disable saving the scans as a .json file, use False. Defaults to True.",
77
+ default=True,
78
+ prompt=False,
79
+ required=False,
80
+ )
81
+ def export_past_scans(save_output_to: Path, days: int, export: bool = True):
82
+ """Export scans from Qualys Host that were completed
83
+ in the last x days, defaults to last 30 days
84
+ and defaults to save it as a .json file"""
85
+ export_scans(
86
+ save_path=save_output_to,
87
+ days=days,
88
+ export=export,
89
+ )
90
+
91
+
92
+ @qualys.command(name="import_scans")
93
+ @FlatFileImporter.common_scanner_options(
94
+ message="File path to the folder containing Aqua .csv files to process to RegScale.",
95
+ prompt="File path for Qualys files",
96
+ import_name="qualys",
97
+ )
98
+ @click.option(
99
+ "--skip_rows",
100
+ type=click.INT,
101
+ help="The number of rows in the file to skip to get to the column headers, defaults to 129.",
102
+ default=129,
103
+ )
104
+ def import_scans(
105
+ folder_path: os.PathLike[str],
106
+ regscale_ssp_id: int,
107
+ scan_date: datetime,
108
+ mappings_path: os.PathLike[str],
109
+ disable_mapping: bool,
110
+ skip_rows: int,
111
+ s3_bucket: str,
112
+ s3_prefix: str,
113
+ aws_profile: str,
114
+ upload_file: bool,
115
+ ):
116
+ """Import scans from Qualys"""
117
+ import_qualys_scans(
118
+ folder_path=folder_path,
119
+ regscale_ssp_id=regscale_ssp_id,
120
+ scan_date=scan_date,
121
+ mappings_path=mappings_path,
122
+ disable_mapping=disable_mapping,
123
+ skip_rows=skip_rows,
124
+ s3_bucket=s3_bucket,
125
+ s3_prefix=s3_prefix,
126
+ aws_profile=aws_profile,
127
+ upload_file=upload_file,
128
+ )
129
+
130
+
131
+ def import_qualys_scans(
132
+ folder_path: os.PathLike[str],
133
+ regscale_ssp_id: int,
134
+ scan_date: datetime,
135
+ mappings_path: os.PathLike[str],
136
+ disable_mapping: bool,
137
+ skip_rows: int,
138
+ s3_bucket: str,
139
+ s3_prefix: str,
140
+ aws_profile: str,
141
+ upload_file: Optional[bool] = True,
142
+ ) -> None:
143
+ """
144
+ Import scans from Qualys
145
+
146
+ :param os.PathLike[str] folder_path: File path to the folder containing Qualys .csv files to process to RegScale
147
+ :param int regscale_ssp_id: The RegScale SSP ID
148
+ :param datetime scan_date: The date of the scan
149
+ :param os.PathLike[str] mappings_path: The path to the mappings file
150
+ :param bool disable_mapping: Whether to disable custom mappings
151
+ :param int skip_rows: The number of rows in the file to skip to get to the column headers
152
+ :param str s3_bucket: The S3 bucket to download the files from
153
+ :param str s3_prefix: The S3 prefix to download the files from
154
+ :param str aws_profile: The AWS profile to use for S3 access
155
+ :param Optional[bool] upload_file: Whether to upload the file to RegScale after processing, defaults to True
156
+ :rtype: None
157
+ """
158
+ FlatFileImporter.import_files(
159
+ import_type=Qualys,
160
+ import_name="Qualys",
161
+ file_types=".csv",
162
+ folder_path=folder_path,
163
+ regscale_ssp_id=regscale_ssp_id,
164
+ scan_date=scan_date,
165
+ mappings_path=mappings_path,
166
+ disable_mapping=disable_mapping,
167
+ s3_bucket=s3_bucket,
168
+ s3_prefix=s3_prefix,
169
+ aws_profile=aws_profile,
170
+ upload_file=upload_file,
171
+ skip_rows=skip_rows,
172
+ )
173
+
174
+
175
+ @qualys.command(name="save_results")
176
+ @save_output_to()
177
+ @click.option(
178
+ "--scan_id",
179
+ type=click.STRING,
180
+ help="Qualys scan reference ID to get results, defaults to all.",
181
+ default="all",
182
+ )
183
+ def save_results(save_output_to: Path, scan_id: str):
184
+ """Get scan results from Qualys using a scan ID or all scans and save them to a .json file."""
185
+ save_scan_results_by_id(save_path=save_output_to, scan_id=scan_id)
186
+
187
+
188
+ @qualys.command(name="sync_qualys")
189
+ @click.option(
190
+ "--regscale_ssp_id",
191
+ type=click.INT,
192
+ required=True,
193
+ prompt="Enter RegScale System Security Plan ID",
194
+ help="The ID number from RegScale of the System Security Plan",
195
+ )
196
+ @click.option(
197
+ "--create_issue",
198
+ type=click.BOOL,
199
+ required=False,
200
+ help="Create Issue in RegScale from vulnerabilities in Qualys.",
201
+ default=False,
202
+ )
203
+ @click.option(
204
+ "--asset_group_id",
205
+ type=click.INT,
206
+ help="Filter assets from Qualys with an asset group ID.",
207
+ default=None,
208
+ cls=NotRequiredIf,
209
+ not_required_if=["asset_group_name"],
210
+ )
211
+ @click.option(
212
+ "--asset_group_name",
213
+ type=click.STRING,
214
+ help="Filter assets from Qualys with an asset group name.",
215
+ default=None,
216
+ cls=NotRequiredIf,
217
+ not_required_if=["asset_group_id"],
218
+ )
219
+ def sync_qualys(
220
+ regscale_ssp_id: int,
221
+ create_issue: bool = False,
222
+ asset_group_id: int = None,
223
+ asset_group_name: str = None,
224
+ ):
225
+ """
226
+ Query Qualys and sync assets & their associated
227
+ vulnerabilities to a Security Plan in RegScale.
228
+ """
229
+ sync_qualys_to_regscale(
230
+ regscale_ssp_id=regscale_ssp_id,
231
+ create_issue=create_issue,
232
+ asset_group_id=asset_group_id,
233
+ asset_group_name=asset_group_name,
234
+ )
235
+
236
+
237
+ @qualys.command(name="get_asset_groups")
238
+ @save_output_to()
239
+ def get_asset_groups(save_output_to: Path):
240
+ """
241
+ Get all asset groups from Qualys via API and save them to a .json file.
242
+ """
243
+ # see if user has enterprise license
244
+ check_license()
245
+
246
+ date = get_current_datetime("%Y%m%d")
247
+ check_file_path(save_output_to)
248
+ asset_groups = get_asset_groups_from_qualys()
249
+ save_data_to(
250
+ file=Path(f"{save_output_to}/qualys_asset_groups_{date}.json"),
251
+ data=asset_groups,
252
+ )
253
+
254
+
255
+ @qualys.command(name="import_container_scans")
256
+ @FlatFileImporter.common_scanner_options(
257
+ message="File path to the folder containing container .csv files to process to RegScale.",
258
+ prompt="File path for Qualys files",
259
+ import_name="qualys_container_scan",
260
+ )
261
+ @click.option(
262
+ "--skip_rows",
263
+ type=click.INT,
264
+ help="The number of rows in the file to skip to get to the column headers, defaults to 5.",
265
+ default=5,
266
+ )
267
+ def import_container_scans(
268
+ folder_path: os.PathLike[str],
269
+ regscale_ssp_id: int,
270
+ scan_date: datetime,
271
+ mappings_path: Path,
272
+ disable_mapping: bool,
273
+ s3_bucket: str,
274
+ s3_prefix: str,
275
+ aws_profile: str,
276
+ upload_file: bool,
277
+ skip_rows: int,
278
+ ):
279
+ """
280
+ Import Qualys container scans from a CSV file into a RegScale Security Plan as assets and vulnerabilities.
281
+ """
282
+ process_files_with_importer(
283
+ folder_path=str(folder_path),
284
+ importer_class=QualysContainerScansImporter,
285
+ regscale_ssp_id=regscale_ssp_id,
286
+ importer_args={
287
+ "plan_id": regscale_ssp_id,
288
+ "name": "QualysContainerScan",
289
+ "parent_id": regscale_ssp_id,
290
+ "parent_module": "securityplans",
291
+ "scan_date": scan_date,
292
+ },
293
+ mappings_path=str(mappings_path),
294
+ disable_mapping=disable_mapping,
295
+ skip_rows=skip_rows,
296
+ s3_bucket=s3_bucket,
297
+ s3_prefix=s3_prefix,
298
+ aws_profile=aws_profile,
299
+ upload_file=upload_file,
300
+ )
301
+
302
+
303
+ @qualys.command(name="import_was_scans")
304
+ @FlatFileImporter.common_scanner_options(
305
+ message="File path to the folder containing was .csv files to process to RegScale.",
306
+ prompt="File path for Qualys files",
307
+ import_name="qualys_was_scan",
308
+ )
309
+ @click.option(
310
+ "--skip_rows",
311
+ type=click.INT,
312
+ help="The number of rows in the file to skip to get to the column headers, defaults to 5.",
313
+ default=5,
314
+ )
315
+ def import_was_scans(
316
+ folder_path: os.PathLike[str],
317
+ regscale_ssp_id: int,
318
+ scan_date: datetime,
319
+ mappings_path: Path,
320
+ disable_mapping: bool,
321
+ skip_rows: int,
322
+ s3_bucket: str,
323
+ s3_prefix: str,
324
+ aws_profile: str,
325
+ upload_file: bool,
326
+ ):
327
+ """
328
+ Import Qualys was scans from a CSV file into a RegScale Security Plan as assets and vulnerabilities.
329
+ """
330
+ process_files_with_importer(
331
+ folder_path=str(folder_path),
332
+ importer_class=QualysWasScansImporter,
333
+ regscale_ssp_id=regscale_ssp_id,
334
+ importer_args={
335
+ "plan_id": regscale_ssp_id,
336
+ "name": "QualysWASScan",
337
+ "parent_id": regscale_ssp_id,
338
+ "parent_module": "securityplans",
339
+ "scan_date": scan_date,
340
+ },
341
+ mappings_path=str(mappings_path),
342
+ disable_mapping=disable_mapping,
343
+ skip_rows=skip_rows,
344
+ s3_bucket=s3_bucket,
345
+ s3_prefix=s3_prefix,
346
+ aws_profile=aws_profile,
347
+ upload_file=upload_file,
348
+ )
349
+
350
+
351
+ @qualys.command(name="import_total_cloud")
352
+ @regscale_ssp_id()
353
+ @click.option(
354
+ "--include_tags",
355
+ "-t",
356
+ type=click.STRING,
357
+ required=False,
358
+ default=None,
359
+ help="Include tags in the import comma seperated string of tag names or ids, defaults to None.",
360
+ )
361
+ @click.option(
362
+ "--exclude_tags",
363
+ "-e",
364
+ type=click.STRING,
365
+ required=False,
366
+ default=None,
367
+ help="Exclude tags in the import comma seperated string of tag names or ids, defaults to None.",
368
+ )
369
+ def import_total_cloud_assets_and_vulnerabilities(regscale_ssp_id: int, include_tags: str, exclude_tags: str):
370
+ """
371
+ Import Qualys Total Cloud Assets and Vulnerabilities into RegScale via API."""
372
+ import_total_cloud_data_from_qualys_api(
373
+ security_plan_id=regscale_ssp_id, include_tags=include_tags, exclude_tags=exclude_tags
374
+ )
375
+
376
+
377
+ @qualys.command(name="import_policy_scans")
378
+ @FlatFileImporter.common_scanner_options(
379
+ message="File path to the folder containing policy .csv files to process to RegScale.",
380
+ prompt="File path for Qualys files",
381
+ import_name="qualys_policy_scan",
382
+ )
383
+ @click.option(
384
+ "--skip_rows",
385
+ type=click.INT,
386
+ help="The number of rows in the file to skip to get to the column headers, defaults to 5.",
387
+ default=5,
388
+ )
389
+ def import_policy_scans(
390
+ folder_path: os.PathLike[str],
391
+ regscale_ssp_id: int,
392
+ scan_date: datetime,
393
+ mappings_path: Path,
394
+ disable_mapping: bool,
395
+ skip_rows: int,
396
+ s3_bucket: str,
397
+ s3_prefix: str,
398
+ aws_profile: str,
399
+ upload_file: bool,
400
+ ):
401
+ """
402
+ Import Qualys policy scans from a CSV file into a RegScale Security Plan as assets and vulnerabilities.
403
+ """
404
+ process_files_with_importer(
405
+ folder_path=str(folder_path),
406
+ importer_class=QualysPolicyScansImporter,
407
+ regscale_ssp_id=regscale_ssp_id,
408
+ importer_args={
409
+ "plan_id": regscale_ssp_id,
410
+ "name": "QualysPolicyScan",
411
+ "parent_id": regscale_ssp_id,
412
+ "parent_module": "securityplans",
413
+ "scan_date": scan_date,
414
+ },
415
+ mappings_path=str(mappings_path),
416
+ disable_mapping=disable_mapping,
417
+ skip_rows=skip_rows,
418
+ s3_bucket=s3_bucket,
419
+ s3_prefix=s3_prefix,
420
+ aws_profile=aws_profile,
421
+ upload_file=upload_file,
422
+ )
423
+
424
+
425
+ def process_files_with_importer(
426
+ regscale_ssp_id: int,
427
+ folder_path: str,
428
+ importer_class,
429
+ importer_args: dict,
430
+ s3_bucket: str,
431
+ s3_prefix: str,
432
+ aws_profile: str,
433
+ mappings_path: str = None,
434
+ disable_mapping: bool = False,
435
+ skip_rows: int = 0,
436
+ scan_date: datetime = None,
437
+ upload_file: Optional[bool] = True,
438
+ ):
439
+ """
440
+ Process files in a folder using a specified importer class.
441
+
442
+ :param int regscale_ssp_id: ID of the RegScale Security Plan to import the data into.
443
+ :param str folder_path: Path to the folder containing files.
444
+ :param Any importer_class: The importer class to instantiate for processing.
445
+ :param dict importer_args: Additional arguments to pass to the importer class.
446
+ :param str s3_bucket: S3 bucket to download the files from.
447
+ :param str s3_prefix: S3 prefix to download the files from.
448
+ :param str aws_profile: AWS profile to use for S3 access.
449
+ :param str mappings_path: Path to mapping configurations.
450
+ :param bool disable_mapping: Flag to disable mappings.
451
+ :param int skip_rows: Number of rows to skip in files.
452
+ :param scan_date: Date of the scan. Defaults to current datetime if not provided.
453
+ :param Optional[bool] upload_file: Whether to upload the file to RegScale after processing, defaults to True.
454
+ """
455
+ import csv
456
+ from openpyxl import Workbook
457
+
458
+ if s3_bucket:
459
+ download_from_s3(s3_bucket, s3_prefix, folder_path, aws_profile)
460
+
461
+ files_lst = list(Path(folder_path).glob("*.csv"))
462
+
463
+ # If no files are found in the folder, return a warning
464
+ if len(files_lst) == 0:
465
+ logger.warning("No Qualys files found in the folder path provided.")
466
+ return
467
+
468
+ if not scan_date:
469
+ scan_date = datetime.now(timezone.utc)
470
+
471
+ for file in files_lst:
472
+ try:
473
+ original_file_name = str(file)
474
+ xlsx_file = (
475
+ f"{file.name}.xlsx" if not file.name.endswith(".csv") else str(file.name).replace(".csv", ".xlsx")
476
+ )
477
+
478
+ # Convert CSV to XLSX
479
+ wb = Workbook()
480
+ ws = wb.active
481
+ with open(file, "r") as f:
482
+ for row in csv.reader(f):
483
+ ws.append(row)
484
+
485
+ # Save the Excel file
486
+ full_file_path = Path(f"{file.parent}/{xlsx_file}")
487
+ wb.save(full_file_path)
488
+
489
+ # Initialize and use the importer
490
+ importer = importer_class(
491
+ plan_id=regscale_ssp_id,
492
+ name=importer_args.get("name", "QualysFileScan"),
493
+ file_path=str(full_file_path),
494
+ parent_id=regscale_ssp_id,
495
+ parent_module=importer_args.get("parent_module", "securityplans"),
496
+ scan_date=scan_date,
497
+ mappings_path=mappings_path,
498
+ disable_mapping=disable_mapping,
499
+ skip_rows=skip_rows,
500
+ upload_file=upload_file,
501
+ )
502
+ importer.clean_up(file_path=original_file_name)
503
+ except Exception as e:
504
+ error_message = traceback.format_exc()
505
+ logger.error(f"Failed to process file {file}: {error_message}\n{e}")
506
+ continue
507
+
508
+
509
+ def export_scans(
510
+ save_path: Path,
511
+ days: int = 30,
512
+ export: bool = True,
513
+ ) -> None:
514
+ """
515
+ Function to export scans from Qualys that were completed in the last x days, defaults to 30
516
+
517
+ :param Path save_path: Path to save the scans to as a .json file
518
+ :param int days: # of days of completed scans to export, defaults to 30 days
519
+ :param bool export: Whether to save the scan data as a .json, defaults to True
520
+ :rtype: None
521
+ """
522
+ # see if user has enterprise license
523
+ check_license()
524
+ date = get_current_datetime("%Y%m%d")
525
+ results = get_detailed_scans(days)
526
+ if export:
527
+ check_file_path(save_path)
528
+ save_data_to(
529
+ file=Path(f"{save_path.name}/qualys_scans_{date}.json"),
530
+ data=results,
531
+ )
532
+ else:
533
+ pprint.pprint(results, indent=4)
534
+
535
+
536
+ def save_scan_results_by_id(save_path: Path, scan_id: str) -> None:
537
+ """
538
+ Function to save the queries from Qualys using an ID a .json file
539
+
540
+ :param Path save_path: Path to save the scan results to as a .json file
541
+ :param str scan_id: Qualys scan ID to get the results for
542
+ :rtype: None
543
+ """
544
+ # see if user has enterprise license
545
+ check_license()
546
+
547
+ check_file_path(save_path)
548
+ with job_progress:
549
+ if scan_id.lower() == "all":
550
+ # get all the scan results from Qualys
551
+ scans = get_scans_summary("all")
552
+
553
+ # add task to job progress to let user know # of scans to fetch
554
+ task1 = job_progress.add_task(
555
+ f"[#f8b737]Getting scan results for {len(scans['SCAN'])} scan(s)...",
556
+ total=len(scans["SCAN"]),
557
+ )
558
+ # get the scan results from Qualys
559
+ scan_data = get_scan_results(scans, task1)
560
+ else:
561
+ task1 = job_progress.add_task(f"[#f8b737]Getting scan results for {scan_id}...", total=1)
562
+ # get the scan result for the provided scan id
563
+ scan_data = get_scan_results(scan_id, task1)
564
+ # save the scan_data as the provided file_path
565
+ save_data_to(file=save_path, data=scan_data)
566
+
567
+
568
+ def sync_qualys_to_regscale(
569
+ regscale_ssp_id: int,
570
+ create_issue: bool = False,
571
+ asset_group_id: int = None,
572
+ asset_group_name: str = None,
573
+ ) -> None:
574
+ """
575
+ Sync Qualys assets and vulnerabilities to a security plan in RegScale
576
+
577
+ :param int regscale_ssp_id: ID # of the SSP in RegScale
578
+ :param bool create_issue: Flag whether to create an issue in RegScale from Qualys vulnerabilities, defaults to False
579
+ :param int asset_group_id: Optional filter for assets in Qualys with an asset group ID, defaults to None
580
+ :param str asset_group_name: Optional filter for assets in Qualys with an asset group name, defaults to None
581
+ :rtype: None
582
+ """
583
+ # see if user has enterprise license
584
+ check_license()
585
+
586
+ # check if the user provided an asset group id or name
587
+ if asset_group_id:
588
+ # get the assets from Qualys using the group name
589
+ sync_qualys_assets_and_vulns(
590
+ ssp_id=regscale_ssp_id,
591
+ create_issue=create_issue,
592
+ asset_group_filter=asset_group_name,
593
+ )
594
+ elif asset_group_name:
595
+ # get the assets from Qualys using the group name
596
+ sync_qualys_assets_and_vulns(
597
+ ssp_id=regscale_ssp_id,
598
+ create_issue=create_issue,
599
+ asset_group_filter=asset_group_id,
600
+ )
601
+ else:
602
+ sync_qualys_assets_and_vulns(ssp_id=regscale_ssp_id, create_issue=create_issue)
603
+
604
+
605
+ def get_scan_results(scans: Any, task: TaskID) -> dict:
606
+ """
607
+ Function to retrieve scan results from Qualys using provided scan list and returns a dictionary
608
+
609
+ :param Any scans: list of scans to retrieve from Qualys
610
+ :param TaskID task: task to update in the progress object
611
+ :return: dictionary of detailed Qualys scans
612
+ :rtype: dict
613
+ """
614
+ app = check_license()
615
+ config = app.config
616
+
617
+ # set the auth for the QUALYS_API session
618
+ QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
619
+
620
+ scan_data = {}
621
+ # check number of scans requested
622
+ if isinstance(scans, str):
623
+ # only one scan was requested, set up variable for the for loop
624
+ scans = {"SCAN": [{"REF": scans}]}
625
+ for scan in scans["SCAN"]:
626
+ # set up data and parameters for the scans query
627
+ try:
628
+ # try and get the scan id ref #
629
+ scan_id = scan["REF"]
630
+ # set the parameters for the Qualys API call
631
+ params = {
632
+ "action": "fetch",
633
+ "scan_ref": scan_id,
634
+ "mode": "extended",
635
+ "output_format": "json_extended",
636
+ }
637
+ # get the scan data via API
638
+ res = QUALYS_API.get(
639
+ url=urljoin(config["qualysUrl"], "/api/2.0/fo/scan/"),
640
+ headers=HEADERS,
641
+ params=params,
642
+ )
643
+ # convert response to json
644
+ if res.status_code == 200:
645
+ try:
646
+ res_data = res.json()
647
+ scan_data[scan_id] = res_data
648
+ except JSONDecodeError:
649
+ error_and_exit("Unable to convert response to JSON.")
650
+ else:
651
+ error_and_exit(f"Received unexpected response from Qualys API: {res.status_code}: {res.text}")
652
+ except KeyError:
653
+ # unable to get the scan id ref #
654
+ continue
655
+ job_progress.update(task, advance=1)
656
+ return scan_data
657
+
658
+
659
+ def get_detailed_scans(days: int) -> list:
660
+ """
661
+ function to get the list of all scans from Qualys using QUALYS_API
662
+
663
+ :param int days: # of days before today to filter scans
664
+ :return: list of results from Qualys API
665
+ :rtype: list
666
+ """
667
+ app = check_license()
668
+ config = app.config
669
+
670
+ # set the auth for the QUALYS_API session
671
+ QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
672
+
673
+ today = datetime.now()
674
+ scan_date = today - timedelta(days=days)
675
+
676
+ # set up data and parameters for the scans query
677
+ params = {
678
+ "action": "list",
679
+ "scan_date_since": scan_date.strftime("%Y-%m-%d"),
680
+ "output_format": "json",
681
+ }
682
+ params2 = {
683
+ "action": "list",
684
+ "scan_datetime_since": scan_date.strftime("%Y-%m-%dT%H:%I:%S%ZZ"),
685
+ }
686
+ res = QUALYS_API.get(
687
+ url=urljoin(config["qualysUrl"], "/api/2.0/fo/scan/summary/"),
688
+ headers=HEADERS,
689
+ params=params,
690
+ )
691
+ response = QUALYS_API.get(
692
+ url=urljoin(config["qualysUrl"], "/api/2.0/fo/scan/vm/summary/"),
693
+ headers=HEADERS,
694
+ params=params2,
695
+ )
696
+ # convert response to json
697
+ res_data = res.json()
698
+ try:
699
+ response_data = xmltodict.parse(response.text)["SCAN_SUMMARY_OUTPUT"]["RESPONSE"]["SCAN_SUMMARY_LIST"][
700
+ "SCAN_SUMMARY"
701
+ ]
702
+ if len(res_data) < 1:
703
+ res_data = response_data
704
+ else:
705
+ res_data.extend(response_data)
706
+ except JSONDecodeError:
707
+ logger.error("ERROR: Unable to convert to JSON.")
708
+ return res_data
709
+
710
+
711
+ def import_total_cloud_data_from_qualys_api(security_plan_id: int, include_tags: str, exclude_tags: str):
712
+ """
713
+ Function to get the total cloud data from Qualys API
714
+ :param int security_plan_id: The ID of the plan to get the data for
715
+ :param str include_tags: The tags to include in the data
716
+ :param str exclude_tags: The tags to exclude from the data
717
+ """
718
+ try:
719
+
720
+ app = check_license()
721
+ config = app.config
722
+ # set the auth for the QUALYS_API session
723
+ QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
724
+ params = {
725
+ "action": "list",
726
+ "show_asset_id": "1",
727
+ "show_tags": "1",
728
+ }
729
+ if exclude_tags or include_tags:
730
+ params["use_tags"] = "1"
731
+ if exclude_tags:
732
+ params["tag_set_exclude"] = exclude_tags
733
+ if include_tags:
734
+ params["tag_set_include"] = include_tags
735
+ response = QUALYS_API.get(
736
+ url=urljoin(config["qualysUrl"], "/api/2.0/fo/asset/host/vm/detection/"),
737
+ headers=HEADERS,
738
+ params=params,
739
+ )
740
+ if response and response.ok:
741
+ response_data = xmltodict.parse(response.text)
742
+ qt = QualysTotalCloudIntegration(plan_id=security_plan_id, xml_data=response_data)
743
+ qt.fetch_assets()
744
+ qt.fetch_findings()
745
+
746
+ else:
747
+ logger.error(
748
+ f"Received unexpected response from Qualys API: {response.status_code}: {response.text if response.text else 'response is null'}"
749
+ )
750
+ except Exception:
751
+ error_message = traceback.format_exc()
752
+ logger.error("Error occurred while processing Qualys data")
753
+ logger.error(error_message)
754
+
755
+
756
+ def get_scans_summary(scan_choice: str) -> dict:
757
+ """
758
+ Get all scans from Qualys Host
759
+
760
+ :param str scan_choice: The type of scan to retrieve from Qualys API
761
+ :return: Detailed summary of scans from Qualys API as a dictionary
762
+ :rtype: dict
763
+ """
764
+ app = check_license()
765
+ config = app.config
766
+ urls = []
767
+
768
+ # set the auth for the QUALYS_API session
769
+ QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
770
+
771
+ # set up variables for function
772
+ scan_data = {}
773
+ responses = []
774
+ scan_url = urljoin(config["qualysUrl"], "/api/2.0/fo/scan/")
775
+
776
+ # set up parameters for the scans query
777
+ params = {"action": "list"}
778
+ # check what scan list was requested and set urls list accordingly
779
+ if scan_choice.lower() == "all":
780
+ urls = [scan_url, scan_url + "compliance", scan_url + "scap"]
781
+ elif scan_choice.lower() == "vm":
782
+ urls = [scan_url]
783
+ elif scan_choice.lower() in ["compliance", "scap"]:
784
+ urls = [scan_url + scan_choice.lower()]
785
+ # get the list of vm scans
786
+ for url in urls:
787
+ # get the scan data
788
+ response = QUALYS_API.get(url=url, headers=HEADERS, params=params)
789
+ # store response into a list
790
+ responses.append(response)
791
+ # check the responses received for data
792
+ for response in responses:
793
+ # see if response was successful
794
+ if response.status_code == 200:
795
+ # parse the data
796
+ data = xmltodict.parse(response.text)["SCAN_LIST_OUTPUT"]["RESPONSE"]
797
+ # see if the scan has any data
798
+ try:
799
+ # add the data to the scan_data dictionary
800
+ scan_data.update(data["SCAN_LIST"])
801
+ except KeyError:
802
+ # no data found, continue the for loop
803
+ continue
804
+ return scan_data
805
+
806
+
807
+ def get_scan_details(days: int) -> list:
808
+ """
809
+ Retrieve completed scans from last x days from Qualys Host
810
+
811
+ :param int days: # of days before today to filter scans
812
+ :return: Detailed summary of scans from Qualys API as a dictionary
813
+ :rtype: list
814
+ """
815
+ app = check_license()
816
+ config = app.config
817
+
818
+ # set the auth for the QUALYS_API session
819
+ QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
820
+ # get since date for API call
821
+ since_date = datetime.now() - timedelta(days=days)
822
+ # set up data and parameters for the scans query
823
+ headers = {
824
+ "Content-Type": "application/json",
825
+ "Accept": "application/json",
826
+ "X-Requested-With": "RegScale CLI",
827
+ }
828
+ params = {
829
+ "action": "list",
830
+ "scan_date_since": since_date.strftime("%Y-%m-%d"),
831
+ "output_format": "json",
832
+ }
833
+ params2 = {
834
+ "action": "list",
835
+ "scan_datetime_since": since_date.strftime("%Y-%m-%dT%H:%M:%SZ"),
836
+ }
837
+ res = QUALYS_API.get(
838
+ url=urljoin(config["qualysUrl"], "/api/2.0/fo/scan/summary/"),
839
+ headers=headers,
840
+ params=params,
841
+ )
842
+ response = QUALYS_API.get(
843
+ url=urljoin(config["qualysUrl"], "/api/2.0/fo/scan/vm/summary/"),
844
+ headers=headers,
845
+ params=params2,
846
+ )
847
+ # convert response to json
848
+ res_data = res.json()
849
+ try:
850
+ response_data = xmltodict.parse(response.text)["SCAN_SUMMARY_OUTPUT"]["RESPONSE"]["SCAN_SUMMARY_LIST"][
851
+ "SCAN_SUMMARY"
852
+ ]
853
+ if len(res_data) < 1:
854
+ res_data = response_data
855
+ else:
856
+ res_data.update(response_data)
857
+ except JSONDecodeError as ex:
858
+ error_and_exit(f"Unable to convert to JSON.\n{ex}")
859
+ except KeyError:
860
+ error_and_exit(f"No data found.\n{response.text}")
861
+ return res_data
862
+
863
+
864
+ def sync_qualys_assets_and_vulns(
865
+ ssp_id: int,
866
+ create_issue: bool,
867
+ asset_group_filter: Optional[Union[int, str]] = None,
868
+ ) -> None:
869
+ """
870
+ Function to query Qualys and sync assets & associated vulnerabilities to RegScale
871
+
872
+ :param int ssp_id: RegScale System Security Plan ID
873
+ :param bool create_issue: Flag to create an issue in RegScale for each vulnerability from Qualys
874
+ :param Optional[Union[int, str]] asset_group_filter: Filter the Qualys assets by an asset group ID or name, if any
875
+ :rtype: None
876
+ """
877
+ app = check_license()
878
+ config = app.config
879
+
880
+ # set the auth for the QUALYS_API session
881
+ QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
882
+
883
+ # Get the assets from RegScale with the provided SSP ID
884
+ logger.info("Getting assets from RegScale for SSP #%s...", ssp_id)
885
+ reg_assets = Asset.get_all_by_search(search=Search(parentID=ssp_id, module="securityplans"))
886
+ logger.info(
887
+ "Located %s asset(s) associated with SSP #%s in RegScale.",
888
+ len(reg_assets),
889
+ ssp_id,
890
+ )
891
+ logger.debug(reg_assets)
892
+
893
+ if qualys_assets := get_qualys_assets_and_scan_results(asset_group_filter):
894
+ logger.info("Received %s assets from Qualys.", len(qualys_assets))
895
+ logger.debug(qualys_assets)
896
+ else:
897
+ error_and_exit("No assets found in Qualys.")
898
+ sync_assets(
899
+ qualys_assets=qualys_assets,
900
+ reg_assets=reg_assets,
901
+ ssp_id=ssp_id,
902
+ config=config,
903
+ )
904
+ if create_issue:
905
+ # Get vulnerabilities from Qualys for the Qualys assets
906
+ logger.info("Getting vulnerabilities for %s asset(s) from Qualys...", len(qualys_assets))
907
+ qualys_assets_and_issues, total_vuln_count = get_issue_data_for_assets(qualys_assets)
908
+ logger.info("Received %s vulnerabilities from Qualys.", total_vuln_count)
909
+ logger.debug(qualys_assets_and_issues)
910
+ sync_issues(
911
+ ssp_id=ssp_id,
912
+ qualys_assets_and_issues=qualys_assets_and_issues,
913
+ )
914
+
915
+
916
+ def sync_assets(qualys_assets: list[dict], reg_assets: list[Asset], ssp_id: int, config: dict) -> None:
917
+ """
918
+ Function to sync Qualys assets to RegScale
919
+
920
+ :param list[dict] qualys_assets: List of Qualys assets
921
+ :param list[Asset] reg_assets: List of RegScale assets
922
+ :param int ssp_id: RegScale System Security Plan ID
923
+ :param dict config: Configuration dictionary
924
+ :rtype: None
925
+ """
926
+ update_assets = []
927
+ for qualys_asset in qualys_assets: # you can list as many input dicts as you want here
928
+ # Update parent id to SSP on insert
929
+ if lookup_assets := lookup_asset(reg_assets, qualys_asset["ASSET_ID"]):
930
+ for asset in set(lookup_assets):
931
+ asset.parentId = ssp_id
932
+ asset.parentModule = "securityplans"
933
+ asset.otherTrackingNumber = qualys_asset["ID"]
934
+ asset.ipAddress = qualys_asset["IP"]
935
+ asset.qualysId = qualys_asset["ASSET_ID"]
936
+ try:
937
+ assert asset.id
938
+ # avoid duplication
939
+ if asset.qualysId not in [v["qualysId"] for v in update_assets]:
940
+ update_assets.append(asset)
941
+ except AssertionError as aex:
942
+ logger.error("Asset does not have an id, unable to update!\n%s", aex)
943
+ update_and_insert_assets(
944
+ qualys_assets=qualys_assets, reg_assets=reg_assets, ssp_id=ssp_id, config=config, update_assets=update_assets
945
+ )
946
+
947
+
948
+ def update_and_insert_assets(
949
+ qualys_assets: list[dict], reg_assets: list[Asset], ssp_id: int, config: dict, update_assets: list[Asset]
950
+ ) -> None:
951
+ """
952
+ Function to update and insert Qualys assets into RegScale
953
+
954
+ :param list[dict] qualys_assets: List of Qualys assets as dictionaries
955
+ :param list[Asset] reg_assets: List of RegScale assets
956
+ :param int ssp_id: RegScale System Security Plan ID
957
+ :param dict config: RegScale CLI Configuration dictionary
958
+ :param list[Asset] update_assets: List of assets to update in RegScale
959
+ :rtype: None
960
+ """
961
+ insert_assets = []
962
+ if assets_to_be_inserted := [
963
+ qualys_asset
964
+ for qualys_asset in qualys_assets
965
+ if qualys_asset["ASSET_ID"] not in [asset["ASSET_ID"] for asset in inner_join(reg_assets, qualys_assets)]
966
+ ]:
967
+ for qualys_asset in assets_to_be_inserted:
968
+ # Do Insert
969
+ r_asset = Asset(
970
+ name=f'Qualys Asset #{qualys_asset["ASSET_ID"]} IP: {qualys_asset["IP"]}',
971
+ otherTrackingNumber=qualys_asset["ID"],
972
+ parentId=ssp_id,
973
+ parentModule="securityplans",
974
+ ipAddress=qualys_asset["IP"],
975
+ assetOwnerId=config["userId"],
976
+ assetType="Other",
977
+ assetCategory=regscale_models.AssetCategory.Hardware,
978
+ status="Off-Network",
979
+ qualysId=qualys_asset["ASSET_ID"],
980
+ )
981
+ # avoid duplication
982
+ if r_asset.qualysId not in set(v["qualysId"] for v in insert_assets):
983
+ insert_assets.append(r_asset)
984
+ try:
985
+ created_assets = Asset.batch_create(insert_assets, job_progress)
986
+ logger.info(
987
+ "RegScale Asset(s) successfully created: %i/%i",
988
+ len(created_assets),
989
+ len(insert_assets),
990
+ )
991
+ except requests.exceptions.RequestException as rex:
992
+ logger.error("Unable to create Qualys Assets in RegScale\n%s", rex)
993
+ if update_assets:
994
+ try:
995
+ updated_assets = Asset.batch_update(update_assets, job_progress)
996
+ logger.info(
997
+ "RegScale Asset(s) successfully updated: %i/%i",
998
+ len(updated_assets),
999
+ len(update_assets),
1000
+ )
1001
+ except requests.RequestException as rex:
1002
+ logger.error("Unable to Update Qualys Assets to RegScale\n%s", rex)
1003
+
1004
+
1005
+ def sync_issues(ssp_id: int, qualys_assets_and_issues: list[dict]) -> None:
1006
+ """
1007
+ Function to sync Qualys issues to RegScale
1008
+
1009
+ :param int ssp_id: RegScale System Security Plan ID
1010
+ :param list[dict] qualys_assets_and_issues: List of Qualys assets and their issues
1011
+ :rtype: None
1012
+ """
1013
+ update_issues = []
1014
+ insert_issues = []
1015
+ vuln_count = 0
1016
+ ssp_assets = Asset.get_all_by_parent(parent_id=ssp_id, parent_module="securityplans")
1017
+ for asset in qualys_assets_and_issues:
1018
+ # Create issues in RegScale from Qualys vulnerabilities
1019
+ regscale_issue_updates, regscale_new_issues = create_regscale_issue_from_vuln(
1020
+ regscale_ssp_id=ssp_id, qualys_asset=asset, regscale_assets=ssp_assets, vulns=asset["ISSUES"]
1021
+ )
1022
+ update_issues.extend(regscale_issue_updates)
1023
+ insert_issues.extend(regscale_new_issues)
1024
+ vuln_count += len(asset.get("ISSUES", []))
1025
+ if insert_issues:
1026
+ deduped_vulns = combine_duplicate_qualys_vulns(insert_issues)
1027
+ logger.info(
1028
+ "Creating %i new issue(s) in RegScale, condensed from %i Qualys vulnerabilities.",
1029
+ len(deduped_vulns),
1030
+ vuln_count,
1031
+ )
1032
+ created_issues = Issue.batch_create(deduped_vulns, job_progress)
1033
+ logger.info(
1034
+ "RegScale Issue(s) successfully created: %i/%i",
1035
+ len(created_issues),
1036
+ len(deduped_vulns),
1037
+ )
1038
+ if update_issues:
1039
+ deduped_vulns = combine_duplicate_qualys_vulns(update_issues)
1040
+ logger.info(
1041
+ "Updating %i existing issue(s) in RegScale, condensed from %i Qualys vulnerabilities.",
1042
+ len(deduped_vulns),
1043
+ vuln_count,
1044
+ )
1045
+ updated_issues = Issue.batch_update(deduped_vulns, job_progress)
1046
+ logger.info("RegScale Issue(s) successfully updated: %i/%i", len(updated_issues), len(deduped_vulns))
1047
+
1048
+
1049
+ def combine_duplicate_qualys_vulns(qualys_vulns: list[Issue]) -> list:
1050
+ """
1051
+ Function to combine duplicate Qualys vulnerabilities
1052
+
1053
+ :param list[Issue] qualys_vulns: List of Qualys vulnerabilities as RegScale issues
1054
+ :return: List of Qualys vulnerabilities with duplicates combined
1055
+ :rtype: list
1056
+ """
1057
+ with job_progress:
1058
+ logger.info("Combining duplicate Qualys vulnerabilities found across multiple assets...")
1059
+ deduping_task = job_progress.add_task(
1060
+ f"Combining {len(qualys_vulns)} Qualys vulnerabilities...",
1061
+ total=len(qualys_vulns),
1062
+ )
1063
+ combined_vulns: dict[str, Issue] = {}
1064
+ for vuln in qualys_vulns:
1065
+ if vuln.qualysId in combined_vulns:
1066
+ if current_identifier := combined_vulns[vuln.qualysId].assetIdentifier:
1067
+ combined_vulns[vuln.qualysId].assetIdentifier = update_asset_identifier(
1068
+ vuln.assetIdentifier, current_identifier
1069
+ )
1070
+ else:
1071
+ combined_vulns[vuln.qualysId].assetIdentifier = vuln.assetIdentifier
1072
+ else:
1073
+ combined_vulns[vuln.qualysId] = vuln
1074
+ job_progress.update(deduping_task, advance=1)
1075
+ return list(combined_vulns.values())
1076
+
1077
+
1078
+ def get_qualys_assets_and_scan_results(
1079
+ url: Optional[str] = None, asset_group_filter: Optional[Union[int, str]] = None
1080
+ ) -> list:
1081
+ """
1082
+ function to gather all assets from Qualys API host along with their scan results
1083
+
1084
+ :param Optional[str] url: URL to get the assets from, defaults to None, used for pagination
1085
+ :param Optional[Union[int, str]] asset_group_filter: Qualys asset group ID or name to filter by, if provided
1086
+ :return: list of dictionaries containing asset data
1087
+ :rtype: list
1088
+ """
1089
+ app = check_license()
1090
+ config = app.config
1091
+
1092
+ # set the auth for the QUALYS_API session
1093
+ QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
1094
+ # set url
1095
+ if not url:
1096
+ url = urljoin(config["qualysUrl"], "api/2.0/fo/asset/host/vm/detection?action=list&show_asset_id=1")
1097
+
1098
+ # check if an asset group filter was provided and append it to the url
1099
+ if asset_group_filter:
1100
+ if isinstance(asset_group_filter, str):
1101
+ # Get the asset group ID from Qualys
1102
+ url += f"&ag_titles={asset_group_filter}"
1103
+ logger.info("Getting assets from Qualys by group name: %s...", asset_group_filter)
1104
+ else:
1105
+ url += f"&ag_ids={asset_group_filter}"
1106
+ logger.info(
1107
+ "Getting assets from from Qualys by group ID: #%s...",
1108
+ asset_group_filter,
1109
+ )
1110
+ else:
1111
+ # Get all assets from Qualys
1112
+ logger.info("Getting all assets from Qualys...")
1113
+
1114
+ # get the data via Qualys API host
1115
+ response = QUALYS_API.get(url=url, headers=HEADERS)
1116
+ res_data = xmltodict.parse(response.text)
1117
+
1118
+ try:
1119
+ # parse the xml data from response.text and convert it to a dictionary
1120
+ # and try to extract the data from the parsed XML dictionary
1121
+ asset_data = res_data["HOST_LIST_VM_DETECTION_OUTPUT"]["RESPONSE"]["HOST_LIST"]["HOST"]
1122
+ # check if we need to paginate the asset data
1123
+ if "WARNING" in res_data["HOST_LIST_VM_DETECTION_OUTPUT"]["RESPONSE"]:
1124
+ logger.warning("Not all assets were fetched, fetching more assets from Qualys...")
1125
+ asset_data.extend(
1126
+ get_qualys_assets_and_scan_results(
1127
+ url=res_data["HOST_LIST_VM_DETECTION_OUTPUT"]["RESPONSE"]["WARNING"]["URL"],
1128
+ asset_group_filter=asset_group_filter,
1129
+ )
1130
+ )
1131
+ except KeyError:
1132
+ # if there is a KeyError set the dictionary to nothing
1133
+ asset_data = []
1134
+ # return the asset_data variable
1135
+ return asset_data
1136
+
1137
+
1138
+ def get_issue_data_for_assets(asset_list: list) -> Tuple[list[dict], int]:
1139
+ """
1140
+ Function to get issue data from Qualys via API for assets in Qualys
1141
+
1142
+ :param list asset_list: Assets and their scan results from Qualys
1143
+ :return: Updated asset list of Qualys assets and their associated vulnerabilities, total number of vulnerabilities
1144
+ :rtype: Tuple[list[dict], int]
1145
+ """
1146
+ app = check_license()
1147
+ config = app.config
1148
+
1149
+ # set the auth for the QUALYS_API session
1150
+ QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
1151
+
1152
+ with job_progress:
1153
+ issues = {}
1154
+ for asset in asset_list:
1155
+ # check if the asset has any vulnerabilities
1156
+ if vulns := asset.get("DETECTION_LIST", {}).get("DETECTION", {}):
1157
+ asset_vulns = {}
1158
+ analyzing_vulns = job_progress.add_task(
1159
+ f"Analyzing {len(vulns)} vulnerabilities for asset #{asset['ASSET_ID']} from Qualys..."
1160
+ )
1161
+ # iterate through the vulnerabilities & verify they have a confirmed status
1162
+ for vuln in vulns:
1163
+ if vuln["TYPE"] == "Confirmed":
1164
+ issues[vuln["QID"]] = vuln
1165
+ asset_vulns[vuln["QID"]] = vuln
1166
+ job_progress.update(analyzing_vulns, advance=1)
1167
+ job_progress.update(analyzing_vulns, completed=len(vulns))
1168
+ # add the issues to the asset's dictionary
1169
+ asset["ISSUES"] = asset_vulns
1170
+ job_progress.remove_task(analyzing_vulns)
1171
+ asset_list = fetch_vulns_from_qualys(issue_ids=list(issues.keys()), asset_list=asset_list, config=config)
1172
+ return asset_list, len(issues)
1173
+
1174
+
1175
+ def parse_and_map_vuln_data(xml_data: str) -> dict:
1176
+ """
1177
+ Function to parse Qualys vulnerability data from XML and map it to a dictionary using the Qualys ID as the key
1178
+
1179
+ :param str xml_data: XML data from Qualys API
1180
+ :return: Dictionary of Qualys vulnerability data
1181
+ :rtype: dict
1182
+ """
1183
+ issue_data = (
1184
+ xmltodict.parse(xml_data)
1185
+ .get("KNOWLEDGE_BASE_VULN_LIST_OUTPUT", {})
1186
+ .get("RESPONSE", {})
1187
+ .get("VULN_LIST", {})
1188
+ .get("VULN", {})
1189
+ )
1190
+ # change the key for the fetched issues to be the qualys ID
1191
+ return {issue["QID"]: issue for issue in issue_data}
1192
+
1193
+
1194
+ def fetch_vulns_from_qualys(issue_ids: list[str], asset_list: list[dict], config: dict, retries: int = 0) -> list[dict]:
1195
+ """
1196
+ Function to fetch vulnerability data from Qualys for a list of issues and assets
1197
+
1198
+ :param list[str] issue_ids: List of Qualys issue IDs to fetch data for
1199
+ :param list[dict] asset_list: List of Qualys assets to update with vulnerability data
1200
+ :param dict config: CLI Configuration dictionary
1201
+ :param int retries: Number of retries for fetching data, defaults to 0
1202
+ :return: Updated asset list with vulnerability data
1203
+ :rtype: list[dict]
1204
+ """
1205
+ logger.info(
1206
+ f"Getting vulnerability data for {len(issue_ids)} issue(s) from Qualys for {len(asset_list)} asset(s)..."
1207
+ )
1208
+ base_url = urljoin(config["qualysUrl"], "api/2.0/fo/knowledge_base/vuln?action=list&details=All")
1209
+ if len(issue_ids) > 100:
1210
+ logger.warning(
1211
+ "Too many issues to fetch from Qualys. Downloading the Qualys database to prevent rate limits..."
1212
+ )
1213
+ # since there are a lot of vulnerabilities, download the database and reference it locally
1214
+ chunk_size_calc = 20 * 1024
1215
+ with QUALYS_API.post(
1216
+ url=base_url,
1217
+ headers=HEADERS,
1218
+ stream=True,
1219
+ ) as response:
1220
+ check_file_path("artifacts")
1221
+ with open("./artifacts/qualys_vuln_db.xml", "wb") as f:
1222
+ for chunk in response.iter_content(chunk_size=chunk_size_calc):
1223
+ f.write(chunk)
1224
+ with open("./artifacts/qualys_vuln_db.xml", "r") as f:
1225
+ qualys_issue_data = parse_and_map_vuln_data(f.read())
1226
+ else:
1227
+ response = QUALYS_API.get(
1228
+ url=f"{base_url}&ids={','.join(issue_ids)}",
1229
+ headers=HEADERS,
1230
+ )
1231
+ if response.ok:
1232
+ qualys_issue_data = parse_and_map_vuln_data(response.text)
1233
+ logger.info("Received vulnerability data for %s issues from Qualys.", len(qualys_issue_data))
1234
+ elif response.status_code == 409:
1235
+ response_data = xmltodict.parse(response.text)["SIMPLE_RETURN"]["RESPONSE"]
1236
+ logger.warning(
1237
+ "Received timeout error from Qualys API: %s. Waiting %s seconds...",
1238
+ response_data["TEXT"],
1239
+ response_data["ITEM_LIST"]["ITEM"]["VALUE"],
1240
+ )
1241
+ sleep(int(response_data["ITEM_LIST"]["ITEM"]["VALUE"]))
1242
+ if retries < 3:
1243
+ fetch_vulns_from_qualys(issue_ids, asset_list, config, retries + 1)
1244
+ else:
1245
+ error_and_exit(
1246
+ "Unable to fetch vulnerability data from Qualys after 3 attempts. Please try again later."
1247
+ )
1248
+ else:
1249
+ error_and_exit(
1250
+ f"Received unexpected response from Qualys: {response.status_code}: {response.text}: {response.reason}"
1251
+ )
1252
+ return map_issue_data_to_assets(asset_list, qualys_issue_data)
1253
+
1254
+
1255
+ def map_issue_data_to_assets(assets: list[dict], qualys_issue_data: dict) -> list[dict]:
1256
+ """
1257
+ Function to map Qualys issue data to Qualys assets
1258
+
1259
+ :param list[dict] assets: List of Qualys assets to map issue data to
1260
+ :param dict qualys_issue_data: List of Qualys issues to map to assets
1261
+ :return: Updated asset list with Qualys issue data
1262
+ :rtype: list[dict]
1263
+ """
1264
+ for asset in assets:
1265
+ if issues := asset.get("ISSUES"):
1266
+ mapping_vulns = job_progress.add_task(
1267
+ f"Mapping {len(issues)} vulnerabilities to Asset #{asset['ASSET_ID']} from Qualys...",
1268
+ total=len(issues),
1269
+ )
1270
+ for issue in issues:
1271
+ if issue in qualys_issue_data:
1272
+ issues[issue]["ISSUE_DATA"] = qualys_issue_data[issue]
1273
+ job_progress.update(mapping_vulns, advance=1)
1274
+ job_progress.remove_task(mapping_vulns)
1275
+ return assets
1276
+
1277
+
1278
+ def lookup_asset(asset_list: list, asset_id: str = None) -> list[Asset]:
1279
+ """
1280
+ Function to look up an asset in the asset list and returns an Asset object
1281
+
1282
+ :param list asset_list: List of assets from RegScale
1283
+ :param str asset_id: Qualys asset ID to search for, defaults to None
1284
+ :return: list of Asset objects
1285
+ :rtype: list[Asset]
1286
+ """
1287
+ if asset_id:
1288
+ results = [Asset(**asset) for asset in asset_list if getattr(asset, "qualysId", None) == asset_id]
1289
+ else:
1290
+ results = [Asset(**asset) for asset in asset_list]
1291
+ # Return unique list
1292
+ return list(set(results)) or []
1293
+
1294
+
1295
+ def map_qualys_severity_to_regscale(severity: int) -> tuple[str, str]:
1296
+ """
1297
+ Map Qualys vulnerability severity to RegScale Issue severity
1298
+
1299
+ :param int severity: Qualys vulnerability severity
1300
+ :return: RegScale Issue severity and key for init.yaml
1301
+ :rtype: tuple[str, str]
1302
+ """
1303
+ if severity <= 2:
1304
+ return "III - Low - Other Weakness", "low"
1305
+ if severity == 3:
1306
+ return "II - Moderate - Reportable Condition", "moderate"
1307
+ if severity > 3:
1308
+ return "I - High - Significant Deficiency", "high"
1309
+ return "IV - Not Assigned", "low"
1310
+
1311
+
1312
+ def create_regscale_issue_from_vuln(
1313
+ regscale_ssp_id: int, qualys_asset: dict, regscale_assets: list[Asset], vulns: dict
1314
+ ) -> Tuple[list[Issue], list[Issue]]:
1315
+ """
1316
+ Sync Qualys vulnerabilities to RegScale issues.
1317
+
1318
+ :param int regscale_ssp_id: RegScale SSP ID
1319
+ :param dict qualys_asset: Qualys asset as a dictionary
1320
+ :param list[Asset] regscale_assets: list of RegScale assets
1321
+ :param dict vulns: dictionary of Qualys vulnerabilities associated with the provided asset
1322
+ :return: list of RegScale issues to update, and a list of issues to be created
1323
+ :rtype: Tuple[list[Issue], list[Issue]]
1324
+ """
1325
+ app = check_license()
1326
+ config = app.config
1327
+
1328
+ # set the auth for the QUALYS_API session
1329
+ QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
1330
+ default_status = config["issues"]["qualys"]["status"]
1331
+ regscale_issues = []
1332
+ regscale_existing_issues = Issue.get_all_by_parent(parent_id=regscale_ssp_id, parent_module="securityplans")
1333
+ for vuln in vulns.values():
1334
+ asset_identifier = None
1335
+ severity, key = map_qualys_severity_to_regscale(int(vuln["SEVERITY"]))
1336
+
1337
+ default_due_delta = config["issues"]["qualys"][key]
1338
+ logger.debug("Processing vulnerability# %s", vuln["QID"])
1339
+ fmt = "%Y-%m-%dT%H:%M:%SZ"
1340
+ due_date = datetime.strptime(vuln["LAST_FOUND_DATETIME"], fmt) + timedelta(days=default_due_delta)
1341
+ regscale_asset = [asset for asset in regscale_assets if asset.qualysId == qualys_asset["ASSET_ID"]]
1342
+ if "DNS" not in qualys_asset.keys() or "IP" not in qualys_asset.keys():
1343
+ if regscale_asset:
1344
+ asset_identifier = f"RegScale Asset #{regscale_asset[0].id}: {regscale_asset[0].name}"
1345
+ else:
1346
+ if regscale_asset:
1347
+ asset_identifier = (
1348
+ f'RegScale Asset #{regscale_asset[0].id}: {regscale_asset[0].name} Qualys DNS: "'
1349
+ f'{qualys_asset["DNS"]} - IP: {qualys_asset["IP"]}'
1350
+ )
1351
+ else:
1352
+ asset_identifier = f'DNS: {qualys_asset["DNS"]} - IP: {qualys_asset["IP"]}'
1353
+ issue = Issue(
1354
+ title=vuln["ISSUE_DATA"]["TITLE"],
1355
+ description=vuln["ISSUE_DATA"]["CONSEQUENCE"] + "</br>" + vuln["ISSUE_DATA"]["DIAGNOSIS"],
1356
+ issueOwnerId=config["userId"],
1357
+ status=default_status,
1358
+ severityLevel=severity,
1359
+ qualysId=vuln["QID"],
1360
+ dueDate=due_date.strftime(fmt),
1361
+ identification="Vulnerability Assessment",
1362
+ parentId=regscale_ssp_id,
1363
+ parentModule="securityplans",
1364
+ recommendedActions=vuln["ISSUE_DATA"]["SOLUTION"],
1365
+ assetIdentifier=asset_identifier,
1366
+ )
1367
+ regscale_issues.append(issue)
1368
+ regscale_new_issues, regscale_update_issues = determine_issue_update_or_create(
1369
+ regscale_issues, regscale_existing_issues
1370
+ )
1371
+ return regscale_update_issues, regscale_new_issues
1372
+
1373
+
1374
+ def update_asset_identifier(new_identifier: Optional[str], current_identifier: Optional[str]) -> Optional[str]:
1375
+ """
1376
+ Function to update the asset identifier for a RegScale issue
1377
+
1378
+ :param Optional[str] new_identifier: New asset identifier to add
1379
+ :param Optional[str] current_identifier: Current asset identifier
1380
+ :return: Updated asset identifier
1381
+ :rtype: str
1382
+ """
1383
+ if not current_identifier and new_identifier:
1384
+ return new_identifier
1385
+ if current_identifier and new_identifier:
1386
+ if new_identifier not in current_identifier:
1387
+ return f"{current_identifier}<br>{new_identifier}"
1388
+ if new_identifier in current_identifier:
1389
+ return current_identifier
1390
+ if new_identifier == current_identifier:
1391
+ return current_identifier
1392
+
1393
+
1394
+ def determine_issue_update_or_create(
1395
+ qualys_issues: list[Issue], regscale_issues: list[Issue]
1396
+ ) -> Tuple[list[Issue], list[Issue]]:
1397
+ """
1398
+ Function to determine if Qualys issues needs to be updated or created in RegScale
1399
+
1400
+ :param list[Issue] qualys_issues: List of Qualys issues
1401
+ :param list[Issue] regscale_issues: List of existing RegScale issues
1402
+ :return: List of new issues and list of issues to update
1403
+ :rtype: Tuple[list[Issue], list[Issue]]
1404
+ """
1405
+ new_issues = []
1406
+ update_issues = []
1407
+ for issue in qualys_issues:
1408
+ if issue.qualysId in [iss.qualysId for iss in regscale_issues]:
1409
+ update_issue = [iss for iss in regscale_issues if iss.qualysId == issue.qualysId][0]
1410
+ # Check if we need to concatenate the asset identifier
1411
+ update_issue.assetIdentifier = update_asset_identifier(issue.assetIdentifier, update_issue.assetIdentifier)
1412
+ update_issues.append(update_issue)
1413
+ else:
1414
+ new_issues.append(issue)
1415
+ return new_issues, update_issues
1416
+
1417
+
1418
+ def inner_join(reg_list: list, qualys_list: list) -> list:
1419
+ """
1420
+ Function to compare assets from Qualys and assets from RegScale
1421
+
1422
+ :param list reg_list: list of assets from RegScale
1423
+ :param list qualys_list: list of assets from Qualys
1424
+ :return: list of assets that are in both RegScale and Qualys
1425
+ :rtype: list
1426
+ """
1427
+
1428
+ set1 = set(getattr(lst, "qualysId", None) for lst in reg_list)
1429
+ data = []
1430
+ try:
1431
+ data = [list_qualys for list_qualys in qualys_list if getattr(list_qualys, "ASSET_ID", None) in set1]
1432
+ except KeyError as ex:
1433
+ logger.error(ex)
1434
+ return data
1435
+
1436
+
1437
+ def get_asset_groups_from_qualys() -> list:
1438
+ """
1439
+ Get all asset groups from Qualys via API
1440
+
1441
+ :return: list of assets from Qualys
1442
+ :rtype: list
1443
+ """
1444
+ app = check_license()
1445
+ config = app.config
1446
+ asset_groups = []
1447
+
1448
+ # set the auth for the QUALYS_API session
1449
+ QUALYS_API.auth = (config["qualysUserName"], config["qualysPassword"])
1450
+ response = QUALYS_API.get(url=urljoin(config["qualysUrl"], "api/2.0/fo/asset/group?action=list"), headers=HEADERS)
1451
+ if response.ok:
1452
+ logger.debug(response.text)
1453
+ try:
1454
+ asset_groups = xmltodict.parse(response.text)["ASSET_GROUP_LIST_OUTPUT"]["RESPONSE"]["ASSET_GROUP_LIST"][
1455
+ "ASSET_GROUP"
1456
+ ]
1457
+ except KeyError:
1458
+ logger.debug(response.text)
1459
+ error_and_exit(
1460
+ f"Unable to retrieve asset groups from Qualys.\nReceived: #{response.status_code}: {response.text}"
1461
+ )
1462
+ return asset_groups