regscale-cli 6.16.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of regscale-cli might be problematic. Click here for more details.

Files changed (481) hide show
  1. regscale/__init__.py +1 -0
  2. regscale/airflow/__init__.py +9 -0
  3. regscale/airflow/azure/__init__.py +9 -0
  4. regscale/airflow/azure/cli.py +89 -0
  5. regscale/airflow/azure/upload_dags.py +116 -0
  6. regscale/airflow/click_dags.py +127 -0
  7. regscale/airflow/click_mixins.py +82 -0
  8. regscale/airflow/config.py +25 -0
  9. regscale/airflow/factories/__init__.py +0 -0
  10. regscale/airflow/factories/connections.py +58 -0
  11. regscale/airflow/factories/workflows.py +78 -0
  12. regscale/airflow/hierarchy.py +88 -0
  13. regscale/airflow/operators/__init__.py +0 -0
  14. regscale/airflow/operators/click.py +36 -0
  15. regscale/airflow/sensors/__init__.py +0 -0
  16. regscale/airflow/sensors/sql.py +107 -0
  17. regscale/airflow/sessions/__init__.py +0 -0
  18. regscale/airflow/sessions/sql/__init__.py +3 -0
  19. regscale/airflow/sessions/sql/queries.py +64 -0
  20. regscale/airflow/sessions/sql/sql_server_queries.py +248 -0
  21. regscale/airflow/tasks/__init__.py +0 -0
  22. regscale/airflow/tasks/branches.py +22 -0
  23. regscale/airflow/tasks/cli.py +116 -0
  24. regscale/airflow/tasks/click.py +73 -0
  25. regscale/airflow/tasks/debugging.py +9 -0
  26. regscale/airflow/tasks/groups.py +116 -0
  27. regscale/airflow/tasks/init.py +60 -0
  28. regscale/airflow/tasks/states.py +47 -0
  29. regscale/airflow/tasks/workflows.py +36 -0
  30. regscale/ansible/__init__.py +9 -0
  31. regscale/core/__init__.py +0 -0
  32. regscale/core/app/__init__.py +3 -0
  33. regscale/core/app/api.py +571 -0
  34. regscale/core/app/application.py +665 -0
  35. regscale/core/app/internal/__init__.py +136 -0
  36. regscale/core/app/internal/admin_actions.py +230 -0
  37. regscale/core/app/internal/assessments_editor.py +873 -0
  38. regscale/core/app/internal/catalog.py +316 -0
  39. regscale/core/app/internal/comparison.py +459 -0
  40. regscale/core/app/internal/control_editor.py +571 -0
  41. regscale/core/app/internal/encrypt.py +79 -0
  42. regscale/core/app/internal/evidence.py +1240 -0
  43. regscale/core/app/internal/file_uploads.py +151 -0
  44. regscale/core/app/internal/healthcheck.py +66 -0
  45. regscale/core/app/internal/login.py +305 -0
  46. regscale/core/app/internal/migrations.py +240 -0
  47. regscale/core/app/internal/model_editor.py +1701 -0
  48. regscale/core/app/internal/poam_editor.py +632 -0
  49. regscale/core/app/internal/workflow.py +105 -0
  50. regscale/core/app/logz.py +74 -0
  51. regscale/core/app/utils/XMLIR.py +258 -0
  52. regscale/core/app/utils/__init__.py +0 -0
  53. regscale/core/app/utils/api_handler.py +358 -0
  54. regscale/core/app/utils/app_utils.py +1110 -0
  55. regscale/core/app/utils/catalog_utils/__init__.py +0 -0
  56. regscale/core/app/utils/catalog_utils/common.py +91 -0
  57. regscale/core/app/utils/catalog_utils/compare_catalog.py +193 -0
  58. regscale/core/app/utils/catalog_utils/diagnostic_catalog.py +97 -0
  59. regscale/core/app/utils/catalog_utils/download_catalog.py +103 -0
  60. regscale/core/app/utils/catalog_utils/update_catalog.py +718 -0
  61. regscale/core/app/utils/catalog_utils/update_catalog_v2.py +1378 -0
  62. regscale/core/app/utils/catalog_utils/update_catalog_v3.py +1272 -0
  63. regscale/core/app/utils/catalog_utils/update_plans.py +334 -0
  64. regscale/core/app/utils/file_utils.py +238 -0
  65. regscale/core/app/utils/parser_utils.py +81 -0
  66. regscale/core/app/utils/pickle_file_handler.py +57 -0
  67. regscale/core/app/utils/regscale_utils.py +319 -0
  68. regscale/core/app/utils/report_utils.py +119 -0
  69. regscale/core/app/utils/variables.py +226 -0
  70. regscale/core/decorators.py +31 -0
  71. regscale/core/lazy_group.py +65 -0
  72. regscale/core/login.py +63 -0
  73. regscale/core/server/__init__.py +0 -0
  74. regscale/core/server/flask_api.py +473 -0
  75. regscale/core/server/helpers.py +373 -0
  76. regscale/core/server/rest.py +64 -0
  77. regscale/core/server/static/css/bootstrap.css +6030 -0
  78. regscale/core/server/static/css/bootstrap.min.css +6 -0
  79. regscale/core/server/static/css/main.css +176 -0
  80. regscale/core/server/static/images/regscale-cli.svg +49 -0
  81. regscale/core/server/static/images/regscale.svg +38 -0
  82. regscale/core/server/templates/base.html +74 -0
  83. regscale/core/server/templates/index.html +43 -0
  84. regscale/core/server/templates/login.html +28 -0
  85. regscale/core/server/templates/make_base64.html +22 -0
  86. regscale/core/server/templates/upload_STIG.html +109 -0
  87. regscale/core/server/templates/upload_STIG_result.html +26 -0
  88. regscale/core/server/templates/upload_ssp.html +144 -0
  89. regscale/core/server/templates/upload_ssp_result.html +128 -0
  90. regscale/core/static/__init__.py +0 -0
  91. regscale/core/static/regex.py +14 -0
  92. regscale/core/utils/__init__.py +117 -0
  93. regscale/core/utils/click_utils.py +13 -0
  94. regscale/core/utils/date.py +238 -0
  95. regscale/core/utils/graphql.py +254 -0
  96. regscale/core/utils/urls.py +23 -0
  97. regscale/dev/__init__.py +6 -0
  98. regscale/dev/analysis.py +454 -0
  99. regscale/dev/cli.py +235 -0
  100. regscale/dev/code_gen.py +492 -0
  101. regscale/dev/dirs.py +69 -0
  102. regscale/dev/docs.py +384 -0
  103. regscale/dev/monitoring.py +26 -0
  104. regscale/dev/profiling.py +216 -0
  105. regscale/exceptions/__init__.py +4 -0
  106. regscale/exceptions/license_exception.py +7 -0
  107. regscale/exceptions/validation_exception.py +9 -0
  108. regscale/integrations/__init__.py +1 -0
  109. regscale/integrations/commercial/__init__.py +486 -0
  110. regscale/integrations/commercial/ad.py +433 -0
  111. regscale/integrations/commercial/amazon/__init__.py +0 -0
  112. regscale/integrations/commercial/amazon/common.py +106 -0
  113. regscale/integrations/commercial/aqua/__init__.py +0 -0
  114. regscale/integrations/commercial/aqua/aqua.py +91 -0
  115. regscale/integrations/commercial/aws/__init__.py +6 -0
  116. regscale/integrations/commercial/aws/cli.py +322 -0
  117. regscale/integrations/commercial/aws/inventory/__init__.py +110 -0
  118. regscale/integrations/commercial/aws/inventory/base.py +64 -0
  119. regscale/integrations/commercial/aws/inventory/resources/__init__.py +19 -0
  120. regscale/integrations/commercial/aws/inventory/resources/compute.py +234 -0
  121. regscale/integrations/commercial/aws/inventory/resources/containers.py +113 -0
  122. regscale/integrations/commercial/aws/inventory/resources/database.py +101 -0
  123. regscale/integrations/commercial/aws/inventory/resources/integration.py +237 -0
  124. regscale/integrations/commercial/aws/inventory/resources/networking.py +253 -0
  125. regscale/integrations/commercial/aws/inventory/resources/security.py +240 -0
  126. regscale/integrations/commercial/aws/inventory/resources/storage.py +91 -0
  127. regscale/integrations/commercial/aws/scanner.py +823 -0
  128. regscale/integrations/commercial/azure/__init__.py +0 -0
  129. regscale/integrations/commercial/azure/common.py +32 -0
  130. regscale/integrations/commercial/azure/intune.py +488 -0
  131. regscale/integrations/commercial/azure/scanner.py +49 -0
  132. regscale/integrations/commercial/burp.py +78 -0
  133. regscale/integrations/commercial/cpe.py +144 -0
  134. regscale/integrations/commercial/crowdstrike.py +1117 -0
  135. regscale/integrations/commercial/defender.py +1511 -0
  136. regscale/integrations/commercial/dependabot.py +210 -0
  137. regscale/integrations/commercial/durosuite/__init__.py +0 -0
  138. regscale/integrations/commercial/durosuite/api.py +1546 -0
  139. regscale/integrations/commercial/durosuite/process_devices.py +101 -0
  140. regscale/integrations/commercial/durosuite/scanner.py +637 -0
  141. regscale/integrations/commercial/durosuite/variables.py +21 -0
  142. regscale/integrations/commercial/ecr.py +90 -0
  143. regscale/integrations/commercial/gcp/__init__.py +237 -0
  144. regscale/integrations/commercial/gcp/auth.py +96 -0
  145. regscale/integrations/commercial/gcp/control_tests.py +238 -0
  146. regscale/integrations/commercial/gcp/variables.py +18 -0
  147. regscale/integrations/commercial/gitlab.py +332 -0
  148. regscale/integrations/commercial/grype.py +165 -0
  149. regscale/integrations/commercial/ibm.py +90 -0
  150. regscale/integrations/commercial/import_all/__init__.py +0 -0
  151. regscale/integrations/commercial/import_all/import_all_cmd.py +467 -0
  152. regscale/integrations/commercial/import_all/scan_file_fingerprints.json +27 -0
  153. regscale/integrations/commercial/jira.py +1046 -0
  154. regscale/integrations/commercial/mappings/__init__.py +0 -0
  155. regscale/integrations/commercial/mappings/csf_controls.json +713 -0
  156. regscale/integrations/commercial/mappings/nist_800_53_r5_controls.json +1516 -0
  157. regscale/integrations/commercial/nessus/__init__.py +0 -0
  158. regscale/integrations/commercial/nessus/nessus_utils.py +429 -0
  159. regscale/integrations/commercial/nessus/scanner.py +416 -0
  160. regscale/integrations/commercial/nexpose.py +90 -0
  161. regscale/integrations/commercial/okta.py +798 -0
  162. regscale/integrations/commercial/opentext/__init__.py +0 -0
  163. regscale/integrations/commercial/opentext/click.py +99 -0
  164. regscale/integrations/commercial/opentext/scanner.py +143 -0
  165. regscale/integrations/commercial/prisma.py +91 -0
  166. regscale/integrations/commercial/qualys.py +1462 -0
  167. regscale/integrations/commercial/salesforce.py +980 -0
  168. regscale/integrations/commercial/sap/__init__.py +0 -0
  169. regscale/integrations/commercial/sap/click.py +31 -0
  170. regscale/integrations/commercial/sap/sysdig/__init__.py +0 -0
  171. regscale/integrations/commercial/sap/sysdig/click.py +57 -0
  172. regscale/integrations/commercial/sap/sysdig/sysdig_scanner.py +190 -0
  173. regscale/integrations/commercial/sap/tenable/__init__.py +0 -0
  174. regscale/integrations/commercial/sap/tenable/click.py +49 -0
  175. regscale/integrations/commercial/sap/tenable/scanner.py +196 -0
  176. regscale/integrations/commercial/servicenow.py +1756 -0
  177. regscale/integrations/commercial/sicura/__init__.py +0 -0
  178. regscale/integrations/commercial/sicura/api.py +855 -0
  179. regscale/integrations/commercial/sicura/commands.py +73 -0
  180. regscale/integrations/commercial/sicura/scanner.py +481 -0
  181. regscale/integrations/commercial/sicura/variables.py +16 -0
  182. regscale/integrations/commercial/snyk.py +90 -0
  183. regscale/integrations/commercial/sonarcloud.py +260 -0
  184. regscale/integrations/commercial/sqlserver.py +369 -0
  185. regscale/integrations/commercial/stig_mapper_integration/__init__.py +0 -0
  186. regscale/integrations/commercial/stig_mapper_integration/click_commands.py +38 -0
  187. regscale/integrations/commercial/stig_mapper_integration/mapping_engine.py +353 -0
  188. regscale/integrations/commercial/stigv2/__init__.py +0 -0
  189. regscale/integrations/commercial/stigv2/ckl_parser.py +349 -0
  190. regscale/integrations/commercial/stigv2/click_commands.py +95 -0
  191. regscale/integrations/commercial/stigv2/stig_integration.py +202 -0
  192. regscale/integrations/commercial/synqly/__init__.py +0 -0
  193. regscale/integrations/commercial/synqly/assets.py +46 -0
  194. regscale/integrations/commercial/synqly/ticketing.py +132 -0
  195. regscale/integrations/commercial/synqly/vulnerabilities.py +223 -0
  196. regscale/integrations/commercial/synqly_jira.py +840 -0
  197. regscale/integrations/commercial/tenablev2/__init__.py +0 -0
  198. regscale/integrations/commercial/tenablev2/authenticate.py +31 -0
  199. regscale/integrations/commercial/tenablev2/click.py +1584 -0
  200. regscale/integrations/commercial/tenablev2/scanner.py +504 -0
  201. regscale/integrations/commercial/tenablev2/stig_parsers.py +140 -0
  202. regscale/integrations/commercial/tenablev2/utils.py +78 -0
  203. regscale/integrations/commercial/tenablev2/variables.py +17 -0
  204. regscale/integrations/commercial/trivy.py +162 -0
  205. regscale/integrations/commercial/veracode.py +96 -0
  206. regscale/integrations/commercial/wizv2/WizDataMixin.py +97 -0
  207. regscale/integrations/commercial/wizv2/__init__.py +0 -0
  208. regscale/integrations/commercial/wizv2/click.py +429 -0
  209. regscale/integrations/commercial/wizv2/constants.py +1001 -0
  210. regscale/integrations/commercial/wizv2/issue.py +361 -0
  211. regscale/integrations/commercial/wizv2/models.py +112 -0
  212. regscale/integrations/commercial/wizv2/parsers.py +339 -0
  213. regscale/integrations/commercial/wizv2/sbom.py +115 -0
  214. regscale/integrations/commercial/wizv2/scanner.py +416 -0
  215. regscale/integrations/commercial/wizv2/utils.py +796 -0
  216. regscale/integrations/commercial/wizv2/variables.py +39 -0
  217. regscale/integrations/commercial/wizv2/wiz_auth.py +159 -0
  218. regscale/integrations/commercial/xray.py +91 -0
  219. regscale/integrations/integration/__init__.py +2 -0
  220. regscale/integrations/integration/integration.py +26 -0
  221. regscale/integrations/integration/inventory.py +17 -0
  222. regscale/integrations/integration/issue.py +100 -0
  223. regscale/integrations/integration_override.py +149 -0
  224. regscale/integrations/public/__init__.py +103 -0
  225. regscale/integrations/public/cisa.py +641 -0
  226. regscale/integrations/public/criticality_updater.py +70 -0
  227. regscale/integrations/public/emass.py +411 -0
  228. regscale/integrations/public/emass_slcm_import.py +697 -0
  229. regscale/integrations/public/fedramp/__init__.py +0 -0
  230. regscale/integrations/public/fedramp/appendix_parser.py +548 -0
  231. regscale/integrations/public/fedramp/click.py +479 -0
  232. regscale/integrations/public/fedramp/components.py +714 -0
  233. regscale/integrations/public/fedramp/docx_parser.py +259 -0
  234. regscale/integrations/public/fedramp/fedramp_cis_crm.py +1124 -0
  235. regscale/integrations/public/fedramp/fedramp_common.py +3181 -0
  236. regscale/integrations/public/fedramp/fedramp_docx.py +388 -0
  237. regscale/integrations/public/fedramp/fedramp_five.py +2343 -0
  238. regscale/integrations/public/fedramp/fedramp_traversal.py +138 -0
  239. regscale/integrations/public/fedramp/import_fedramp_r4_ssp.py +279 -0
  240. regscale/integrations/public/fedramp/import_workbook.py +495 -0
  241. regscale/integrations/public/fedramp/inventory_items.py +244 -0
  242. regscale/integrations/public/fedramp/mappings/__init__.py +0 -0
  243. regscale/integrations/public/fedramp/mappings/fedramp_r4_parts.json +7388 -0
  244. regscale/integrations/public/fedramp/mappings/fedramp_r5_params.json +8636 -0
  245. regscale/integrations/public/fedramp/mappings/fedramp_r5_parts.json +9605 -0
  246. regscale/integrations/public/fedramp/mappings/system_roles.py +34 -0
  247. regscale/integrations/public/fedramp/mappings/user.py +175 -0
  248. regscale/integrations/public/fedramp/mappings/values.py +141 -0
  249. regscale/integrations/public/fedramp/markdown_parser.py +150 -0
  250. regscale/integrations/public/fedramp/metadata.py +689 -0
  251. regscale/integrations/public/fedramp/models/__init__.py +59 -0
  252. regscale/integrations/public/fedramp/models/leveraged_auth_new.py +168 -0
  253. regscale/integrations/public/fedramp/models/poam_importer.py +522 -0
  254. regscale/integrations/public/fedramp/parts_mapper.py +107 -0
  255. regscale/integrations/public/fedramp/poam/__init__.py +0 -0
  256. regscale/integrations/public/fedramp/poam/scanner.py +851 -0
  257. regscale/integrations/public/fedramp/properties.py +201 -0
  258. regscale/integrations/public/fedramp/reporting.py +84 -0
  259. regscale/integrations/public/fedramp/resources.py +496 -0
  260. regscale/integrations/public/fedramp/rosetta.py +110 -0
  261. regscale/integrations/public/fedramp/ssp_logger.py +87 -0
  262. regscale/integrations/public/fedramp/system_characteristics.py +922 -0
  263. regscale/integrations/public/fedramp/system_control_implementations.py +582 -0
  264. regscale/integrations/public/fedramp/system_implementation.py +190 -0
  265. regscale/integrations/public/fedramp/xml_utils.py +87 -0
  266. regscale/integrations/public/nist_catalog.py +275 -0
  267. regscale/integrations/public/oscal.py +1946 -0
  268. regscale/integrations/public/otx.py +169 -0
  269. regscale/integrations/scanner_integration.py +2692 -0
  270. regscale/integrations/variables.py +25 -0
  271. regscale/models/__init__.py +7 -0
  272. regscale/models/app_models/__init__.py +5 -0
  273. regscale/models/app_models/catalog_compare.py +213 -0
  274. regscale/models/app_models/click.py +252 -0
  275. regscale/models/app_models/datetime_encoder.py +21 -0
  276. regscale/models/app_models/import_validater.py +321 -0
  277. regscale/models/app_models/mapping.py +260 -0
  278. regscale/models/app_models/pipeline.py +37 -0
  279. regscale/models/click_models.py +413 -0
  280. regscale/models/config.py +154 -0
  281. regscale/models/email_style.css +67 -0
  282. regscale/models/hierarchy.py +8 -0
  283. regscale/models/inspect_models.py +79 -0
  284. regscale/models/integration_models/__init__.py +0 -0
  285. regscale/models/integration_models/amazon_models/__init__.py +0 -0
  286. regscale/models/integration_models/amazon_models/inspector.py +262 -0
  287. regscale/models/integration_models/amazon_models/inspector_scan.py +206 -0
  288. regscale/models/integration_models/aqua.py +247 -0
  289. regscale/models/integration_models/azure_alerts.py +255 -0
  290. regscale/models/integration_models/base64.py +23 -0
  291. regscale/models/integration_models/burp.py +433 -0
  292. regscale/models/integration_models/burp_models.py +128 -0
  293. regscale/models/integration_models/cisa_kev_data.json +19333 -0
  294. regscale/models/integration_models/defender_data.py +93 -0
  295. regscale/models/integration_models/defenderimport.py +143 -0
  296. regscale/models/integration_models/drf.py +443 -0
  297. regscale/models/integration_models/ecr_models/__init__.py +0 -0
  298. regscale/models/integration_models/ecr_models/data.py +69 -0
  299. regscale/models/integration_models/ecr_models/ecr.py +239 -0
  300. regscale/models/integration_models/flat_file_importer.py +1079 -0
  301. regscale/models/integration_models/grype_import.py +247 -0
  302. regscale/models/integration_models/ibm.py +126 -0
  303. regscale/models/integration_models/implementation_results.py +85 -0
  304. regscale/models/integration_models/nexpose.py +140 -0
  305. regscale/models/integration_models/prisma.py +202 -0
  306. regscale/models/integration_models/qualys.py +720 -0
  307. regscale/models/integration_models/qualys_scanner.py +160 -0
  308. regscale/models/integration_models/sbom/__init__.py +0 -0
  309. regscale/models/integration_models/sbom/cyclone_dx.py +139 -0
  310. regscale/models/integration_models/send_reminders.py +620 -0
  311. regscale/models/integration_models/snyk.py +155 -0
  312. regscale/models/integration_models/synqly_models/__init__.py +0 -0
  313. regscale/models/integration_models/synqly_models/capabilities.json +1 -0
  314. regscale/models/integration_models/synqly_models/connector_types.py +22 -0
  315. regscale/models/integration_models/synqly_models/connectors/__init__.py +7 -0
  316. regscale/models/integration_models/synqly_models/connectors/assets.py +97 -0
  317. regscale/models/integration_models/synqly_models/connectors/ticketing.py +583 -0
  318. regscale/models/integration_models/synqly_models/connectors/vulnerabilities.py +169 -0
  319. regscale/models/integration_models/synqly_models/ocsf_mapper.py +331 -0
  320. regscale/models/integration_models/synqly_models/param.py +72 -0
  321. regscale/models/integration_models/synqly_models/synqly_model.py +733 -0
  322. regscale/models/integration_models/synqly_models/tenants.py +39 -0
  323. regscale/models/integration_models/tenable_models/__init__.py +0 -0
  324. regscale/models/integration_models/tenable_models/integration.py +187 -0
  325. regscale/models/integration_models/tenable_models/models.py +513 -0
  326. regscale/models/integration_models/trivy_import.py +231 -0
  327. regscale/models/integration_models/veracode.py +217 -0
  328. regscale/models/integration_models/xray.py +135 -0
  329. regscale/models/locking.py +100 -0
  330. regscale/models/platform.py +110 -0
  331. regscale/models/regscale_models/__init__.py +67 -0
  332. regscale/models/regscale_models/assessment.py +570 -0
  333. regscale/models/regscale_models/assessment_plan.py +52 -0
  334. regscale/models/regscale_models/asset.py +567 -0
  335. regscale/models/regscale_models/asset_mapping.py +190 -0
  336. regscale/models/regscale_models/case.py +42 -0
  337. regscale/models/regscale_models/catalog.py +261 -0
  338. regscale/models/regscale_models/cci.py +46 -0
  339. regscale/models/regscale_models/change.py +167 -0
  340. regscale/models/regscale_models/checklist.py +372 -0
  341. regscale/models/regscale_models/comment.py +49 -0
  342. regscale/models/regscale_models/compliance_settings.py +112 -0
  343. regscale/models/regscale_models/component.py +412 -0
  344. regscale/models/regscale_models/component_mapping.py +65 -0
  345. regscale/models/regscale_models/control.py +38 -0
  346. regscale/models/regscale_models/control_implementation.py +1128 -0
  347. regscale/models/regscale_models/control_objective.py +261 -0
  348. regscale/models/regscale_models/control_parameter.py +100 -0
  349. regscale/models/regscale_models/control_test.py +34 -0
  350. regscale/models/regscale_models/control_test_plan.py +75 -0
  351. regscale/models/regscale_models/control_test_result.py +52 -0
  352. regscale/models/regscale_models/custom_field.py +245 -0
  353. regscale/models/regscale_models/data.py +109 -0
  354. regscale/models/regscale_models/data_center.py +40 -0
  355. regscale/models/regscale_models/deviation.py +203 -0
  356. regscale/models/regscale_models/email.py +97 -0
  357. regscale/models/regscale_models/evidence.py +47 -0
  358. regscale/models/regscale_models/evidence_mapping.py +40 -0
  359. regscale/models/regscale_models/facility.py +59 -0
  360. regscale/models/regscale_models/file.py +382 -0
  361. regscale/models/regscale_models/filetag.py +37 -0
  362. regscale/models/regscale_models/form_field_value.py +94 -0
  363. regscale/models/regscale_models/group.py +169 -0
  364. regscale/models/regscale_models/implementation_objective.py +335 -0
  365. regscale/models/regscale_models/implementation_option.py +275 -0
  366. regscale/models/regscale_models/implementation_role.py +33 -0
  367. regscale/models/regscale_models/incident.py +177 -0
  368. regscale/models/regscale_models/interconnection.py +43 -0
  369. regscale/models/regscale_models/issue.py +1176 -0
  370. regscale/models/regscale_models/leveraged_authorization.py +125 -0
  371. regscale/models/regscale_models/line_of_inquiry.py +52 -0
  372. regscale/models/regscale_models/link.py +205 -0
  373. regscale/models/regscale_models/meta_data.py +64 -0
  374. regscale/models/regscale_models/mixins/__init__.py +0 -0
  375. regscale/models/regscale_models/mixins/parent_cache.py +124 -0
  376. regscale/models/regscale_models/module.py +224 -0
  377. regscale/models/regscale_models/modules.py +191 -0
  378. regscale/models/regscale_models/objective.py +14 -0
  379. regscale/models/regscale_models/parameter.py +87 -0
  380. regscale/models/regscale_models/ports_protocol.py +81 -0
  381. regscale/models/regscale_models/privacy.py +89 -0
  382. regscale/models/regscale_models/profile.py +50 -0
  383. regscale/models/regscale_models/profile_link.py +68 -0
  384. regscale/models/regscale_models/profile_mapping.py +124 -0
  385. regscale/models/regscale_models/project.py +63 -0
  386. regscale/models/regscale_models/property.py +278 -0
  387. regscale/models/regscale_models/question.py +85 -0
  388. regscale/models/regscale_models/questionnaire.py +87 -0
  389. regscale/models/regscale_models/questionnaire_instance.py +177 -0
  390. regscale/models/regscale_models/rbac.py +132 -0
  391. regscale/models/regscale_models/reference.py +86 -0
  392. regscale/models/regscale_models/regscale_model.py +1643 -0
  393. regscale/models/regscale_models/requirement.py +29 -0
  394. regscale/models/regscale_models/risk.py +274 -0
  395. regscale/models/regscale_models/sbom.py +54 -0
  396. regscale/models/regscale_models/scan_history.py +436 -0
  397. regscale/models/regscale_models/search.py +53 -0
  398. regscale/models/regscale_models/security_control.py +132 -0
  399. regscale/models/regscale_models/security_plan.py +204 -0
  400. regscale/models/regscale_models/software_inventory.py +159 -0
  401. regscale/models/regscale_models/stake_holder.py +64 -0
  402. regscale/models/regscale_models/stig.py +647 -0
  403. regscale/models/regscale_models/supply_chain.py +152 -0
  404. regscale/models/regscale_models/system_role.py +188 -0
  405. regscale/models/regscale_models/system_role_external_assignment.py +40 -0
  406. regscale/models/regscale_models/tag.py +37 -0
  407. regscale/models/regscale_models/tag_mapping.py +19 -0
  408. regscale/models/regscale_models/task.py +133 -0
  409. regscale/models/regscale_models/threat.py +196 -0
  410. regscale/models/regscale_models/user.py +175 -0
  411. regscale/models/regscale_models/user_group.py +55 -0
  412. regscale/models/regscale_models/vulnerability.py +242 -0
  413. regscale/models/regscale_models/vulnerability_mapping.py +162 -0
  414. regscale/models/regscale_models/workflow.py +55 -0
  415. regscale/models/regscale_models/workflow_action.py +34 -0
  416. regscale/models/regscale_models/workflow_instance.py +269 -0
  417. regscale/models/regscale_models/workflow_instance_step.py +114 -0
  418. regscale/models/regscale_models/workflow_template.py +58 -0
  419. regscale/models/regscale_models/workflow_template_step.py +45 -0
  420. regscale/regscale.py +815 -0
  421. regscale/utils/__init__.py +7 -0
  422. regscale/utils/b64conversion.py +14 -0
  423. regscale/utils/click_utils.py +118 -0
  424. regscale/utils/decorators.py +48 -0
  425. regscale/utils/dict_utils.py +59 -0
  426. regscale/utils/files.py +79 -0
  427. regscale/utils/fxns.py +30 -0
  428. regscale/utils/graphql_client.py +113 -0
  429. regscale/utils/lists.py +16 -0
  430. regscale/utils/numbers.py +12 -0
  431. regscale/utils/shell.py +148 -0
  432. regscale/utils/string.py +121 -0
  433. regscale/utils/synqly_utils.py +165 -0
  434. regscale/utils/threading/__init__.py +8 -0
  435. regscale/utils/threading/threadhandler.py +131 -0
  436. regscale/utils/threading/threadsafe_counter.py +47 -0
  437. regscale/utils/threading/threadsafe_dict.py +242 -0
  438. regscale/utils/threading/threadsafe_list.py +83 -0
  439. regscale/utils/version.py +104 -0
  440. regscale/validation/__init__.py +0 -0
  441. regscale/validation/address.py +37 -0
  442. regscale/validation/record.py +48 -0
  443. regscale/visualization/__init__.py +5 -0
  444. regscale/visualization/click.py +34 -0
  445. regscale_cli-6.16.0.0.dist-info/LICENSE +21 -0
  446. regscale_cli-6.16.0.0.dist-info/METADATA +659 -0
  447. regscale_cli-6.16.0.0.dist-info/RECORD +481 -0
  448. regscale_cli-6.16.0.0.dist-info/WHEEL +5 -0
  449. regscale_cli-6.16.0.0.dist-info/entry_points.txt +6 -0
  450. regscale_cli-6.16.0.0.dist-info/top_level.txt +2 -0
  451. tests/fixtures/__init__.py +2 -0
  452. tests/fixtures/api.py +87 -0
  453. tests/fixtures/models.py +91 -0
  454. tests/fixtures/test_fixture.py +144 -0
  455. tests/mocks/__init__.py +0 -0
  456. tests/mocks/objects.py +3 -0
  457. tests/mocks/response.py +32 -0
  458. tests/mocks/xml.py +13 -0
  459. tests/regscale/__init__.py +0 -0
  460. tests/regscale/core/__init__.py +0 -0
  461. tests/regscale/core/test_api.py +232 -0
  462. tests/regscale/core/test_app.py +406 -0
  463. tests/regscale/core/test_login.py +37 -0
  464. tests/regscale/core/test_logz.py +66 -0
  465. tests/regscale/core/test_sbom_generator.py +87 -0
  466. tests/regscale/core/test_validation_utils.py +163 -0
  467. tests/regscale/core/test_version.py +78 -0
  468. tests/regscale/models/__init__.py +0 -0
  469. tests/regscale/models/test_asset.py +71 -0
  470. tests/regscale/models/test_config.py +26 -0
  471. tests/regscale/models/test_control_implementation.py +27 -0
  472. tests/regscale/models/test_import.py +97 -0
  473. tests/regscale/models/test_issue.py +36 -0
  474. tests/regscale/models/test_mapping.py +52 -0
  475. tests/regscale/models/test_platform.py +31 -0
  476. tests/regscale/models/test_regscale_model.py +346 -0
  477. tests/regscale/models/test_report.py +32 -0
  478. tests/regscale/models/test_tenable_integrations.py +118 -0
  479. tests/regscale/models/test_user_model.py +121 -0
  480. tests/regscale/test_about.py +19 -0
  481. tests/regscale/test_authorization.py +65 -0
@@ -0,0 +1,1584 @@
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ """Tenable integration for RegScale CLI"""
4
+
5
+ import queue
6
+ from typing import TYPE_CHECKING, Any
7
+
8
+ from regscale.integrations.integration_override import IntegrationOverride
9
+
10
+ # Delay import of Tenable libraries
11
+ if TYPE_CHECKING:
12
+ from tenable.io import TenableIO # type: ignore
13
+ from tenable.sc import TenableSC # type: ignore
14
+ import pandas as pd # Type Checking
15
+
16
+ import collections
17
+ import json
18
+ import os
19
+ import re
20
+ import tempfile
21
+ import time
22
+ import uuid
23
+ from concurrent.futures import ThreadPoolExecutor
24
+ from datetime import datetime, timedelta
25
+ from itertools import groupby
26
+ from pathlib import Path
27
+ from threading import current_thread, get_ident, get_native_id
28
+ from typing import Dict, List, Optional, Set, Tuple, Union
29
+ from urllib.parse import urljoin
30
+
31
+ import click
32
+ import requests
33
+ from requests.exceptions import RequestException
34
+ from rich.console import Console
35
+ from rich.pretty import pprint
36
+ from rich.progress import track
37
+ from tenable.sc.analysis import AnalysisResultsIterator
38
+
39
+ from regscale import __version__
40
+ from regscale.core.app.api import Api
41
+ from regscale.core.app.application import Application
42
+ from regscale.core.app.logz import create_logger
43
+ from regscale.core.app.utils.app_utils import (
44
+ check_file_path,
45
+ check_license,
46
+ create_progress_object,
47
+ epoch_to_datetime,
48
+ format_dict_to_html,
49
+ get_current_datetime,
50
+ regscale_string_to_epoch,
51
+ save_data_to,
52
+ )
53
+ from regscale.core.app.utils.pickle_file_handler import PickleFileHandler
54
+ from regscale.integrations.commercial.nessus.nessus_utils import get_cpe_file
55
+ from regscale.models.app_models.click import file_types, hidden_file_path, regscale_ssp_id, save_output_to
56
+ from regscale.models.integration_models.tenable_models.integration import SCIntegration
57
+ from regscale.models.integration_models.tenable_models.models import AssetCheck, TenableAsset, TenableIOAsset
58
+ from regscale.models.regscale_models import ControlImplementation
59
+ from regscale.models.regscale_models.asset import Asset
60
+ from regscale.models.regscale_models.issue import Issue
61
+ from regscale.models.regscale_models.scan_history import ScanHistory
62
+ from regscale.utils.threading import ThreadSafeCounter
63
+ from regscale.validation.address import validate_mac_address
64
+
65
+ console = Console()
66
+
67
+ logger = create_logger("rich")
68
+ REGSCALE_INC = "RegScale, Inc."
69
+ REGSCALE_CLI = "RegScale CLI"
70
+
71
+ FULLY_IMPLEMENTED = "Fully Implemented"
72
+ NOT_IMPLEMENTED = "Not Implemented"
73
+ IN_REMEDIATION = "In Remediation"
74
+
75
+ DONE_MSG = "Done!"
76
+
77
+
78
+ #####################################################################################################
79
+ #
80
+ # Tenable.sc Documentation: https://docs.tenable.com/tenablesc/api/index.htm
81
+ # pyTenable GitHub repo: https://github.com/tenable/pyTenable
82
+ # Python tenable.sc documentation: https://pytenable.readthedocs.io/en/stable/api/sc/index.html
83
+ #
84
+ #####################################################################################################
85
+
86
+
87
+ # Create group to handle OSCAL processing
88
+ @click.group()
89
+ def tenable():
90
+ """Performs actions on the Tenable APIs."""
91
+
92
+
93
+ @tenable.group(help="[BETA] Performs actions on the Tenable.io API.")
94
+ def io():
95
+ """Performs actions on the Tenable.io API."""
96
+
97
+
98
+ @tenable.group(help="[BETA] Performs actions on the Tenable.sc API.")
99
+ def sc():
100
+ """Performs actions on the Tenable.sc API."""
101
+
102
+
103
+ @tenable.group(help="[BETA] Import Nessus scans and assets to RegScale.")
104
+ def nessus():
105
+ """Performs actions on the Tenable.sc API."""
106
+
107
+
108
+ @nessus.command(name="import_nessus")
109
+ @click.option(
110
+ "--folder_path",
111
+ prompt="Enter the folder path of the Nessus files to process",
112
+ help="RegScale will load the Nessus Scans",
113
+ type=click.Path(exists=True),
114
+ )
115
+ @click.option(
116
+ "--scan_date",
117
+ type=click.DateTime(formats=["%Y-%m-%d"]),
118
+ help="The the scan date of the file.",
119
+ required=False,
120
+ )
121
+ @regscale_ssp_id()
122
+ def import_nessus(folder_path: click.Path, regscale_ssp_id: click.INT, scan_date: click.DateTime):
123
+ """Import Nessus scans, vulnerabilities and assets to RegScale."""
124
+ from regscale.integrations.commercial.nessus.scanner import NessusIntegration
125
+
126
+ NessusIntegration.sync_assets(plan_id=regscale_ssp_id, path=folder_path)
127
+ NessusIntegration.sync_findings(
128
+ plan_id=regscale_ssp_id, path=folder_path, enable_finding_date_update=True, scan_date=scan_date
129
+ )
130
+
131
+
132
+ @nessus.command(name="update_cpe_dictionary")
133
+ def update_cpe_dictionary():
134
+ """
135
+ Manually update the CPE 2.2 dictionary from NIST.
136
+ """
137
+ get_cpe_file(download=True)
138
+
139
+
140
+ @sc.command(name="export_scans")
141
+ @save_output_to()
142
+ @file_types([".json", ".csv", ".xlsx"])
143
+ def export_scans(save_output_to: Path, file_type: str):
144
+ """Export scans from Tenable Host to a .json, .csv or .xlsx file."""
145
+ # get the scan results
146
+ results = get_usable_scan_list()
147
+
148
+ # check if file path exists
149
+ check_file_path(save_output_to)
150
+
151
+ # set the file name
152
+ file_name = f"tenable_scans_{get_current_datetime('%m%d%Y')}"
153
+
154
+ # save the data as the selected file by the user
155
+ save_data_to(
156
+ file=Path(f"{save_output_to}/{file_name}{file_type}"),
157
+ data=results,
158
+ )
159
+
160
+
161
+ def validate_tags(ctx: click.Context, param: click.Option, value: str) -> List[Tuple[str, str]]:
162
+ """
163
+ Validate the tuple elements.
164
+
165
+ :param click.Context ctx: Click context
166
+ :param click.Option param: Click option
167
+ :param str value: A string value to parse and validate
168
+ :return: Tuple of validated values
169
+ :rtype: List[Tuple[str,str]]
170
+ :raise ValueError: If the value is not in the correct format
171
+ """
172
+ if not value:
173
+ return []
174
+
175
+ tuple_list = []
176
+ for item in value.split(","):
177
+ parts = [part for part in item.strip().split(":") if part]
178
+ if len(parts) != 2:
179
+ raise ValueError(f"""Invalid format: "{item}". Expected 'key:value'""")
180
+ tuple_list.append((parts[0], parts[1]))
181
+
182
+ return tuple_list
183
+
184
+
185
+ def get_usable_scan_list() -> list:
186
+ """
187
+ Usable Scans from Tenable Host
188
+
189
+ :return: List of scans from Tenable
190
+ :rtype: list
191
+ """
192
+ results = []
193
+ try:
194
+ client = gen_client()
195
+ results = client.scans.list()["usable"]
196
+ except Exception as ex:
197
+ logger.error(ex)
198
+ return results
199
+
200
+
201
+ def get_detailed_scans(scan_list: list = None) -> list:
202
+ """
203
+ Generate list of detailed scans (Warning: this action could take 20 minutes or more to complete)
204
+
205
+ :param list scan_list: List of scans from Tenable, defaults to None
206
+ :raise SystemExit: If there is an error with the request
207
+ :return: Detailed list of Tenable scans
208
+ :rtype: list
209
+ """
210
+ client = gen_client()
211
+ detailed_scans = []
212
+ for scan in track(scan_list, description="Fetching detailed scans..."):
213
+ try:
214
+ det = client.scans.details(id=scan["id"])
215
+ detailed_scans.append(det)
216
+ except RequestException as ex: # This is the correct syntax
217
+ raise SystemExit(ex) from ex
218
+
219
+ return detailed_scans
220
+
221
+
222
+ @sc.command(name="save_queries")
223
+ @save_output_to()
224
+ @file_types([".json", ".csv", ".xlsx"])
225
+ def save_queries(save_output_to: Path, file_type: str):
226
+ """Get a list of query definitions and save them as a .json, .csv or .xlsx file."""
227
+ # get the queries from Tenable
228
+ query_list = get_queries()
229
+
230
+ # check if file path exists
231
+ check_file_path(save_output_to)
232
+
233
+ # set the file name
234
+ file_name = f"tenable_queries_{get_current_datetime('%m%d%Y')}"
235
+
236
+ # save the data as a .json file
237
+ save_data_to(
238
+ file=Path(f"{save_output_to}{os.sep}{file_name}{file_type}"),
239
+ data=query_list,
240
+ )
241
+
242
+
243
+ def get_queries() -> list:
244
+ """
245
+ List of query definitions
246
+
247
+ :return: List of queries from Tenable
248
+ :rtype: list
249
+ """
250
+ app = Application()
251
+ tsc = gen_tsc(app.config)
252
+ return tsc.queries.list()
253
+
254
+
255
+ @sc.command(name="query_vuln")
256
+ @click.option(
257
+ "--query_id",
258
+ type=click.INT,
259
+ help="Tenable query ID to retrieve via API",
260
+ prompt="Enter Tenable query ID",
261
+ required=True,
262
+ )
263
+ @regscale_ssp_id()
264
+ # Add Prompt for RegScale SSP name
265
+ def query_vuln(query_id: int, regscale_ssp_id: int):
266
+ """Query Tenable vulnerabilities and sync assets to RegScale."""
267
+ q_vuln(
268
+ query_id=query_id,
269
+ ssp_id=regscale_ssp_id,
270
+ )
271
+
272
+
273
+ @io.command(name="sync_assets")
274
+ @regscale_ssp_id()
275
+ @click.option(
276
+ "--tags",
277
+ type=click.STRING,
278
+ help='Optional tags to filter assets, wrap in double quotes, e.g. --tags "Tag1:tag1a,Tag2:tag2b"',
279
+ default=None,
280
+ required=False,
281
+ callback=validate_tags,
282
+ )
283
+ # Add Prompt for RegScale SSP name
284
+ def query_assets(regscale_ssp_id: int, tags: Optional[List[Tuple[str, str]]] = None):
285
+ """Query Tenable Assets and sync to RegScale."""
286
+ # Validate ssp
287
+ from regscale.integrations.commercial.tenablev2.scanner import TenableIntegration
288
+
289
+ TenableIntegration.sync_assets(plan_id=regscale_ssp_id, tags=tags)
290
+
291
+
292
+ @io.command(name="sync_vulns")
293
+ @regscale_ssp_id()
294
+ @click.option(
295
+ "--tags",
296
+ type=click.STRING,
297
+ help='Optional tags to filter vulns, wrap in double quotes, e.g. --tags "Tag1:tag1a,Tag2:tag2b"',
298
+ default=None,
299
+ required=False,
300
+ callback=validate_tags,
301
+ )
302
+ # Add Prompt for RegScale SSP name
303
+ def query_vulns(regscale_ssp_id: int, tags: Optional[List[Tuple[str, str]]] = None):
304
+ """
305
+ Query Tenable vulnerabilities and sync assets, vulnerabilities and issues to RegScale.
306
+ """
307
+ from regscale.integrations.commercial.tenablev2.scanner import TenableIntegration
308
+
309
+ TenableIntegration.sync_findings(plan_id=regscale_ssp_id, tags=tags)
310
+
311
+
312
+ def validate_regscale_security_plan(parent_id: int) -> bool:
313
+ """
314
+ Validate RegScale Security Plan exists
315
+
316
+ :param int parent_id: The ID number from RegScale of the System Security Plan
317
+ :return: If API call was successful
318
+ :rtype: bool
319
+ """
320
+ app = check_license()
321
+ config = app.config
322
+ headers = {
323
+ "Authorization": config["token"],
324
+ }
325
+ url = urljoin(config["domain"], f"/api/securityplans/{parent_id}")
326
+ response = requests.get(url, headers=headers)
327
+ return response.ok
328
+
329
+
330
+ @io.command(name="list_jobs")
331
+ @click.option(
332
+ "--job_type",
333
+ default="vulns",
334
+ type=click.Choice(["vulns", "assets"]),
335
+ show_default=True,
336
+ help="Tenable job type.",
337
+ required=False,
338
+ )
339
+ @click.option(
340
+ "--last",
341
+ type=click.INT,
342
+ default=100,
343
+ show_default=True,
344
+ help="Filter the last n jobs.",
345
+ required=False,
346
+ )
347
+ @click.option(
348
+ "--job_status",
349
+ type=click.Choice(["processing", "finished", "cancelled"]),
350
+ help="Filter by status.",
351
+ required=False,
352
+ )
353
+ def list_jobs(job_type: str, last: int, job_status: str):
354
+ """Retrieve a list of jobs from Tenable.io."""
355
+ app = Application()
356
+ config = app.config
357
+ client = gen_tio(config)
358
+ if job_status:
359
+ jobs = [job for job in client.exports.jobs(job_type) if job["status"] == str(job_status).upper()]
360
+ else:
361
+ jobs = client.exports.jobs(job_type)
362
+ jobs = sorted(jobs, key=lambda k: (k["created"]), reverse=False)
363
+ # filter the last N jobs
364
+ for job in jobs[len(jobs) - last :]:
365
+ console.print(
366
+ f"UUID: {job['uuid']}, STATUS: {job['status']}, CREATED: {epoch_to_datetime(job['created'], epoch_type='milliseconds')}"
367
+ )
368
+
369
+
370
+ @io.command(name="cancel_job")
371
+ @click.option(
372
+ "--uuid",
373
+ type=click.STRING,
374
+ help="Tenable job UUID.",
375
+ prompt="Enter the UUID of the job to cancel.",
376
+ required=True,
377
+ )
378
+ @click.option(
379
+ "--job_type",
380
+ default="vulns",
381
+ type=click.Choice(["vulns", "assets"]),
382
+ show_default=True,
383
+ help="Tenable job type.",
384
+ required=False,
385
+ )
386
+ def cancel_job(uuid: str, job_type: str):
387
+ """Cancel a Tenable IO job."""
388
+ app = Application()
389
+ config = app.config
390
+ client = gen_tio(config)
391
+ client.exports.cancel(job_type, export_uuid=uuid)
392
+
393
+
394
+ def process_vulnerabilities(counts: collections.Counter, reg_assets: list, ssp_id: int, tenable_vulns: list) -> list:
395
+ """
396
+ Process Tenable vulnerabilities
397
+
398
+ :param collections.Counter counts: Dictionary of counts of each vulnerability
399
+ :param list reg_assets: List of RegScale assets
400
+ :param int ssp_id: RegScale System Security Plan ID
401
+ :param list tenable_vulns: List of Tenable vulnerabilities
402
+ :return: List of assets to update
403
+ :rtype: list
404
+ """
405
+ update_assets = []
406
+ for vuln in set(tenable_vulns):
407
+ update_assets = process_vuln(counts, reg_assets, ssp_id, vuln)
408
+ return update_assets
409
+
410
+
411
+ def q_vuln(query_id: int, ssp_id: int) -> list:
412
+ """
413
+ Query Tenable vulnerabilities
414
+
415
+ :param int query_id: Tenable query ID
416
+ :param int ssp_id: RegScale System Security Plan ID
417
+ :return: List of queries from Tenable
418
+ :rtype: list
419
+ """
420
+ check_license()
421
+ # At SSP level, provide a list of vulnerabilities and the counts of each
422
+ fetch_vulns(query_id=query_id, regscale_ssp_id=ssp_id)
423
+
424
+
425
+ def process_vuln(counts: collections.Counter, reg_assets: list, ssp_id: int, vuln: TenableAsset) -> list:
426
+ """
427
+ Process Tenable vulnerability data
428
+
429
+ :param collections.Counter counts: Dictionary of counts of each vulnerability
430
+ :param list reg_assets: List of RegScale assets
431
+ :param int ssp_id: RegScale System Security Plan ID
432
+ :param TenableAsset vuln: Tenable vulnerability object
433
+ :return: List of assets to update
434
+ :rtype: list
435
+ """
436
+ update_assets = []
437
+ vuln.count = dict(counts)[vuln.pluginName]
438
+ lookup_assets = lookup_asset(reg_assets, vuln.macAddress, vuln.dnsName)
439
+ # Update parent id to SSP on insert
440
+ if len(lookup_assets) > 0:
441
+ for asset in set(lookup_assets):
442
+ # Do update
443
+ # asset = reg_asset[0]
444
+ asset.parentId = ssp_id
445
+ asset.parentModule = "securityplans"
446
+ asset.macAddress = vuln.macAddress.upper()
447
+ asset.osVersion = vuln.operatingSystem
448
+ asset.purchaseDate = "01-01-1970"
449
+ asset.endOfLifeDate = "01-01-1970"
450
+ if asset.ipAddress is None:
451
+ asset.ipAddress = vuln.ip
452
+ asset.operatingSystem = determine_os(asset.operatingSystem)
453
+ try:
454
+ assert asset.id
455
+ # avoid duplication
456
+ if asset not in update_assets:
457
+ update_assets.append(asset)
458
+ except AssertionError as aex:
459
+ logger.error("Asset does not have an id, unable to update!\n%s", aex)
460
+ return update_assets
461
+
462
+
463
+ def determine_os(os_string: str) -> str:
464
+ """
465
+ Determine RegScale friendly OS name
466
+
467
+ :param str os_string: String of the asset's OS
468
+ :return: RegScale acceptable OS
469
+ :rtype: str
470
+ """
471
+ linux_words = ["linux", "ubuntu", "hat", "centos", "rocky", "alma", "alpine"]
472
+ if re.compile("|".join(linux_words), re.IGNORECASE).search(os_string):
473
+ return "Linux"
474
+ elif (os_string.lower()).startswith("windows"):
475
+ return "Windows Server" if "server" in os_string else "Windows Desktop"
476
+ else:
477
+ return "Other"
478
+
479
+
480
+ def lookup_asset(asset_list: list, mac_address: str, dns_name: str = None) -> list:
481
+ """
482
+ Lookup asset in Tenable and return the data from Tenable
483
+
484
+ :param list asset_list: List of assets to lookup in Tenable
485
+ :param str mac_address: Mac address of asset
486
+ :param str dns_name: DNS Name of the asset, defaults to None
487
+ :return: List of assets that fit the provided filters
488
+ :rtype: list
489
+ """
490
+ results = []
491
+ if validate_mac_address(mac_address):
492
+ if dns_name:
493
+ results = [
494
+ Asset(**asset)
495
+ for asset in asset_list
496
+ if "macAddress" in asset
497
+ and asset["macAddress"] == mac_address
498
+ and asset["name"] == dns_name
499
+ and "macAddress" in asset
500
+ and "name" in asset
501
+ ]
502
+ else:
503
+ results = [asset for asset in asset_list if asset["macAddress"] == mac_address]
504
+ # Return unique list
505
+ return list(set(results))
506
+
507
+
508
+ def create_issue_from_vuln(app: Application, row: "pd.Series", default_due_delta: int) -> "Issue":
509
+ """
510
+ Creates an Issue object from a Tenable vulnerability
511
+
512
+ :param Application app: Application object
513
+ :param pd.Series row: Row of data from Tenable
514
+ :param int default_due_delta: Default due delta
515
+ :return: Issue object
516
+ :rtype: Issue
517
+ """
518
+
519
+ default_status = app.config["issues"]["tenable"]["status"]
520
+ fmt = "%Y-%m-%d %H:%M:%S"
521
+ plugin_id = row["pluginID"]
522
+ port = row["port"]
523
+ protocol = row["protocol"]
524
+ due_date = datetime.strptime(row["last_scan"], fmt) + timedelta(days=default_due_delta)
525
+ if due_date < datetime.now():
526
+ due_date = datetime.now() + timedelta(days=default_due_delta)
527
+ if "synopsis" in row:
528
+ title = row["synopsis"]
529
+ return Issue(
530
+ title=title or row["pluginName"],
531
+ description=row["description"] or row["pluginName"] + f"<br>Port: {port}<br>Protocol: {protocol}",
532
+ issueOwnerId=app.config["userId"],
533
+ status=default_status,
534
+ severityLevel=Issue.assign_severity(row["severity"]),
535
+ dueDate=due_date.strftime(fmt),
536
+ identification="Vulnerability Assessment",
537
+ parentId=row["regscale_ssp_id"],
538
+ parentModule="securityplans",
539
+ pluginId=plugin_id,
540
+ vendorActions=row["solution"],
541
+ assetIdentifier=f'DNS: {row["dnsName"]} - IP: {row["ip"]}',
542
+ )
543
+
544
+
545
+ def create_issue_from_row(app: Application, row: "pd.Series", default_due_delta: int) -> "Issue":
546
+ """
547
+ Creates an Issue object from a Tenable vulnerability
548
+
549
+ :param Application app: Application object
550
+ :param pd.Series row: Row of data from Tenable
551
+ :param int default_due_delta: Default due delta
552
+ :return: Issue object
553
+ :rtype: Issue
554
+ """
555
+ if row["severity"] != "Info":
556
+ issue = create_issue_from_vuln(app, row, default_due_delta)
557
+ if isinstance(issue, Issue):
558
+ return issue
559
+ return None
560
+
561
+
562
+ def prepare_issues_for_sync(
563
+ app: Application, df: "pd.DataFrame", regscale_ssp_id: int
564
+ ) -> Tuple[List["Issue"], List["Issue"]]:
565
+ """
566
+ Prepares Tenable vulnerabilities for synchronization as RegScale issues
567
+
568
+ :param Application app: Application object
569
+ :param pd.DataFrame df: Dataframe of Tenable data
570
+ :param int regscale_ssp_id: RegScale System Security Plan ID
571
+ :return: List of issues to insert, list of issues to update
572
+ :rtype: Tuple[List[Issue], List[Issue]]
573
+ """
574
+
575
+ default_due_delta = app.config["issues"]["tenable"]["moderate"]
576
+ existing_issues = Issue.get_all_by_parent(parent_id=regscale_ssp_id, parent_module="securityplans")
577
+ sc_issues = []
578
+ new_issues = set()
579
+ update_issues = set()
580
+ for index, row in df.iterrows():
581
+ issue = create_issue_from_row(app, row, default_due_delta)
582
+ if isinstance(issue, Issue):
583
+ sc_issues.append(issue)
584
+ # Generate list of completely new issues, and merge with existing issues if they have the same title
585
+ # group issues by title
586
+ grouped_issues = {k: list(g) for k, g in groupby(sc_issues, key=lambda x: x.title)}
587
+ for title in grouped_issues:
588
+ reg_key = 0
589
+ regs = [iss for iss in existing_issues if iss.title == title and iss.id]
590
+ if regs:
591
+ reg_key = regs[0].id
592
+ issues = set(grouped_issues[title])
593
+ for issue in issues:
594
+ asset_ident = combine_strings({iss.assetIdentifier for iss in issues})
595
+ if reg_key and issue.title not in {iss.title for iss in update_issues}:
596
+ issue.id = reg_key
597
+ issue.assetIdentifier = asset_ident
598
+ update_issues.add(issue)
599
+ elif not reg_key:
600
+ issue.assetIdentifier = asset_ident
601
+ new_issues.add(issue)
602
+
603
+ return list(new_issues), list(update_issues)
604
+
605
+
606
+ def combine_strings(set_of_strings: Set[str]) -> str:
607
+ """
608
+ Combines a set of strings into a single string
609
+
610
+ :param Set[str] set_of_strings: Set of strings
611
+ :rtype: str
612
+ :return: Combined string
613
+ """
614
+ return "<br>".join(set_of_strings)
615
+
616
+
617
+ def sync_issues_to_regscale(new_issues: List["Issue"], update_issues: List["Issue"]) -> None:
618
+ """
619
+ Synchronizes issues to RegScale
620
+
621
+ :param List[Issue] new_issues: New issues
622
+ :param List[Issue] update_issues: Updated issues
623
+ :rtype: None
624
+ """
625
+ logger = create_logger()
626
+
627
+ if new_issues:
628
+ logger.info(f"Creating {len(new_issues)} new issue(s) in RegScale...")
629
+ Issue.batch_create(new_issues)
630
+ logger.info("Finished creating issue(s) in RegScale.")
631
+ else:
632
+ logger.info("No new issues to create.")
633
+
634
+ if update_issues:
635
+ logger.info(f"Updating {len(update_issues)} existing issue(s) in RegScale...")
636
+ Issue.batch_update(update_issues)
637
+ logger.info("Finished updating issue(s) in RegScale.")
638
+ else:
639
+ logger.info("No issues to update.")
640
+
641
+
642
+ def create_regscale_issue_from_vuln(regscale_ssp_id: int, df: Optional["pd.DataFrame"] = None) -> None:
643
+ """
644
+ Sync Tenable Vulnerabilities to RegScale issues
645
+
646
+ :param int regscale_ssp_id: RegScale System Security Plan ID
647
+ :param Optional["pd.DataFrame"] df: Pandas dataframe of Tenable data
648
+ :rtype: None
649
+ """
650
+ import pandas as pd # Optimize import performance
651
+
652
+ if df is None:
653
+ df = pd.DataFrame()
654
+ app = Application()
655
+ new_issues, update_issues = prepare_issues_for_sync(app, df, regscale_ssp_id)
656
+ sync_issues_to_regscale(new_issues, update_issues)
657
+
658
+
659
+ def fetch_assets(ssp_id: int) -> list[TenableIOAsset]:
660
+ """
661
+ Fetch assets from Tenable IO and sync to RegScale
662
+
663
+ :param int ssp_id: RegScale System Security Plan ID
664
+ :return: List of Tenable assets
665
+ :rtype: list[TenableIOAsset]
666
+ """
667
+ tenable_last_updated: int = 0
668
+ app = Application()
669
+ config = app.config
670
+ client = gen_tio(config=config)
671
+ assets: List[TenableIOAsset] = []
672
+ logger.info("Fetching existing assets from RegScale...")
673
+
674
+ existing_assets: List[Asset] = Asset.get_all_by_parent(parent_id=ssp_id, parent_module="securityplans")
675
+
676
+ logger.info("Found %i existing asset(s) in RegScale.", len(existing_assets))
677
+
678
+ filtered_assets = [asset for asset in existing_assets if asset.tenableId and asset.dateLastUpdated]
679
+ # Get last epoch updated from RegScale, limit to Tenable assets
680
+ if filtered_assets:
681
+ tenable_last_updated = max([regscale_string_to_epoch(asset.dateLastUpdated) for asset in filtered_assets])
682
+ export = client.exports.assets(updated_at=tenable_last_updated)
683
+ logger.info("Saving chunked asset files from Tenable IO for processing...")
684
+ temp_loc = Path(tempfile.gettempdir()) / "tenable_io" / str(uuid.uuid4()) # random folder name
685
+ # show process status
686
+ box_len = 0
687
+ status = client.exports.status(export_type=export.type, export_uuid=export.uuid)
688
+ with create_progress_object(indeterminate=True) as job_progress:
689
+ job_progress.add_task("Fetching Chunked Tenable IO data...", start=False, total=None)
690
+ while status["status"] == "PROCESSING":
691
+ box_len = len(status["chunks_available"])
692
+ time.sleep(0.5)
693
+ status = client.exports.status(export_type=export.type, export_uuid=export.uuid)
694
+ # Process chunks of data
695
+ with create_progress_object(indeterminate=True) as saving_progress:
696
+ saving_task = saving_progress.add_task(
697
+ "Saving Tenable IO data to disk...",
698
+ total=box_len,
699
+ )
700
+ export.run_threaded(
701
+ func=write_io_chunk,
702
+ kwargs={"data_dir": temp_loc},
703
+ num_threads=3,
704
+ )
705
+ saving_progress.update(saving_task, advance=1)
706
+ process_to_regscale(data_dir=temp_loc, ssp_id=ssp_id, existing_assets=existing_assets)
707
+ return assets
708
+
709
+
710
+ def fetch_vulns(query_id: int = 0, regscale_ssp_id: int = 0):
711
+ """
712
+ Fetch vulnerabilities from Tenable by query ID
713
+
714
+ :param int query_id: Tenable query ID, defaults to 0
715
+ :param int regscale_ssp_id: RegScale System Security Plan ID, defaults to 0
716
+ """
717
+
718
+ client = gen_client()
719
+ if query_id and client._env_base == "TSC":
720
+ vulns = client.analysis.vulns(query_id=query_id)
721
+ sc = SCIntegration(plan_id=regscale_ssp_id)
722
+ # Create pickle file to cache data
723
+ # make sure folder exists
724
+ with tempfile.TemporaryDirectory() as temp_dir:
725
+ logger.info("Saving Tenable SC data to disk...%s", temp_dir)
726
+ consume_iterator_to_file(iterator=vulns, dir_path=Path(temp_dir), scanner=sc)
727
+ iterables = tenable_dir_to_tuple_generator(Path(temp_dir))
728
+ try:
729
+ sc.sync_assets(
730
+ plan_id=regscale_ssp_id,
731
+ integration_assets=(asset for sublist in iterables[0] for asset in sublist),
732
+ )
733
+ sc.sync_findings(
734
+ plan_id=regscale_ssp_id,
735
+ integration_findings=(finding for sublist in iterables[1] for finding in sublist),
736
+ )
737
+ except IndexError as ex:
738
+ logger.error("Error processing Tenable SC data: %s", ex)
739
+
740
+
741
+ def tenable_dir_to_tuple_generator(dir_path: Path):
742
+ """
743
+ Generate a tuple of chained generators for Tenable directories.
744
+ """
745
+ from itertools import chain
746
+
747
+ assets_gen = chain.from_iterable(
748
+ (dat["assets"] for dat in PickleFileHandler(file).read()) for file in dir_path.iterdir()
749
+ )
750
+ findings_gen = chain.from_iterable(
751
+ (dat["findings"] for dat in PickleFileHandler(file).read()) for file in dir_path.iterdir()
752
+ )
753
+
754
+ return assets_gen, findings_gen
755
+
756
+
757
+ def consume_iterator_to_file(iterator: AnalysisResultsIterator, dir_path: Path, scanner: SCIntegration) -> int:
758
+ """
759
+ Consume an iterator and write the results to a file
760
+
761
+ :param AnalysisResultsIterator iterator: Tenable SC iterator
762
+ :param Path dir_path: The directory to save the pickled files
763
+ :param SCIntegration scanner: Tenable SC Integration object
764
+ :rtype: int
765
+ :return: The total count of items processed
766
+ """
767
+ app = Application()
768
+ logger.info("Consuming Tenable SC iterator...")
769
+ override = IntegrationOverride(app)
770
+ total_count = ThreadSafeCounter()
771
+ page_number = ThreadSafeCounter()
772
+ rec_count = ThreadSafeCounter()
773
+ process_list = queue.Queue()
774
+ with ThreadPoolExecutor(max_workers=5) as executor:
775
+ for dat in iterator:
776
+ total_count.increment()
777
+ process_list.put(dat)
778
+ rec_count.increment()
779
+ if rec_count.value == len(iterator.page):
780
+ page_number.increment()
781
+ executor.submit(
782
+ process_sc_chunk,
783
+ app=app,
784
+ vulns=pop_queue(queue=process_list, queue_len=len(iterator.page)),
785
+ page=page_number.value,
786
+ dir_path=dir_path,
787
+ sc=scanner,
788
+ override=override,
789
+ )
790
+ rec_count.set(0)
791
+ if total_count.value == 0:
792
+ logger.warning("No Tenable SC data found.")
793
+ return total_count.value
794
+
795
+
796
+ def pop_queue(queue: queue.Queue, queue_len: int) -> list:
797
+ """
798
+ Pop items from a queue
799
+
800
+ :param queue.Queue queue: Queue object
801
+ :param int queue_len: Length of the queue
802
+ :return: List of items from the queue
803
+ :rtype: list
804
+ """
805
+ retrieved_items = []
806
+
807
+ # Use a for loop to get 1000 items
808
+ for _ in range(queue_len):
809
+ # Check if the queue is not empty
810
+ if not queue.empty():
811
+ # Get an item from the queue and append it to the list
812
+ retrieved_items.append(queue.get())
813
+ else:
814
+ # Break the loop if the queue is empty
815
+ break
816
+ return retrieved_items
817
+
818
+
819
+ def process_sc_chunk(**kwargs) -> None:
820
+ """
821
+ Process Tenable SC chunk
822
+
823
+ :param kwargs: Keyword arguments
824
+ :rtype: None
825
+ """
826
+ # iterator.page, iterator.page_count, file_path, query_id, ssp_id
827
+ integration_mapping = kwargs.get("override")
828
+
829
+ vulns = kwargs.get("vulns")
830
+ dir_path = kwargs.get("dir_path")
831
+ generated_file_name = f"tenable_scan_page_{kwargs.get('page')}.pkl"
832
+ pickled_file_handler = PickleFileHandler(str(dir_path / generated_file_name))
833
+ tenable_sc: SCIntegration = kwargs.get("sc")
834
+ thread = current_thread()
835
+ if not len(vulns):
836
+ return
837
+ # I can't add a to-do thanks to sonarlint, but we need to add CVE lookup from plugin id
838
+ # append file to path
839
+ # Process to RegScale
840
+ tenable_vulns = [TenableAsset(**vuln) for vuln in vulns]
841
+ # Empty "DNS" should just be IP
842
+ for vuln in tenable_vulns:
843
+ if not vuln.dnsName:
844
+ vuln.dnsName = vuln.ip
845
+ findings = []
846
+ assets = []
847
+ for vuln in tenable_vulns:
848
+ findings += tenable_sc.parse_findings(vuln=vuln, integration_mapping=integration_mapping)
849
+ if vuln.dnsName not in {asset.name for asset in assets}: # avoid duplicates
850
+ assets.append(tenable_sc.to_integration_asset(vuln, **kwargs))
851
+ pickled_file_handler.write({"assets": assets, "findings": findings})
852
+
853
+ logger.info(
854
+ "Submitting %i findings and %i assets to the CLI Job Queue from Tenable SC Page %i...",
855
+ len(findings),
856
+ len(assets),
857
+ kwargs.get("page"),
858
+ )
859
+ logger.debug(f"Completed thread: name={thread.name}, idnet={get_ident()}, id={get_native_id()}")
860
+
861
+
862
+ def get_last_pull_epoch(regscale_ssp_id: int) -> int:
863
+ """
864
+ Gather last pull epoch from RegScale Security Plan
865
+
866
+ :param int regscale_ssp_id: RegScale System Security Plan ID
867
+ :return: Last pull epoch
868
+ :rtype: int
869
+
870
+ """
871
+ fmt: str = "%Y-%m-%d"
872
+ two_months_ago: datetime = datetime.now() - timedelta(weeks=8)
873
+ two_weeks_ago: datetime = datetime.now() - timedelta(weeks=2)
874
+ last_pull: int = round(two_weeks_ago.timestamp()) # default the last pull date to two weeks
875
+ # Limit the query with a filter_date to avoid taxing the database in the case of a large number of scans
876
+ if res := ScanHistory.get_by_parent_recursive(
877
+ parent_id=regscale_ssp_id, parent_module="securityplans", filter_date=two_months_ago.strftime(fmt)
878
+ ):
879
+ # order by ScanDate desc
880
+ fmt = "%Y-%m-%dT%H:%M:%S"
881
+ res = sorted(res, key=lambda x: datetime.strptime(x.scanDate, fmt), reverse=True)
882
+ # Convert to timestampe
883
+ last_pull = round((datetime.strptime(res[0].scanDate, fmt)).timestamp())
884
+ return last_pull
885
+
886
+
887
+ @sc.command(name="list_tags")
888
+ def sc_tags():
889
+ """List tags from Tenable"""
890
+ list_tags()
891
+
892
+
893
+ def list_tags() -> None:
894
+ """
895
+ Query a list of tags on the server and print to console
896
+
897
+ :rtype: None
898
+ """
899
+ tag_list = get_tags()
900
+ pprint(tag_list)
901
+
902
+
903
+ def get_tags() -> list:
904
+ """
905
+ List of Tenable query definitions
906
+
907
+ :return: List of unique tags for Tenable queries
908
+ :rtype: list
909
+ """
910
+ client = gen_client()
911
+ logger.debug(client._env_base)
912
+ if client._env_base == "TSC":
913
+ return client.queries.tags()
914
+ return list(client.tags.list())
915
+
916
+
917
+ def gen_client() -> Union["TenableIO", "TenableSC"]:
918
+ """
919
+ Return the appropriate Tenable client based on the URL
920
+
921
+ :return: Client type
922
+ :rtype: Union["TenableIO", "TenableSC"]
923
+ """
924
+ app = Application()
925
+ config = app.config
926
+ if "cloud.tenable.com" in config["tenableUrl"]:
927
+ return gen_tio(config)
928
+ return gen_tsc(config)
929
+
930
+
931
+ def gen_tsc(config: dict) -> "TenableSC":
932
+ """
933
+ Generate Tenable Object
934
+
935
+ :param dict config: Configuration dictionary
936
+ :return: Tenable client
937
+ :rtype: "TenableSC"
938
+ """
939
+ from tenable.sc import TenableSC
940
+
941
+ if not config:
942
+ app = Application()
943
+ config = app.config
944
+ return TenableSC(
945
+ url=config["tenableUrl"],
946
+ access_key=config["tenableAccessKey"],
947
+ secret_key=config["tenableSecretKey"],
948
+ vendor=REGSCALE_INC,
949
+ product=REGSCALE_CLI,
950
+ build=__version__,
951
+ )
952
+
953
+
954
+ def gen_tio(config: dict) -> "TenableIO":
955
+ """
956
+ Generate Tenable Object
957
+
958
+ :param dict config: Configuration dictionary
959
+ :return: Tenable client
960
+ :rtype: "TenableIO"
961
+ """
962
+
963
+ from tenable.io import TenableIO
964
+
965
+ return TenableIO(
966
+ url=config["tenableUrl"],
967
+ access_key=config["tenableAccessKey"],
968
+ secret_key=config["tenableSecretKey"],
969
+ vendor=REGSCALE_INC,
970
+ product=REGSCALE_CLI,
971
+ build=__version__,
972
+ )
973
+
974
+
975
+ def get_controls(catalog_id: int) -> List[Dict]:
976
+ """
977
+ Gets all the controls
978
+
979
+ :param int catalog_id: catalog id
980
+ :return: list of controls
981
+ :rtype: List[Dict]
982
+ """
983
+ app = Application()
984
+ api = Api()
985
+ url = urljoin(app.config.get("domain"), f"/api/SecurityControls/getList/{catalog_id}")
986
+ response = api.get(url)
987
+ if response.ok:
988
+ return response.json()
989
+ else:
990
+ response.raise_for_status()
991
+ return []
992
+
993
+
994
+ def create_control_implementations(
995
+ controls: list,
996
+ parent_id: int,
997
+ parent_module: str,
998
+ existing_implementation_dict: Dict,
999
+ passing_controls: Dict,
1000
+ failing_controls: Dict,
1001
+ ) -> List[Dict]:
1002
+ """
1003
+ Creates a list of control implementations
1004
+
1005
+ :param list controls: list of controls
1006
+ :param int parent_id: parent control id
1007
+ :param str parent_module: parent module
1008
+ :param Dict existing_implementation_dict: Dictionary of existing control implementations
1009
+ :param Dict passing_controls: Dictionary of passing controls
1010
+ :param Dict failing_controls: Dictionary of failing controls
1011
+ :return: list of control implementations
1012
+ :rtype: List[Dict]
1013
+ """
1014
+ app = Application()
1015
+ api = Api()
1016
+ user_id = app.config.get("userId")
1017
+ domain = app.config.get("domain")
1018
+ control_implementations = []
1019
+ to_create = []
1020
+ to_update = []
1021
+ for control in controls:
1022
+ lower_case_control_id = control["controlId"].lower()
1023
+ status = check_implementation(
1024
+ passing_controls=passing_controls,
1025
+ failing_controls=failing_controls,
1026
+ control_id=lower_case_control_id,
1027
+ )
1028
+ if control["controlId"] not in existing_implementation_dict.keys():
1029
+ cim = ControlImplementation(
1030
+ controlOwnerId=user_id,
1031
+ dateLastAssessed=get_current_datetime(),
1032
+ status=status,
1033
+ controlID=control["id"],
1034
+ parentId=parent_id,
1035
+ parentModule=parent_module,
1036
+ createdById=user_id,
1037
+ dateCreated=get_current_datetime(),
1038
+ lastUpdatedById=user_id,
1039
+ dateLastUpdated=get_current_datetime(),
1040
+ ).dict()
1041
+ cim["controlSource"] = "Baseline"
1042
+ to_create.append(cim)
1043
+
1044
+ else:
1045
+ # update existing control implementation data
1046
+ existing_imp = existing_implementation_dict.get(control["controlId"])
1047
+ existing_imp["status"] = status
1048
+ existing_imp["dateLastAssessed"] = get_current_datetime()
1049
+ existing_imp["lastUpdatedById"] = user_id
1050
+ existing_imp["dateLastUpdated"] = get_current_datetime()
1051
+ del existing_imp["createdBy"]
1052
+ del existing_imp["systemRole"]
1053
+ del existing_imp["controlOwner"]
1054
+ del existing_imp["lastUpdatedBy"]
1055
+ to_update.append(existing_imp)
1056
+
1057
+ if len(to_create) > 0:
1058
+ ci_url = urljoin(domain, "/api/controlImplementation/batchCreate")
1059
+ resp = api.post(url=ci_url, json=to_create)
1060
+ if resp.ok:
1061
+ control_implementations.extend(resp.json())
1062
+ logger.info(f"Created {len(to_create)} Control Implementation(s), Successfully!")
1063
+ else:
1064
+ resp.raise_for_status()
1065
+ if len(to_update) > 0:
1066
+ ci_url = urljoin(domain, "/api/controlImplementation/batchUpdate")
1067
+ resp = api.post(url=ci_url, json=to_update)
1068
+ if resp.ok:
1069
+ control_implementations.extend(resp.json())
1070
+ logger.info(f"Updated {len(to_update)} Control Implementation(s), Successfully!")
1071
+ else:
1072
+ resp.raise_for_status()
1073
+ return control_implementations
1074
+
1075
+
1076
+ def check_implementation(passing_controls: Dict, failing_controls: Dict, control_id: str) -> str:
1077
+ """
1078
+ Checks the status of a control implementation
1079
+
1080
+ :param Dict passing_controls: Dictionary of passing controls
1081
+ :param Dict failing_controls: Dictionary of failing controls
1082
+ :param str control_id: control id
1083
+ :return: status of control implementation
1084
+ :rtype: str
1085
+ """
1086
+ if control_id in passing_controls.keys():
1087
+ return FULLY_IMPLEMENTED
1088
+ elif control_id in failing_controls.keys():
1089
+ return IN_REMEDIATION
1090
+ else:
1091
+ return NOT_IMPLEMENTED
1092
+
1093
+
1094
+ def get_existing_control_implementations(parent_id: int) -> Dict:
1095
+ """
1096
+ fetch existing control implementations
1097
+
1098
+ :param int parent_id: parent control id
1099
+ :return: Dictionary of existing control implementations
1100
+ :rtype: Dict
1101
+ """
1102
+ app = Application()
1103
+ api = Api()
1104
+ domain = app.config.get("domain")
1105
+ existing_implementation_dict = {}
1106
+ get_url = urljoin(domain, f"/api/controlImplementation/getAllByPlan/{parent_id}")
1107
+ response = api.get(get_url)
1108
+ if response.ok:
1109
+ existing_control_implementations_json = response.json()
1110
+ for cim in existing_control_implementations_json:
1111
+ existing_implementation_dict[cim["controlName"]] = cim
1112
+ logger.info(f"Found {len(existing_implementation_dict)} existing control implementations")
1113
+ elif response.status_code == 404:
1114
+ logger.info(f"No existing control implementations found for {parent_id}")
1115
+ else:
1116
+ logger.warn(f"Unable to get existing control implementations. {response.text}")
1117
+
1118
+ return existing_implementation_dict
1119
+
1120
+
1121
+ def get_matched_controls(tenable_controls: List[Dict], catalog_controls: List[Dict]) -> List[Dict]:
1122
+ """
1123
+ Get controls that match between Tenable and the catalog
1124
+
1125
+ :param List[Dict] tenable_controls: List of controls from Tenable
1126
+ :param List[Dict] catalog_controls: List of controls from the catalog
1127
+ :return: List of matched controls
1128
+ :rtype: List[Dict]
1129
+ """
1130
+ matched_controls = []
1131
+ for control in tenable_controls:
1132
+ formatted_control = convert_control_id(control)
1133
+ logger.info(formatted_control)
1134
+ for catalog_control in catalog_controls:
1135
+ if catalog_control["controlId"].lower() == formatted_control.lower():
1136
+ logger.info(f"Catalog Control {formatted_control} matched")
1137
+ matched_controls.append(catalog_control)
1138
+ break
1139
+ return matched_controls
1140
+
1141
+
1142
+ def get_assessment_status_from_implementation_status(status: str) -> str:
1143
+ """
1144
+ Get the assessment status from the implementation status
1145
+
1146
+ :param str status: Implementation status
1147
+ :return: Assessment status
1148
+ :rtype: str
1149
+ """
1150
+ if status == FULLY_IMPLEMENTED:
1151
+ return "Pass"
1152
+ if status == IN_REMEDIATION:
1153
+ return "Fail"
1154
+ else:
1155
+ return "N/A"
1156
+
1157
+
1158
+ def create_assessment_from_cim(cim: Dict, user_id: str, control: Dict, check: List[AssetCheck]) -> Dict:
1159
+ """
1160
+ Create an assessment from a control implementation
1161
+
1162
+ :param Dict cim: Control Implementation
1163
+ :param str user_id: User ID
1164
+ :param Dict control: Control
1165
+ :param List[AssetCheck] check: Asset Check
1166
+ :return: Assessment
1167
+ :rtype: Dict
1168
+ """
1169
+ assessment_result = get_assessment_status_from_implementation_status(cim.get("status"))
1170
+ summary_dict = check[0].dict() if check else dict()
1171
+ summary_dict.pop("reference", None)
1172
+ title = summary_dict.get("check_name") if summary_dict else control.get("title")
1173
+ html_summary = format_dict_to_html(summary_dict)
1174
+ document_reviewed = check[0].audit_file if check else None
1175
+ check_name = check[0].check_name if check else None
1176
+ methodology = check[0].check_info if check else None
1177
+ summary_of_results = check[0].description if check else None
1178
+ uuid = check[0].asset_uuid if check and check[0].asset_uuid is not None else None
1179
+ title_part = f"{title} - {uuid}" if uuid else f"{title}"
1180
+ uuid_title = f"{title_part} Automated Assessment test"
1181
+ return {
1182
+ "leadAssessorId": user_id,
1183
+ "title": uuid_title,
1184
+ "assessmentType": "Control Testing",
1185
+ "plannedStart": get_current_datetime(),
1186
+ "plannedFinish": get_current_datetime(),
1187
+ "status": "Complete",
1188
+ "assessmentResult": assessment_result if assessment_result else "N/A",
1189
+ "controlID": cim["id"],
1190
+ "actualFinish": get_current_datetime(),
1191
+ "assessmentReport": html_summary if html_summary else "Passed",
1192
+ "parentId": cim["id"],
1193
+ "parentModule": "controls",
1194
+ "assessmentPlan": check_name if check_name else None,
1195
+ "documentsReviewed": document_reviewed if document_reviewed else None,
1196
+ "methodology": methodology if methodology else None,
1197
+ "summaryOfResults": summary_of_results if summary_of_results else None,
1198
+ }
1199
+
1200
+
1201
+ def get_control_assessments(control: Dict, assessments_to_create: List[Dict]) -> List[Dict]:
1202
+ """
1203
+ Get control assessments
1204
+
1205
+ :param Dict control: Control
1206
+ :param List[Dict] assessments_to_create: List of assessments to create
1207
+ :return: List of control assessments
1208
+ :rtype: List[Dict]
1209
+ """
1210
+ return [
1211
+ assess
1212
+ for assess in assessments_to_create
1213
+ if assess["controlID"] == control["id"] and assess["status"] == "Complete"
1214
+ ]
1215
+
1216
+
1217
+ def sort_assessments(control_assessments: List[Dict]) -> List[Dict]:
1218
+ """
1219
+ Sort assessments by actual finish date
1220
+
1221
+ :param List[Dict] control_assessments: List of control assessments
1222
+ :return: Sorted assessments
1223
+ :rtype: List[Dict]
1224
+ """
1225
+ dt_format = "%Y-%m-%d %H:%M:%S"
1226
+ return sorted(
1227
+ control_assessments,
1228
+ key=lambda x: datetime.strptime(x["actualFinish"], dt_format),
1229
+ reverse=True,
1230
+ )
1231
+
1232
+
1233
+ def update_control_object(control: Dict, sorted_assessments: List[Dict]) -> None:
1234
+ """
1235
+ Update control object
1236
+
1237
+ :param Dict control: Control
1238
+ :param List[Dict] sorted_assessments: Sorted assessments
1239
+ :rtype: None
1240
+ """
1241
+ dt_format = "%Y-%m-%d %H:%M:%S"
1242
+ app = Application()
1243
+ control["dateLastAssessed"] = sorted_assessments[0]["actualFinish"]
1244
+ control["lastAssessmentResult"] = sorted_assessments[0]["assessmentResult"]
1245
+ if control.get("lastAssessmentResult"):
1246
+ control_obj = ControlImplementation(**control)
1247
+ if control_obj.lastAssessmentResult == "Fail" and control_obj.status != IN_REMEDIATION:
1248
+ control_obj.status = IN_REMEDIATION
1249
+ control_obj.plannedImplementationDate = (datetime.now() + timedelta(30)).strftime(dt_format)
1250
+ control_obj.stepsToImplement = "n/a"
1251
+ elif control_obj.status == IN_REMEDIATION:
1252
+ control_obj.plannedImplementationDate = (
1253
+ (datetime.now() + timedelta(30)).strftime(dt_format)
1254
+ if not control_obj.plannedImplementationDate
1255
+ else control_obj.plannedImplementationDate
1256
+ )
1257
+ control_obj.stepsToImplement = "n/a" if not control_obj.stepsToImplement else control_obj.stepsToImplement
1258
+ elif control_obj.lastAssessmentResult == "Pass" and control_obj.status != FULLY_IMPLEMENTED:
1259
+ control_obj.status = FULLY_IMPLEMENTED
1260
+ ControlImplementation.update(app=app, implementation=control_obj)
1261
+
1262
+
1263
+ def update_control_implementations(control_implementations: List[Dict], assessments_to_create: List[Dict]) -> None:
1264
+ """
1265
+ Update control implementations with assessments
1266
+
1267
+ :param List[Dict] control_implementations: List of control implementations
1268
+ :param List[Dict] assessments_to_create: List of assessments to create
1269
+ :rtype: None
1270
+ """
1271
+ for control in control_implementations:
1272
+ control_assessments = get_control_assessments(control, assessments_to_create)
1273
+ if sorted_assessments := sort_assessments(control_assessments):
1274
+ update_control_object(control, sorted_assessments)
1275
+
1276
+
1277
+ def post_assessments_to_api(assessments_to_create: List[Dict]) -> None:
1278
+ """
1279
+ Post assessments to the API
1280
+
1281
+ :param List[Dict] assessments_to_create: List of assessments to create
1282
+ :rtype: None
1283
+ """
1284
+ app = Application()
1285
+ api = Api()
1286
+ assessment_url = urljoin(app.config.get("domain", ""), "/api/assessments/batchCreate")
1287
+ assessment_response = api.post(url=assessment_url, json=assessments_to_create)
1288
+ if assessment_response.ok:
1289
+ logger.info(f"Created {len(assessment_response.json())} Assessments!")
1290
+ else:
1291
+ logger.debug(assessment_response.status_code)
1292
+ logger.error(f"Failed to insert Assessment.\n{assessment_response.text}")
1293
+
1294
+
1295
+ def create_assessments(
1296
+ control_implementations: List[Dict],
1297
+ catalog_controls_dict: Dict,
1298
+ asset_checks: Dict,
1299
+ ) -> None:
1300
+ """
1301
+ Create assessments from control implementations
1302
+
1303
+ :param List[Dict] control_implementations: List of control implementations
1304
+ :param Dict catalog_controls_dict: Dictionary of catalog controls
1305
+ :param Dict asset_checks: Dictionary of asset checks
1306
+ :rtype: None
1307
+ :return: None
1308
+ """
1309
+ app = Application()
1310
+ user_id = app.config.get("userId", "")
1311
+ assessments_to_create = []
1312
+ for cim in control_implementations:
1313
+ control = catalog_controls_dict.get(cim["controlID"], {})
1314
+ check = asset_checks.get(control["controlId"].lower())
1315
+ assessment = create_assessment_from_cim(cim, user_id, control, check)
1316
+ assessments_to_create.append(assessment)
1317
+ update_control_implementations(control_implementations, assessments_to_create)
1318
+ post_assessments_to_api(assessments_to_create)
1319
+
1320
+
1321
+ def process_compliance_data(
1322
+ framework_data: Dict,
1323
+ catalog_id: int,
1324
+ ssp_id: int,
1325
+ framework: str,
1326
+ passing_controls: Dict,
1327
+ failing_controls: Dict,
1328
+ ) -> None:
1329
+ """
1330
+ Processes the compliance data from Tenable.io to create control implementations for controls in frameworks
1331
+
1332
+ :param Dict framework_data: List of tenable.io controls per framework
1333
+ :param int catalog_id: The catalog id
1334
+ :param int ssp_id: The ssp id
1335
+ :param str framework: The framework name
1336
+ :param Dict passing_controls: Dictionary of passing controls
1337
+ :param Dict failing_controls: Dictionary of failing controls
1338
+ :rtype: None
1339
+ """
1340
+ if not framework_data:
1341
+ return
1342
+ framework_controls = framework_data.get("controls", {})
1343
+ asset_checks = framework_data.get("asset_checks", {})
1344
+ existing_implementation_dict = get_existing_control_implementations(ssp_id)
1345
+ catalog_controls = get_controls(catalog_id)
1346
+ matched_controls = []
1347
+ for tenable_framework, tenable_controls in framework_controls.items():
1348
+ logger.info(f"Found {len(tenable_controls)} controls that passed for framework: {tenable_framework}")
1349
+ # logger.info(f"tenable_controls: {tenable_controls[0]}") if len(tenable_controls) >0 else None
1350
+ if tenable_framework == framework:
1351
+ matched_controls = get_matched_controls(tenable_controls, catalog_controls)
1352
+
1353
+ logger.info(f"Found {len(matched_controls)} controls that matched")
1354
+
1355
+ control_implementations = create_control_implementations(
1356
+ controls=matched_controls,
1357
+ parent_id=ssp_id,
1358
+ parent_module="securityplans",
1359
+ existing_implementation_dict=existing_implementation_dict,
1360
+ passing_controls=passing_controls,
1361
+ failing_controls=failing_controls,
1362
+ )
1363
+
1364
+ logger.info(f"SSP now has {len(control_implementations)} control implementations")
1365
+ catalog_controls_dict = {c["id"]: c for c in catalog_controls}
1366
+ create_assessments(control_implementations, catalog_controls_dict, asset_checks)
1367
+
1368
+
1369
+ def convert_control_id(control_id: str) -> str:
1370
+ """
1371
+ Convert the control id to a format that can be used in Tenable.io
1372
+
1373
+ :param str control_id: The control id to convert
1374
+ :return: The converted control id
1375
+ :rtype: str
1376
+ """
1377
+ # Convert to lowercase
1378
+ control_id = control_id.lower()
1379
+
1380
+ # Check if there's a parenthesis and replace its content
1381
+ if "(" in control_id and ")" in control_id:
1382
+ inner_value = control_id.split("(")[1].split(")")[0]
1383
+ control_id = control_id.replace(f"({inner_value})", f".{inner_value}")
1384
+
1385
+ return control_id
1386
+
1387
+
1388
+ @io.command(name="sync_compliance_controls")
1389
+ @regscale_ssp_id()
1390
+ @click.option(
1391
+ "--catalog_id",
1392
+ type=click.INT,
1393
+ help="The ID number from RegScale Catalog that the System Security Plan's controls belong to",
1394
+ prompt="Enter RegScale Catalog ID",
1395
+ required=True,
1396
+ )
1397
+ @click.option(
1398
+ "--framework",
1399
+ required=True,
1400
+ type=click.Choice(["800-53", "800-53r5", "CSF", "800-171"], case_sensitive=True),
1401
+ help="The framework to use. from Tenable.io frameworks MUST be the same RegScale Catalog of controls",
1402
+ )
1403
+ @hidden_file_path(help="The file path to load control data instead of fetching from Tenable.io")
1404
+ def sync_compliance_data(regscale_ssp_id: int, catalog_id: int, framework: str, offline: Optional[Path] = None):
1405
+ """
1406
+ Sync the compliance data from Tenable.io to create control implementations for controls in frameworks.
1407
+ """
1408
+ _sync_compliance_data(ssp_id=regscale_ssp_id, catalog_id=catalog_id, framework=framework, offline=offline)
1409
+
1410
+
1411
+ def _sync_compliance_data(ssp_id: int, catalog_id: int, framework: str, offline: Optional[Path] = None) -> None:
1412
+ """
1413
+ Sync the compliance data from Tenable.io to create control implementations for controls in frameworks
1414
+ :param int ssp_id: The ID number from RegScale of the System Security Plan
1415
+ :param int catalog_id: The ID number from RegScale Catalog that the System Security Plan's controls belong to
1416
+ :param str framework: The framework to use. from Tenable.io frameworks MUST be the same RegScale Catalog of controls
1417
+ :param Optional[Path] offline: The file path to load control data instead of fetching from Tenable.io, defaults to None
1418
+ :rtype: None
1419
+ """
1420
+ logger.info("Note: This command only available for Tenable.io")
1421
+ logger.info("Note: This command Requires admin access.")
1422
+ app = Application()
1423
+ config = app.config
1424
+ # we specifically don't gen client here, so we only get the client for Tenable.io as its only supported there
1425
+
1426
+ compliance_data = _get_compliance_data(config=config, offline=offline) # type: ignore
1427
+
1428
+ dict_of_frameworks_and_asset_checks: Dict = dict()
1429
+ framework_controls: Dict[str, List[str]] = {}
1430
+ asset_checks: Dict[str, List[AssetCheck]] = {}
1431
+ passing_controls: Dict = dict()
1432
+ # partial_passing_controls: Dict = dict()
1433
+ failing_controls: Dict = dict()
1434
+ for findings in compliance_data:
1435
+ asset_check = AssetCheck(**findings)
1436
+ for ref in asset_check.reference:
1437
+ if ref.framework not in framework_controls:
1438
+ framework_controls[ref.framework] = []
1439
+ if ref.control not in framework_controls[ref.framework]: # Avoid duplicate controls
1440
+ framework_controls[ref.framework].append(ref.control)
1441
+ formatted_control_id = convert_control_id(ref.control)
1442
+ # sort controls by status
1443
+ add_control_to_status_dict(
1444
+ control_id=formatted_control_id,
1445
+ status=asset_check.status,
1446
+ dict_obj=failing_controls,
1447
+ desired_status="FAILED",
1448
+ )
1449
+ add_control_to_status_dict(
1450
+ control_id=formatted_control_id,
1451
+ status=asset_check.status,
1452
+ dict_obj=passing_controls,
1453
+ desired_status="PASSED",
1454
+ )
1455
+ remove_passing_controls_if_in_failed_status(passing=passing_controls, failing=failing_controls)
1456
+ if formatted_control_id not in asset_checks:
1457
+ asset_checks[formatted_control_id] = [asset_check]
1458
+ else:
1459
+ asset_checks[formatted_control_id].append(asset_check)
1460
+ dict_of_frameworks_and_asset_checks = {
1461
+ key: {"controls": framework_controls, "asset_checks": asset_checks} for key in framework_controls.keys()
1462
+ }
1463
+ logger.info(f"Found {len(dict_of_frameworks_and_asset_checks)} findings to process")
1464
+ framework_data = dict_of_frameworks_and_asset_checks.get(framework, None)
1465
+ process_compliance_data(
1466
+ framework_data=framework_data,
1467
+ catalog_id=catalog_id,
1468
+ ssp_id=ssp_id,
1469
+ framework=framework,
1470
+ passing_controls=passing_controls,
1471
+ failing_controls=failing_controls,
1472
+ )
1473
+
1474
+
1475
+ def _get_compliance_data(config: dict, offline: Optional[Path] = None) -> Dict:
1476
+ """
1477
+ Get compliance data from Tenable.io
1478
+
1479
+ :param dict config: Configuration dictionary
1480
+ :param Optional[Path] offline: File path to load control data instead of fetching from Tenable.io
1481
+ :return: Compliance data
1482
+ :rtype: Dict
1483
+ """
1484
+ from tenable.io import TenableIO
1485
+
1486
+ if offline:
1487
+ with open(offline.absolute(), "r") as f:
1488
+ compliance_data = json.load(f)
1489
+ else:
1490
+ client = TenableIO(
1491
+ url=config["tenableUrl"],
1492
+ access_key=config["tenableAccessKey"],
1493
+ secret_key=config["tenableSecretKey"],
1494
+ vendor=REGSCALE_INC,
1495
+ product=REGSCALE_CLI,
1496
+ build=__version__,
1497
+ )
1498
+ compliance_data = client.exports.compliance()
1499
+ return compliance_data
1500
+
1501
+
1502
+ def add_control_to_status_dict(control_id: str, status: str, dict_obj: Dict, desired_status: str) -> None:
1503
+ """
1504
+ Add a control to a status dictionary
1505
+
1506
+ :param str control_id: The control id to add to the dictionary
1507
+ :param str status: The status of the control
1508
+ :param Dict dict_obj: The dictionary to add the control to
1509
+ :param str desired_status: The desired status of the control
1510
+ :rtype: None
1511
+ """
1512
+ friendly_control_id = control_id.lower()
1513
+ if status == desired_status and friendly_control_id not in dict_obj:
1514
+ dict_obj[friendly_control_id] = desired_status
1515
+
1516
+
1517
+ def remove_passing_controls_if_in_failed_status(passing: Dict, failing: Dict) -> None:
1518
+ """
1519
+ Remove passing controls if they are in failed status
1520
+
1521
+ :param Dict passing: Dictionary of passing controls
1522
+ :param Dict failing: Dictionary of failing controls
1523
+ :rtype: None
1524
+ """
1525
+ to_remove = []
1526
+ for k in passing.keys():
1527
+ if k in failing.keys():
1528
+ to_remove.append(k)
1529
+
1530
+ for k in to_remove:
1531
+ del passing[k]
1532
+
1533
+
1534
+ def write_io_chunk(
1535
+ data: List[dict],
1536
+ data_dir: Path,
1537
+ export_uuid: str,
1538
+ export_type: str,
1539
+ export_chunk_id: int,
1540
+ ) -> None:
1541
+ """
1542
+ Write a chunk of data to a file, this function is formatted for use with PyTenable and Tenable IO
1543
+
1544
+ :param List[dict] data: Data to write to a file
1545
+ :param Path data_dir: Directory to write the file to
1546
+ :param str export_uuid: UUID of the export (Tenable IO)
1547
+ :param str export_type: Type of export (Tenable IO)
1548
+ :param int export_chunk_id: ID of the chunk (Tenable IO)
1549
+ :rtype: None
1550
+ """
1551
+ # create tenable io directory
1552
+ data_dir.mkdir(parents=True, exist_ok=True)
1553
+ fn = data_dir / f"{export_type}-{export_uuid}-{export_chunk_id}.json"
1554
+ # append file to path
1555
+ with open(file=fn, mode="w", encoding="utf-8") as file_object:
1556
+ json.dump(data, file_object)
1557
+
1558
+
1559
+ def process_to_regscale(data_dir: Path, ssp_id: int, existing_assets: List[Asset]) -> None:
1560
+ """
1561
+ Process the Tenable data to RegScale
1562
+
1563
+ :param Path data_dir: Directory to process the data from
1564
+ :param int ssp_id: The ID of the System Security Plan
1565
+ :param List[Asset] existing_assets: List of existing assets
1566
+ :rtype: None
1567
+ :return: None
1568
+ """
1569
+ # get all files in the directory
1570
+ files = list(data_dir.glob("*.json"))
1571
+ if not files:
1572
+ logger.warning("No Tenable files found in %s.", data_dir)
1573
+ return
1574
+ logger.info("Processing %i chunked file(s) from Tenable...", len(list(files)))
1575
+ for file in files:
1576
+ logger.info("Processing chunked data: %s", file)
1577
+ file_assets = []
1578
+ with open(file=file, mode="r", encoding="utf-8") as file_object:
1579
+ tenable_io_data = json.load(file_object)
1580
+ for asset in tenable_io_data:
1581
+ file_assets.append(TenableIOAsset(**asset))
1582
+ TenableIOAsset.sync_to_regscale(assets=file_assets, ssp_id=ssp_id, existing_assets=existing_assets)
1583
+ # remove processed file
1584
+ file.unlink()