runbooks 0.2.5__py3-none-any.whl → 0.6.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. conftest.py +26 -0
  2. jupyter-agent/.env.template +2 -0
  3. jupyter-agent/.gitattributes +35 -0
  4. jupyter-agent/README.md +16 -0
  5. jupyter-agent/app.py +256 -0
  6. jupyter-agent/cloudops-agent.png +0 -0
  7. jupyter-agent/ds-system-prompt.txt +154 -0
  8. jupyter-agent/jupyter-agent.png +0 -0
  9. jupyter-agent/llama3_template.jinja +123 -0
  10. jupyter-agent/requirements.txt +9 -0
  11. jupyter-agent/utils.py +409 -0
  12. runbooks/__init__.py +71 -3
  13. runbooks/__main__.py +13 -0
  14. runbooks/aws/ec2_describe_instances.py +1 -1
  15. runbooks/aws/ec2_run_instances.py +8 -2
  16. runbooks/aws/ec2_start_stop_instances.py +17 -4
  17. runbooks/aws/ec2_unused_volumes.py +5 -1
  18. runbooks/aws/s3_create_bucket.py +4 -2
  19. runbooks/aws/s3_list_objects.py +6 -1
  20. runbooks/aws/tagging_lambda_handler.py +13 -2
  21. runbooks/aws/tags.json +12 -0
  22. runbooks/base.py +353 -0
  23. runbooks/cfat/README.md +49 -0
  24. runbooks/cfat/__init__.py +74 -0
  25. runbooks/cfat/app.ts +644 -0
  26. runbooks/cfat/assessment/__init__.py +40 -0
  27. runbooks/cfat/assessment/asana-import.csv +39 -0
  28. runbooks/cfat/assessment/cfat-checks.csv +31 -0
  29. runbooks/cfat/assessment/cfat.txt +520 -0
  30. runbooks/cfat/assessment/collectors.py +200 -0
  31. runbooks/cfat/assessment/jira-import.csv +39 -0
  32. runbooks/cfat/assessment/runner.py +387 -0
  33. runbooks/cfat/assessment/validators.py +290 -0
  34. runbooks/cfat/cli.py +103 -0
  35. runbooks/cfat/docs/asana-import.csv +24 -0
  36. runbooks/cfat/docs/cfat-checks.csv +31 -0
  37. runbooks/cfat/docs/cfat.txt +335 -0
  38. runbooks/cfat/docs/checks-output.png +0 -0
  39. runbooks/cfat/docs/cloudshell-console-run.png +0 -0
  40. runbooks/cfat/docs/cloudshell-download.png +0 -0
  41. runbooks/cfat/docs/cloudshell-output.png +0 -0
  42. runbooks/cfat/docs/downloadfile.png +0 -0
  43. runbooks/cfat/docs/jira-import.csv +24 -0
  44. runbooks/cfat/docs/open-cloudshell.png +0 -0
  45. runbooks/cfat/docs/report-header.png +0 -0
  46. runbooks/cfat/models.py +1026 -0
  47. runbooks/cfat/package-lock.json +5116 -0
  48. runbooks/cfat/package.json +38 -0
  49. runbooks/cfat/report.py +496 -0
  50. runbooks/cfat/reporting/__init__.py +46 -0
  51. runbooks/cfat/reporting/exporters.py +337 -0
  52. runbooks/cfat/reporting/formatters.py +496 -0
  53. runbooks/cfat/reporting/templates.py +135 -0
  54. runbooks/cfat/run-assessment.sh +23 -0
  55. runbooks/cfat/runner.py +69 -0
  56. runbooks/cfat/src/actions/check-cloudtrail-existence.ts +43 -0
  57. runbooks/cfat/src/actions/check-config-existence.ts +37 -0
  58. runbooks/cfat/src/actions/check-control-tower.ts +37 -0
  59. runbooks/cfat/src/actions/check-ec2-existence.ts +46 -0
  60. runbooks/cfat/src/actions/check-iam-users.ts +50 -0
  61. runbooks/cfat/src/actions/check-legacy-cur.ts +30 -0
  62. runbooks/cfat/src/actions/check-org-cloudformation.ts +30 -0
  63. runbooks/cfat/src/actions/check-vpc-existence.ts +43 -0
  64. runbooks/cfat/src/actions/create-asanaimport.ts +14 -0
  65. runbooks/cfat/src/actions/create-backlog.ts +372 -0
  66. runbooks/cfat/src/actions/create-jiraimport.ts +15 -0
  67. runbooks/cfat/src/actions/create-report.ts +616 -0
  68. runbooks/cfat/src/actions/define-account-type.ts +51 -0
  69. runbooks/cfat/src/actions/get-enabled-org-policy-types.ts +40 -0
  70. runbooks/cfat/src/actions/get-enabled-org-services.ts +26 -0
  71. runbooks/cfat/src/actions/get-idc-info.ts +34 -0
  72. runbooks/cfat/src/actions/get-org-da-accounts.ts +34 -0
  73. runbooks/cfat/src/actions/get-org-details.ts +35 -0
  74. runbooks/cfat/src/actions/get-org-member-accounts.ts +44 -0
  75. runbooks/cfat/src/actions/get-org-ous.ts +35 -0
  76. runbooks/cfat/src/actions/get-regions.ts +22 -0
  77. runbooks/cfat/src/actions/zip-assessment.ts +27 -0
  78. runbooks/cfat/src/types/index.d.ts +147 -0
  79. runbooks/cfat/tests/__init__.py +141 -0
  80. runbooks/cfat/tests/test_cli.py +340 -0
  81. runbooks/cfat/tests/test_integration.py +290 -0
  82. runbooks/cfat/tests/test_models.py +505 -0
  83. runbooks/cfat/tests/test_reporting.py +354 -0
  84. runbooks/cfat/tsconfig.json +16 -0
  85. runbooks/cfat/webpack.config.cjs +27 -0
  86. runbooks/config.py +260 -0
  87. runbooks/finops/__init__.py +88 -0
  88. runbooks/finops/aws_client.py +245 -0
  89. runbooks/finops/cli.py +151 -0
  90. runbooks/finops/cost_processor.py +410 -0
  91. runbooks/finops/dashboard_runner.py +448 -0
  92. runbooks/finops/helpers.py +355 -0
  93. runbooks/finops/main.py +14 -0
  94. runbooks/finops/profile_processor.py +174 -0
  95. runbooks/finops/types.py +66 -0
  96. runbooks/finops/visualisations.py +80 -0
  97. runbooks/inventory/.gitignore +354 -0
  98. runbooks/inventory/ArgumentsClass.py +261 -0
  99. runbooks/inventory/Inventory_Modules.py +6130 -0
  100. runbooks/inventory/LandingZone/delete_lz.py +1075 -0
  101. runbooks/inventory/README.md +1320 -0
  102. runbooks/inventory/__init__.py +62 -0
  103. runbooks/inventory/account_class.py +532 -0
  104. runbooks/inventory/all_my_instances_wrapper.py +123 -0
  105. runbooks/inventory/aws_decorators.py +201 -0
  106. runbooks/inventory/cfn_move_stack_instances.py +1526 -0
  107. runbooks/inventory/check_cloudtrail_compliance.py +614 -0
  108. runbooks/inventory/check_controltower_readiness.py +1107 -0
  109. runbooks/inventory/check_landingzone_readiness.py +711 -0
  110. runbooks/inventory/cloudtrail.md +727 -0
  111. runbooks/inventory/collectors/__init__.py +20 -0
  112. runbooks/inventory/collectors/aws_compute.py +518 -0
  113. runbooks/inventory/collectors/aws_networking.py +275 -0
  114. runbooks/inventory/collectors/base.py +222 -0
  115. runbooks/inventory/core/__init__.py +19 -0
  116. runbooks/inventory/core/collector.py +303 -0
  117. runbooks/inventory/core/formatter.py +296 -0
  118. runbooks/inventory/delete_s3_buckets_objects.py +169 -0
  119. runbooks/inventory/discovery.md +81 -0
  120. runbooks/inventory/draw_org_structure.py +748 -0
  121. runbooks/inventory/ec2_vpc_utils.py +341 -0
  122. runbooks/inventory/find_cfn_drift_detection.py +272 -0
  123. runbooks/inventory/find_cfn_orphaned_stacks.py +719 -0
  124. runbooks/inventory/find_cfn_stackset_drift.py +733 -0
  125. runbooks/inventory/find_ec2_security_groups.py +669 -0
  126. runbooks/inventory/find_landingzone_versions.py +201 -0
  127. runbooks/inventory/find_vpc_flow_logs.py +1221 -0
  128. runbooks/inventory/inventory.sh +659 -0
  129. runbooks/inventory/list_cfn_stacks.py +558 -0
  130. runbooks/inventory/list_cfn_stackset_operation_results.py +252 -0
  131. runbooks/inventory/list_cfn_stackset_operations.py +734 -0
  132. runbooks/inventory/list_cfn_stacksets.py +453 -0
  133. runbooks/inventory/list_config_recorders_delivery_channels.py +681 -0
  134. runbooks/inventory/list_ds_directories.py +354 -0
  135. runbooks/inventory/list_ec2_availability_zones.py +286 -0
  136. runbooks/inventory/list_ec2_ebs_volumes.py +244 -0
  137. runbooks/inventory/list_ec2_instances.py +425 -0
  138. runbooks/inventory/list_ecs_clusters_and_tasks.py +562 -0
  139. runbooks/inventory/list_elbs_load_balancers.py +411 -0
  140. runbooks/inventory/list_enis_network_interfaces.py +526 -0
  141. runbooks/inventory/list_guardduty_detectors.py +568 -0
  142. runbooks/inventory/list_iam_policies.py +404 -0
  143. runbooks/inventory/list_iam_roles.py +518 -0
  144. runbooks/inventory/list_iam_saml_providers.py +359 -0
  145. runbooks/inventory/list_lambda_functions.py +882 -0
  146. runbooks/inventory/list_org_accounts.py +446 -0
  147. runbooks/inventory/list_org_accounts_users.py +354 -0
  148. runbooks/inventory/list_rds_db_instances.py +406 -0
  149. runbooks/inventory/list_route53_hosted_zones.py +318 -0
  150. runbooks/inventory/list_servicecatalog_provisioned_products.py +575 -0
  151. runbooks/inventory/list_sns_topics.py +360 -0
  152. runbooks/inventory/list_ssm_parameters.py +402 -0
  153. runbooks/inventory/list_vpc_subnets.py +433 -0
  154. runbooks/inventory/list_vpcs.py +422 -0
  155. runbooks/inventory/lockdown_cfn_stackset_role.py +224 -0
  156. runbooks/inventory/models/__init__.py +24 -0
  157. runbooks/inventory/models/account.py +192 -0
  158. runbooks/inventory/models/inventory.py +309 -0
  159. runbooks/inventory/models/resource.py +247 -0
  160. runbooks/inventory/recover_cfn_stack_ids.py +205 -0
  161. runbooks/inventory/requirements.txt +12 -0
  162. runbooks/inventory/run_on_multi_accounts.py +211 -0
  163. runbooks/inventory/tests/common_test_data.py +3661 -0
  164. runbooks/inventory/tests/common_test_functions.py +204 -0
  165. runbooks/inventory/tests/script_test_data.py +0 -0
  166. runbooks/inventory/tests/setup.py +24 -0
  167. runbooks/inventory/tests/src.py +18 -0
  168. runbooks/inventory/tests/test_cfn_describe_stacks.py +208 -0
  169. runbooks/inventory/tests/test_ec2_describe_instances.py +162 -0
  170. runbooks/inventory/tests/test_inventory_modules.py +55 -0
  171. runbooks/inventory/tests/test_lambda_list_functions.py +86 -0
  172. runbooks/inventory/tests/test_moto_integration_example.py +273 -0
  173. runbooks/inventory/tests/test_org_list_accounts.py +49 -0
  174. runbooks/inventory/update_aws_actions.py +173 -0
  175. runbooks/inventory/update_cfn_stacksets.py +1215 -0
  176. runbooks/inventory/update_cloudwatch_logs_retention_policy.py +294 -0
  177. runbooks/inventory/update_iam_roles_cross_accounts.py +478 -0
  178. runbooks/inventory/update_s3_public_access_block.py +539 -0
  179. runbooks/inventory/utils/__init__.py +23 -0
  180. runbooks/inventory/utils/aws_helpers.py +510 -0
  181. runbooks/inventory/utils/threading_utils.py +493 -0
  182. runbooks/inventory/utils/validation.py +682 -0
  183. runbooks/inventory/verify_ec2_security_groups.py +1430 -0
  184. runbooks/main.py +785 -0
  185. runbooks/organizations/__init__.py +12 -0
  186. runbooks/organizations/manager.py +374 -0
  187. runbooks/security_baseline/README.md +324 -0
  188. runbooks/security_baseline/checklist/alternate_contacts.py +8 -1
  189. runbooks/security_baseline/checklist/bucket_public_access.py +4 -1
  190. runbooks/security_baseline/checklist/cloudwatch_alarm_configuration.py +9 -2
  191. runbooks/security_baseline/checklist/guardduty_enabled.py +9 -2
  192. runbooks/security_baseline/checklist/multi_region_instance_usage.py +5 -1
  193. runbooks/security_baseline/checklist/root_access_key.py +6 -1
  194. runbooks/security_baseline/config-origin.json +1 -1
  195. runbooks/security_baseline/config.json +1 -1
  196. runbooks/security_baseline/permission.json +1 -1
  197. runbooks/security_baseline/report_generator.py +10 -2
  198. runbooks/security_baseline/report_template_en.html +7 -7
  199. runbooks/security_baseline/report_template_jp.html +7 -7
  200. runbooks/security_baseline/report_template_kr.html +12 -12
  201. runbooks/security_baseline/report_template_vn.html +7 -7
  202. runbooks/security_baseline/requirements.txt +7 -0
  203. runbooks/security_baseline/run_script.py +8 -2
  204. runbooks/security_baseline/security_baseline_tester.py +10 -2
  205. runbooks/security_baseline/utils/common.py +5 -1
  206. runbooks/utils/__init__.py +204 -0
  207. runbooks-0.6.1.dist-info/METADATA +373 -0
  208. runbooks-0.6.1.dist-info/RECORD +237 -0
  209. {runbooks-0.2.5.dist-info → runbooks-0.6.1.dist-info}/WHEEL +1 -1
  210. runbooks-0.6.1.dist-info/entry_points.txt +7 -0
  211. runbooks-0.6.1.dist-info/licenses/LICENSE +201 -0
  212. runbooks-0.6.1.dist-info/top_level.txt +3 -0
  213. runbooks/python101/calculator.py +0 -34
  214. runbooks/python101/config.py +0 -1
  215. runbooks/python101/exceptions.py +0 -16
  216. runbooks/python101/file_manager.py +0 -218
  217. runbooks/python101/toolkit.py +0 -153
  218. runbooks-0.2.5.dist-info/METADATA +0 -439
  219. runbooks-0.2.5.dist-info/RECORD +0 -61
  220. runbooks-0.2.5.dist-info/entry_points.txt +0 -3
  221. runbooks-0.2.5.dist-info/top_level.txt +0 -1
@@ -0,0 +1,340 @@
1
+ """
2
+ CLI tests for Cloud Foundations Assessment Tool.
3
+
4
+ Tests command-line interface argument parsing, validation, and integration
5
+ with the assessment engine. These tests focus on CLI behavior separately
6
+ from AWS API interactions.
7
+ """
8
+
9
+ import os
10
+ import tempfile
11
+ from unittest.mock import MagicMock, patch
12
+
13
+ import pytest
14
+ from click.testing import CliRunner
15
+
16
+ from runbooks.cfat.tests import create_sample_assessment_report
17
+ from runbooks.main import assess, cfat, main
18
+
19
+
20
+ @pytest.mark.cli
21
+ class TestCFATCLI:
22
+ """Test CFAT CLI functionality."""
23
+
24
+ def setup_method(self):
25
+ """Set up test environment."""
26
+ self.runner = CliRunner()
27
+ self.sample_report = create_sample_assessment_report()
28
+
29
+ def test_cfat_help_command(self):
30
+ """Test CFAT help command."""
31
+ result = self.runner.invoke(main, ["cfat", "--help"])
32
+
33
+ assert result.exit_code == 0
34
+ assert "Cloud Foundations Assessment Tool" in result.output
35
+ assert "assess" in result.output
36
+
37
+ def test_assess_help_command(self):
38
+ """Test assess command help."""
39
+ result = self.runner.invoke(main, ["cfat", "assess", "--help"])
40
+
41
+ assert result.exit_code == 0
42
+ assert "Run enhanced Cloud Foundations assessment" in result.output
43
+ assert "--output" in result.output
44
+ assert "--severity" in result.output
45
+ assert "--parallel" in result.output
46
+ assert "--export-jira" in result.output
47
+
48
+ def test_assess_basic_arguments(self):
49
+ """Test basic assess command arguments."""
50
+ with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
51
+ mock_instance = MagicMock()
52
+ mock_instance.run_assessment.return_value = self.sample_report
53
+ mock_runner.return_value = mock_instance
54
+
55
+ result = self.runner.invoke(main, ["cfat", "assess", "--output", "console", "--severity", "WARNING"])
56
+
57
+ # Should complete successfully
58
+ assert result.exit_code == 0
59
+
60
+ # Verify runner was initialized and configured
61
+ mock_runner.assert_called_once()
62
+ mock_instance.set_min_severity.assert_called_with("WARNING")
63
+ mock_instance.run_assessment.assert_called_once()
64
+
65
+ def test_assess_output_formats(self):
66
+ """Test different output formats."""
67
+ formats = ["console", "html", "csv", "json", "markdown"]
68
+
69
+ for format_type in formats:
70
+ with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
71
+ mock_instance = MagicMock()
72
+ mock_instance.run_assessment.return_value = self.sample_report
73
+ mock_runner.return_value = mock_instance
74
+
75
+ result = self.runner.invoke(main, ["cfat", "assess", "--output", format_type])
76
+
77
+ assert result.exit_code == 0, f"Failed for format: {format_type}"
78
+
79
+ def test_assess_all_output_format(self):
80
+ """Test 'all' output format generates multiple files."""
81
+ with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
82
+ mock_instance = MagicMock()
83
+ mock_instance.run_assessment.return_value = self.sample_report
84
+ mock_runner.return_value = mock_instance
85
+
86
+ with tempfile.TemporaryDirectory() as temp_dir:
87
+ original_cwd = os.getcwd()
88
+ try:
89
+ os.chdir(temp_dir)
90
+
91
+ result = self.runner.invoke(main, ["cfat", "assess", "--output", "all"])
92
+
93
+ assert result.exit_code == 0
94
+ assert "Generated files:" in result.output
95
+
96
+ finally:
97
+ os.chdir(original_cwd)
98
+
99
+ def test_assess_specific_checks(self):
100
+ """Test specifying specific checks to run."""
101
+ with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
102
+ mock_instance = MagicMock()
103
+ mock_instance.run_assessment.return_value = self.sample_report
104
+ mock_runner.return_value = mock_instance
105
+
106
+ result = self.runner.invoke(
107
+ main, ["cfat", "assess", "--checks", "iam_root_mfa", "--checks", "cloudtrail_enabled"]
108
+ )
109
+
110
+ assert result.exit_code == 0
111
+ mock_instance.set_checks.assert_called_with(["iam_root_mfa", "cloudtrail_enabled"])
112
+
113
+ def test_assess_skip_checks(self):
114
+ """Test skipping specific checks."""
115
+ with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
116
+ mock_instance = MagicMock()
117
+ mock_instance.run_assessment.return_value = self.sample_report
118
+ mock_runner.return_value = mock_instance
119
+
120
+ result = self.runner.invoke(
121
+ main, ["cfat", "assess", "--skip-checks", "ec2_instances", "--skip-checks", "rds_instances"]
122
+ )
123
+
124
+ assert result.exit_code == 0
125
+ mock_instance.skip_checks.assert_called_with(["ec2_instances", "rds_instances"])
126
+
127
+ def test_assess_categories(self):
128
+ """Test specifying assessment categories."""
129
+ with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
130
+ mock_instance = MagicMock()
131
+ mock_instance.run_assessment.return_value = self.sample_report
132
+ mock_instance.assessment_config = MagicMock()
133
+ mock_runner.return_value = mock_instance
134
+
135
+ result = self.runner.invoke(
136
+ main, ["cfat", "assess", "--categories", "iam", "--categories", "vpc", "--skip-categories", "ec2"]
137
+ )
138
+
139
+ assert result.exit_code == 0
140
+
141
+ # Verify categories were set
142
+ assert mock_instance.assessment_config.included_categories == ["iam", "vpc"]
143
+ assert mock_instance.assessment_config.excluded_categories == ["ec2"]
144
+
145
+ def test_assess_parallel_execution_options(self):
146
+ """Test parallel execution configuration."""
147
+ with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
148
+ mock_instance = MagicMock()
149
+ mock_instance.run_assessment.return_value = self.sample_report
150
+ mock_instance.assessment_config = MagicMock()
151
+ mock_runner.return_value = mock_instance
152
+
153
+ # Test parallel execution
154
+ result = self.runner.invoke(main, ["cfat", "assess", "--parallel", "--max-workers", "5"])
155
+
156
+ assert result.exit_code == 0
157
+ assert mock_instance.assessment_config.parallel_execution is True
158
+ assert mock_instance.assessment_config.max_workers == 5
159
+
160
+ # Test sequential execution
161
+ result = self.runner.invoke(main, ["cfat", "assess", "--sequential"])
162
+
163
+ assert result.exit_code == 0
164
+ assert mock_instance.assessment_config.parallel_execution is False
165
+
166
+ def test_assess_compliance_framework(self):
167
+ """Test compliance framework specification."""
168
+ with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
169
+ mock_instance = MagicMock()
170
+ mock_instance.run_assessment.return_value = self.sample_report
171
+ mock_instance.assessment_config = MagicMock()
172
+ mock_runner.return_value = mock_instance
173
+
174
+ result = self.runner.invoke(main, ["cfat", "assess", "--compliance-framework", "SOC2"])
175
+
176
+ assert result.exit_code == 0
177
+ assert mock_instance.assessment_config.compliance_framework == "SOC2"
178
+
179
+ def test_assess_export_integrations(self):
180
+ """Test export to project management tools."""
181
+ with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
182
+ with patch("runbooks.cfat.reporting.exporters.JiraExporter") as mock_jira:
183
+ with patch("runbooks.cfat.reporting.exporters.AsanaExporter") as mock_asana:
184
+ mock_instance = MagicMock()
185
+ mock_instance.run_assessment.return_value = self.sample_report
186
+ mock_runner.return_value = mock_instance
187
+
188
+ mock_jira_instance = MagicMock()
189
+ mock_asana_instance = MagicMock()
190
+ mock_jira.return_value = mock_jira_instance
191
+ mock_asana.return_value = mock_asana_instance
192
+
193
+ with tempfile.TemporaryDirectory() as temp_dir:
194
+ jira_file = os.path.join(temp_dir, "jira.csv")
195
+ asana_file = os.path.join(temp_dir, "asana.csv")
196
+
197
+ result = self.runner.invoke(
198
+ main, ["cfat", "assess", "--export-jira", jira_file, "--export-asana", asana_file]
199
+ )
200
+
201
+ assert result.exit_code == 0
202
+
203
+ # Verify exporters were called
204
+ mock_jira_instance.export.assert_called_once_with(self.sample_report, jira_file)
205
+ mock_asana_instance.export.assert_called_once_with(self.sample_report, asana_file)
206
+
207
+ def test_assess_web_server_option(self):
208
+ """Test web server option (without actually starting server)."""
209
+ with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
210
+ with patch("runbooks.main.start_web_server") as mock_web_server:
211
+ mock_instance = MagicMock()
212
+ mock_instance.run_assessment.return_value = self.sample_report
213
+ mock_runner.return_value = mock_instance
214
+
215
+ result = self.runner.invoke(main, ["cfat", "assess", "--serve-web", "--web-port", "9000"])
216
+
217
+ assert result.exit_code == 0
218
+ mock_web_server.assert_called_once_with(self.sample_report, 9000)
219
+
220
+ def test_assess_output_file_specification(self):
221
+ """Test specifying custom output file."""
222
+ with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
223
+ mock_instance = MagicMock()
224
+ mock_instance.run_assessment.return_value = self.sample_report
225
+ mock_runner.return_value = mock_instance
226
+
227
+ with tempfile.TemporaryDirectory() as temp_dir:
228
+ output_file = os.path.join(temp_dir, "custom_report.html")
229
+
230
+ result = self.runner.invoke(main, ["cfat", "assess", "--output", "html", "--output-file", output_file])
231
+
232
+ assert result.exit_code == 0
233
+
234
+ def test_assess_invalid_arguments(self):
235
+ """Test handling of invalid arguments."""
236
+ # Test invalid output format
237
+ result = self.runner.invoke(main, ["cfat", "assess", "--output", "invalid_format"])
238
+ assert result.exit_code != 0
239
+
240
+ # Test invalid severity
241
+ result = self.runner.invoke(main, ["cfat", "assess", "--severity", "INVALID"])
242
+ assert result.exit_code != 0
243
+
244
+ # Test invalid max-workers
245
+ result = self.runner.invoke(main, ["cfat", "assess", "--max-workers", "0"])
246
+ # This should succeed at CLI level but may fail in assessment config validation
247
+ assert result.exit_code in [0, 1] # Either CLI rejects or assessment validates
248
+
249
+ def test_assess_error_handling(self):
250
+ """Test CLI error handling when assessment fails."""
251
+ with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
252
+ mock_instance = MagicMock()
253
+ mock_instance.run_assessment.side_effect = Exception("Assessment failed")
254
+ mock_runner.return_value = mock_instance
255
+
256
+ result = self.runner.invoke(main, ["cfat", "assess"])
257
+
258
+ assert result.exit_code == 1
259
+ assert "Assessment failed" in result.output
260
+
261
+ def test_profile_and_region_options(self):
262
+ """Test AWS profile and region options."""
263
+ with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
264
+ mock_instance = MagicMock()
265
+ mock_instance.run_assessment.return_value = self.sample_report
266
+ mock_runner.return_value = mock_instance
267
+
268
+ result = self.runner.invoke(main, ["--profile", "test-profile", "--region", "eu-west-1", "cfat", "assess"])
269
+
270
+ assert result.exit_code == 0
271
+
272
+ # Verify runner was initialized with correct profile and region
273
+ mock_runner.assert_called_once()
274
+ call_args = mock_runner.call_args
275
+ assert call_args[1]["profile"] == "test-profile"
276
+ assert call_args[1]["region"] == "eu-west-1"
277
+
278
+ def test_debug_mode(self):
279
+ """Test debug mode activation."""
280
+ with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
281
+ mock_instance = MagicMock()
282
+ mock_instance.run_assessment.return_value = self.sample_report
283
+ mock_runner.return_value = mock_instance
284
+
285
+ result = self.runner.invoke(main, ["--debug", "cfat", "assess"])
286
+
287
+ assert result.exit_code == 0
288
+
289
+ def test_comprehensive_cli_workflow(self):
290
+ """Test comprehensive CLI workflow with multiple options."""
291
+ with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
292
+ with patch("runbooks.cfat.reporting.exporters.JiraExporter") as mock_jira:
293
+ mock_instance = MagicMock()
294
+ mock_instance.run_assessment.return_value = self.sample_report
295
+ mock_instance.assessment_config = MagicMock()
296
+ mock_runner.return_value = mock_instance
297
+
298
+ mock_jira_instance = MagicMock()
299
+ mock_jira.return_value = mock_jira_instance
300
+
301
+ with tempfile.TemporaryDirectory() as temp_dir:
302
+ jira_file = os.path.join(temp_dir, "jira_export.csv")
303
+
304
+ result = self.runner.invoke(
305
+ main,
306
+ [
307
+ "--profile",
308
+ "production",
309
+ "--region",
310
+ "us-west-2",
311
+ "--debug",
312
+ "cfat",
313
+ "assess",
314
+ "--output",
315
+ "all",
316
+ "--categories",
317
+ "iam",
318
+ "cloudtrail",
319
+ "--skip-checks",
320
+ "ec2_unused_instances",
321
+ "--severity",
322
+ "CRITICAL",
323
+ "--parallel",
324
+ "--max-workers",
325
+ "8",
326
+ "--compliance-framework",
327
+ "SOC2",
328
+ "--export-jira",
329
+ jira_file,
330
+ ],
331
+ )
332
+
333
+ assert result.exit_code == 0
334
+
335
+ # Verify all configurations were applied
336
+ assert mock_instance.assessment_config.included_categories == ["iam", "cloudtrail"]
337
+ assert mock_instance.assessment_config.excluded_checks == ["ec2_unused_instances"]
338
+ assert mock_instance.assessment_config.parallel_execution is True
339
+ assert mock_instance.assessment_config.max_workers == 8
340
+ assert mock_instance.assessment_config.compliance_framework == "SOC2"
@@ -0,0 +1,290 @@
1
+ """
2
+ Integration tests for Cloud Foundations Assessment Tool.
3
+
4
+ Tests the complete CFAT workflow using moto for AWS service mocking,
5
+ ensuring the assessment engine works correctly with AWS APIs without
6
+ requiring real AWS credentials or making actual API calls.
7
+
8
+ These tests focus on integration patterns and real-world usage scenarios
9
+ while maintaining fast execution and reliability.
10
+ """
11
+
12
+ from unittest.mock import MagicMock, patch
13
+
14
+ import boto3
15
+ import pytest
16
+ from moto import mock_aws
17
+
18
+ from runbooks.cfat.assessment.runner import CloudFoundationsAssessment
19
+ from runbooks.cfat.models import AssessmentConfig, CheckStatus, Severity
20
+ from runbooks.cfat.tests import TEST_ACCOUNT_ID, TEST_PROFILE, TEST_REGION
21
+
22
+
23
+ @pytest.mark.integration
24
+ class TestCFATIntegrationWithMoto:
25
+ """Integration tests using moto for AWS service mocking."""
26
+
27
+ @mock_aws
28
+ def test_iam_assessment_with_mock_services(self):
29
+ """Test IAM assessment using moto-mocked AWS services."""
30
+ # Create mock IAM resources
31
+ iam_client = boto3.client("iam", region_name=TEST_REGION)
32
+
33
+ # Create test user
34
+ iam_client.create_user(UserName="test-user")
35
+
36
+ # Create test role
37
+ assume_role_policy = {
38
+ "Version": "2012-10-17",
39
+ "Statement": [
40
+ {"Effect": "Allow", "Principal": {"Service": "ec2.amazonaws.com"}, "Action": "sts:AssumeRole"}
41
+ ],
42
+ }
43
+ iam_client.create_role(RoleName="test-role", AssumeRolePolicyDocument=str(assume_role_policy))
44
+
45
+ # Create test policy
46
+ policy_document = {
47
+ "Version": "2012-10-17",
48
+ "Statement": [{"Effect": "Allow", "Action": "s3:GetObject", "Resource": "*"}],
49
+ }
50
+ iam_client.create_policy(PolicyName="test-policy", PolicyDocument=str(policy_document))
51
+
52
+ # Mock the assessment runner to use our mocked services
53
+ with patch("runbooks.cfat.assessment.runner.CloudFoundationsAssessment.get_account_id") as mock_account:
54
+ mock_account.return_value = TEST_ACCOUNT_ID
55
+
56
+ # Initialize assessment
57
+ assessment = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
58
+
59
+ # Configure for IAM-only assessment
60
+ assessment.assessment_config.included_categories = ["iam"]
61
+ assessment.assessment_config.parallel_execution = False # Easier to debug
62
+
63
+ # Run assessment
64
+ report = assessment.run_assessment()
65
+
66
+ # Validate results
67
+ assert report is not None
68
+ assert report.account_id == TEST_ACCOUNT_ID
69
+ assert report.region == TEST_REGION
70
+ assert len(report.results) > 0
71
+
72
+ # Should have IAM-related results
73
+ iam_results = report.get_results_by_category("iam")
74
+ assert len(iam_results) > 0
75
+
76
+ # Verify result structure
77
+ for result in iam_results:
78
+ assert result.finding_id is not None
79
+ assert result.check_name is not None
80
+ assert result.check_category == "iam"
81
+ assert result.status in [CheckStatus.PASS, CheckStatus.FAIL, CheckStatus.ERROR]
82
+ assert result.severity in [Severity.INFO, Severity.WARNING, Severity.CRITICAL]
83
+ assert result.execution_time >= 0
84
+
85
+ @mock_aws
86
+ def test_vpc_assessment_with_mock_services(self):
87
+ """Test VPC assessment using moto-mocked EC2 services."""
88
+ # Create mock VPC resources
89
+ ec2_client = boto3.client("ec2", region_name=TEST_REGION)
90
+
91
+ # Create VPC
92
+ vpc_response = ec2_client.create_vpc(CidrBlock="10.0.0.0/16")
93
+ vpc_id = vpc_response["Vpc"]["VpcId"]
94
+
95
+ # Create subnet
96
+ ec2_client.create_subnet(VpcId=vpc_id, CidrBlock="10.0.1.0/24")
97
+
98
+ # Create security group
99
+ ec2_client.create_security_group(GroupName="test-sg", Description="Test security group", VpcId=vpc_id)
100
+
101
+ # Mock assessment execution
102
+ with patch("runbooks.cfat.assessment.runner.CloudFoundationsAssessment.get_account_id") as mock_account:
103
+ mock_account.return_value = TEST_ACCOUNT_ID
104
+
105
+ assessment = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
106
+
107
+ # Configure for VPC-only assessment
108
+ assessment.assessment_config.included_categories = ["vpc"]
109
+ assessment.assessment_config.parallel_execution = False
110
+
111
+ report = assessment.run_assessment()
112
+
113
+ # Validate VPC assessment results
114
+ assert report is not None
115
+ vpc_results = report.get_results_by_category("vpc")
116
+
117
+ # Should have some VPC-related checks
118
+ assert len(vpc_results) >= 0 # May be 0 if no VPC checks implemented yet
119
+
120
+ @mock_aws
121
+ def test_cloudtrail_assessment_with_mock_services(self):
122
+ """Test CloudTrail assessment using moto-mocked services."""
123
+ # Create mock CloudTrail
124
+ cloudtrail_client = boto3.client("cloudtrail", region_name=TEST_REGION)
125
+
126
+ # Create trail
127
+ trail_name = "test-trail"
128
+ s3_bucket = "test-cloudtrail-bucket"
129
+
130
+ cloudtrail_client.create_trail(Name=trail_name, S3BucketName=s3_bucket)
131
+
132
+ # Start logging
133
+ cloudtrail_client.start_logging(Name=trail_name)
134
+
135
+ # Mock assessment
136
+ with patch("runbooks.cfat.assessment.runner.CloudFoundationsAssessment.get_account_id") as mock_account:
137
+ mock_account.return_value = TEST_ACCOUNT_ID
138
+
139
+ assessment = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
140
+
141
+ assessment.assessment_config.included_categories = ["cloudtrail"]
142
+ assessment.assessment_config.parallel_execution = False
143
+
144
+ report = assessment.run_assessment()
145
+
146
+ assert report is not None
147
+ cloudtrail_results = report.get_results_by_category("cloudtrail")
148
+ assert len(cloudtrail_results) >= 0
149
+
150
+ def test_assessment_configuration_integration(self):
151
+ """Test assessment configuration integration."""
152
+ # Test custom configuration
153
+ config = AssessmentConfig(
154
+ included_categories=["iam", "vpc"],
155
+ excluded_checks=["iam_unused_credentials"],
156
+ parallel_execution=True,
157
+ max_workers=5,
158
+ severity_threshold=Severity.WARNING,
159
+ compliance_framework="SOC2",
160
+ )
161
+
162
+ with patch("runbooks.cfat.assessment.runner.CloudFoundationsAssessment.get_account_id") as mock_account:
163
+ mock_account.return_value = TEST_ACCOUNT_ID
164
+
165
+ assessment = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
166
+ assessment.assessment_config = config
167
+
168
+ # Should be able to run without errors
169
+ report = assessment.run_assessment()
170
+ assert report is not None
171
+ assert report.metadata.get("max_workers") == 5
172
+
173
+ def test_parallel_vs_sequential_execution(self):
174
+ """Test parallel vs sequential execution modes."""
175
+ with patch("runbooks.cfat.assessment.runner.CloudFoundationsAssessment.get_account_id") as mock_account:
176
+ mock_account.return_value = TEST_ACCOUNT_ID
177
+
178
+ # Test parallel execution
179
+ assessment_parallel = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
180
+ assessment_parallel.assessment_config.parallel_execution = True
181
+ assessment_parallel.assessment_config.max_workers = 3
182
+
183
+ report_parallel = assessment_parallel.run_assessment()
184
+
185
+ # Test sequential execution
186
+ assessment_sequential = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
187
+ assessment_sequential.assessment_config.parallel_execution = False
188
+
189
+ report_sequential = assessment_sequential.run_assessment()
190
+
191
+ # Both should produce valid reports
192
+ assert report_parallel is not None
193
+ assert report_sequential is not None
194
+
195
+ # Results should be similar (may vary due to mock timing)
196
+ assert len(report_parallel.results) > 0
197
+ assert len(report_sequential.results) > 0
198
+
199
+ def test_error_handling_in_assessment(self):
200
+ """Test error handling during assessment execution."""
201
+ with patch("runbooks.cfat.assessment.runner.CloudFoundationsAssessment.get_account_id") as mock_account:
202
+ mock_account.return_value = TEST_ACCOUNT_ID
203
+
204
+ assessment = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
205
+
206
+ # Mock a check that raises an exception
207
+ original_execute_single_check = assessment._execute_single_check
208
+
209
+ def mock_execute_single_check(check_name):
210
+ if check_name == "failing_check":
211
+ raise Exception("Simulated check failure")
212
+ return original_execute_single_check(check_name)
213
+
214
+ assessment._execute_single_check = mock_execute_single_check
215
+
216
+ # Override available checks to include our failing check
217
+ assessment._available_checks = {"passing_check": "PassingCheck", "failing_check": "FailingCheck"}
218
+
219
+ # Run assessment
220
+ report = assessment.run_assessment()
221
+
222
+ # Should handle errors gracefully
223
+ assert report is not None
224
+
225
+ # Should have error results for failed checks
226
+ error_results = [r for r in report.results if r.status == CheckStatus.ERROR]
227
+ assert len(error_results) > 0
228
+
229
+ def test_report_generation_all_formats(self):
230
+ """Test report generation in all supported formats."""
231
+ with patch("runbooks.cfat.assessment.runner.CloudFoundationsAssessment.get_account_id") as mock_account:
232
+ mock_account.return_value = TEST_ACCOUNT_ID
233
+
234
+ assessment = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
235
+
236
+ # Run assessment
237
+ report = assessment.run_assessment()
238
+
239
+ # Test all export formats (without actually writing files)
240
+ assert hasattr(report, "to_json")
241
+ assert hasattr(report, "to_csv")
242
+ assert hasattr(report, "to_html")
243
+ assert hasattr(report, "to_markdown")
244
+
245
+ # Test methods return without error
246
+ assert callable(report.to_json)
247
+ assert callable(report.to_csv)
248
+ assert callable(report.to_html)
249
+ assert callable(report.to_markdown)
250
+
251
+ def test_compliance_framework_integration(self):
252
+ """Test compliance framework-specific assessments."""
253
+ frameworks = ["SOC2", "PCI-DSS", "HIPAA"]
254
+
255
+ for framework in frameworks:
256
+ with patch("runbooks.cfat.assessment.runner.CloudFoundationsAssessment.get_account_id") as mock_account:
257
+ mock_account.return_value = TEST_ACCOUNT_ID
258
+
259
+ assessment = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
260
+ assessment.assessment_config.compliance_framework = framework
261
+
262
+ report = assessment.run_assessment()
263
+
264
+ assert report is not None
265
+ assert (
266
+ report.metadata.get("compliance_framework") is None
267
+ or report.metadata.get("compliance_framework") == framework
268
+ )
269
+
270
+ @pytest.mark.slow
271
+ def test_large_scale_assessment(self):
272
+ """Test assessment with many checks (performance test)."""
273
+ with patch("runbooks.cfat.assessment.runner.CloudFoundationsAssessment.get_account_id") as mock_account:
274
+ mock_account.return_value = TEST_ACCOUNT_ID
275
+
276
+ assessment = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
277
+
278
+ # Override with many mock checks
279
+ many_checks = {f"check_{i}": f"Check{i}" for i in range(50)}
280
+ assessment._available_checks = many_checks
281
+
282
+ assessment.assessment_config.parallel_execution = True
283
+ assessment.assessment_config.max_workers = 10
284
+
285
+ report = assessment.run_assessment()
286
+
287
+ assert report is not None
288
+ # Should complete in reasonable time with parallel execution
289
+ assert report.summary.total_execution_time < 60 # Should finish within 60 seconds
290
+ assert len(report.results) == 50