runbooks 0.2.3__py3-none-any.whl → 0.6.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- conftest.py +26 -0
- jupyter-agent/.env.template +2 -0
- jupyter-agent/.gitattributes +35 -0
- jupyter-agent/README.md +16 -0
- jupyter-agent/app.py +256 -0
- jupyter-agent/cloudops-agent.png +0 -0
- jupyter-agent/ds-system-prompt.txt +154 -0
- jupyter-agent/jupyter-agent.png +0 -0
- jupyter-agent/llama3_template.jinja +123 -0
- jupyter-agent/requirements.txt +9 -0
- jupyter-agent/utils.py +409 -0
- runbooks/__init__.py +71 -3
- runbooks/__main__.py +13 -0
- runbooks/aws/ec2_describe_instances.py +1 -1
- runbooks/aws/ec2_run_instances.py +8 -2
- runbooks/aws/ec2_start_stop_instances.py +17 -4
- runbooks/aws/ec2_unused_volumes.py +5 -1
- runbooks/aws/s3_create_bucket.py +4 -2
- runbooks/aws/s3_list_objects.py +6 -1
- runbooks/aws/tagging_lambda_handler.py +13 -2
- runbooks/aws/tags.json +12 -0
- runbooks/base.py +353 -0
- runbooks/cfat/README.md +49 -0
- runbooks/cfat/__init__.py +74 -0
- runbooks/cfat/app.ts +644 -0
- runbooks/cfat/assessment/__init__.py +40 -0
- runbooks/cfat/assessment/asana-import.csv +39 -0
- runbooks/cfat/assessment/cfat-checks.csv +31 -0
- runbooks/cfat/assessment/cfat.txt +520 -0
- runbooks/cfat/assessment/collectors.py +200 -0
- runbooks/cfat/assessment/jira-import.csv +39 -0
- runbooks/cfat/assessment/runner.py +387 -0
- runbooks/cfat/assessment/validators.py +290 -0
- runbooks/cfat/cli.py +103 -0
- runbooks/cfat/docs/asana-import.csv +24 -0
- runbooks/cfat/docs/cfat-checks.csv +31 -0
- runbooks/cfat/docs/cfat.txt +335 -0
- runbooks/cfat/docs/checks-output.png +0 -0
- runbooks/cfat/docs/cloudshell-console-run.png +0 -0
- runbooks/cfat/docs/cloudshell-download.png +0 -0
- runbooks/cfat/docs/cloudshell-output.png +0 -0
- runbooks/cfat/docs/downloadfile.png +0 -0
- runbooks/cfat/docs/jira-import.csv +24 -0
- runbooks/cfat/docs/open-cloudshell.png +0 -0
- runbooks/cfat/docs/report-header.png +0 -0
- runbooks/cfat/models.py +1026 -0
- runbooks/cfat/package-lock.json +5116 -0
- runbooks/cfat/package.json +38 -0
- runbooks/cfat/report.py +496 -0
- runbooks/cfat/reporting/__init__.py +46 -0
- runbooks/cfat/reporting/exporters.py +337 -0
- runbooks/cfat/reporting/formatters.py +496 -0
- runbooks/cfat/reporting/templates.py +135 -0
- runbooks/cfat/run-assessment.sh +23 -0
- runbooks/cfat/runner.py +69 -0
- runbooks/cfat/src/actions/check-cloudtrail-existence.ts +43 -0
- runbooks/cfat/src/actions/check-config-existence.ts +37 -0
- runbooks/cfat/src/actions/check-control-tower.ts +37 -0
- runbooks/cfat/src/actions/check-ec2-existence.ts +46 -0
- runbooks/cfat/src/actions/check-iam-users.ts +50 -0
- runbooks/cfat/src/actions/check-legacy-cur.ts +30 -0
- runbooks/cfat/src/actions/check-org-cloudformation.ts +30 -0
- runbooks/cfat/src/actions/check-vpc-existence.ts +43 -0
- runbooks/cfat/src/actions/create-asanaimport.ts +14 -0
- runbooks/cfat/src/actions/create-backlog.ts +372 -0
- runbooks/cfat/src/actions/create-jiraimport.ts +15 -0
- runbooks/cfat/src/actions/create-report.ts +616 -0
- runbooks/cfat/src/actions/define-account-type.ts +51 -0
- runbooks/cfat/src/actions/get-enabled-org-policy-types.ts +40 -0
- runbooks/cfat/src/actions/get-enabled-org-services.ts +26 -0
- runbooks/cfat/src/actions/get-idc-info.ts +34 -0
- runbooks/cfat/src/actions/get-org-da-accounts.ts +34 -0
- runbooks/cfat/src/actions/get-org-details.ts +35 -0
- runbooks/cfat/src/actions/get-org-member-accounts.ts +44 -0
- runbooks/cfat/src/actions/get-org-ous.ts +35 -0
- runbooks/cfat/src/actions/get-regions.ts +22 -0
- runbooks/cfat/src/actions/zip-assessment.ts +27 -0
- runbooks/cfat/src/types/index.d.ts +147 -0
- runbooks/cfat/tests/__init__.py +141 -0
- runbooks/cfat/tests/test_cli.py +340 -0
- runbooks/cfat/tests/test_integration.py +290 -0
- runbooks/cfat/tests/test_models.py +505 -0
- runbooks/cfat/tests/test_reporting.py +354 -0
- runbooks/cfat/tsconfig.json +16 -0
- runbooks/cfat/webpack.config.cjs +27 -0
- runbooks/config.py +260 -0
- runbooks/finops/__init__.py +88 -0
- runbooks/finops/aws_client.py +245 -0
- runbooks/finops/cli.py +151 -0
- runbooks/finops/cost_processor.py +410 -0
- runbooks/finops/dashboard_runner.py +448 -0
- runbooks/finops/helpers.py +355 -0
- runbooks/finops/main.py +14 -0
- runbooks/finops/profile_processor.py +174 -0
- runbooks/finops/types.py +66 -0
- runbooks/finops/visualisations.py +80 -0
- runbooks/inventory/.gitignore +354 -0
- runbooks/inventory/ArgumentsClass.py +261 -0
- runbooks/inventory/Inventory_Modules.py +6130 -0
- runbooks/inventory/LandingZone/delete_lz.py +1075 -0
- runbooks/inventory/README.md +1320 -0
- runbooks/inventory/__init__.py +62 -0
- runbooks/inventory/account_class.py +532 -0
- runbooks/inventory/all_my_instances_wrapper.py +123 -0
- runbooks/inventory/aws_decorators.py +201 -0
- runbooks/inventory/cfn_move_stack_instances.py +1526 -0
- runbooks/inventory/check_cloudtrail_compliance.py +614 -0
- runbooks/inventory/check_controltower_readiness.py +1107 -0
- runbooks/inventory/check_landingzone_readiness.py +711 -0
- runbooks/inventory/cloudtrail.md +727 -0
- runbooks/inventory/collectors/__init__.py +20 -0
- runbooks/inventory/collectors/aws_compute.py +518 -0
- runbooks/inventory/collectors/aws_networking.py +275 -0
- runbooks/inventory/collectors/base.py +222 -0
- runbooks/inventory/core/__init__.py +19 -0
- runbooks/inventory/core/collector.py +303 -0
- runbooks/inventory/core/formatter.py +296 -0
- runbooks/inventory/delete_s3_buckets_objects.py +169 -0
- runbooks/inventory/discovery.md +81 -0
- runbooks/inventory/draw_org_structure.py +748 -0
- runbooks/inventory/ec2_vpc_utils.py +341 -0
- runbooks/inventory/find_cfn_drift_detection.py +272 -0
- runbooks/inventory/find_cfn_orphaned_stacks.py +719 -0
- runbooks/inventory/find_cfn_stackset_drift.py +733 -0
- runbooks/inventory/find_ec2_security_groups.py +669 -0
- runbooks/inventory/find_landingzone_versions.py +201 -0
- runbooks/inventory/find_vpc_flow_logs.py +1221 -0
- runbooks/inventory/inventory.sh +659 -0
- runbooks/inventory/list_cfn_stacks.py +558 -0
- runbooks/inventory/list_cfn_stackset_operation_results.py +252 -0
- runbooks/inventory/list_cfn_stackset_operations.py +734 -0
- runbooks/inventory/list_cfn_stacksets.py +453 -0
- runbooks/inventory/list_config_recorders_delivery_channels.py +681 -0
- runbooks/inventory/list_ds_directories.py +354 -0
- runbooks/inventory/list_ec2_availability_zones.py +286 -0
- runbooks/inventory/list_ec2_ebs_volumes.py +244 -0
- runbooks/inventory/list_ec2_instances.py +425 -0
- runbooks/inventory/list_ecs_clusters_and_tasks.py +562 -0
- runbooks/inventory/list_elbs_load_balancers.py +411 -0
- runbooks/inventory/list_enis_network_interfaces.py +526 -0
- runbooks/inventory/list_guardduty_detectors.py +568 -0
- runbooks/inventory/list_iam_policies.py +404 -0
- runbooks/inventory/list_iam_roles.py +518 -0
- runbooks/inventory/list_iam_saml_providers.py +359 -0
- runbooks/inventory/list_lambda_functions.py +882 -0
- runbooks/inventory/list_org_accounts.py +446 -0
- runbooks/inventory/list_org_accounts_users.py +354 -0
- runbooks/inventory/list_rds_db_instances.py +406 -0
- runbooks/inventory/list_route53_hosted_zones.py +318 -0
- runbooks/inventory/list_servicecatalog_provisioned_products.py +575 -0
- runbooks/inventory/list_sns_topics.py +360 -0
- runbooks/inventory/list_ssm_parameters.py +402 -0
- runbooks/inventory/list_vpc_subnets.py +433 -0
- runbooks/inventory/list_vpcs.py +422 -0
- runbooks/inventory/lockdown_cfn_stackset_role.py +224 -0
- runbooks/inventory/models/__init__.py +24 -0
- runbooks/inventory/models/account.py +192 -0
- runbooks/inventory/models/inventory.py +309 -0
- runbooks/inventory/models/resource.py +247 -0
- runbooks/inventory/recover_cfn_stack_ids.py +205 -0
- runbooks/inventory/requirements.txt +12 -0
- runbooks/inventory/run_on_multi_accounts.py +211 -0
- runbooks/inventory/tests/common_test_data.py +3661 -0
- runbooks/inventory/tests/common_test_functions.py +204 -0
- runbooks/inventory/tests/script_test_data.py +0 -0
- runbooks/inventory/tests/setup.py +24 -0
- runbooks/inventory/tests/src.py +18 -0
- runbooks/inventory/tests/test_cfn_describe_stacks.py +208 -0
- runbooks/inventory/tests/test_ec2_describe_instances.py +162 -0
- runbooks/inventory/tests/test_inventory_modules.py +55 -0
- runbooks/inventory/tests/test_lambda_list_functions.py +86 -0
- runbooks/inventory/tests/test_moto_integration_example.py +273 -0
- runbooks/inventory/tests/test_org_list_accounts.py +49 -0
- runbooks/inventory/update_aws_actions.py +173 -0
- runbooks/inventory/update_cfn_stacksets.py +1215 -0
- runbooks/inventory/update_cloudwatch_logs_retention_policy.py +294 -0
- runbooks/inventory/update_iam_roles_cross_accounts.py +478 -0
- runbooks/inventory/update_s3_public_access_block.py +539 -0
- runbooks/inventory/utils/__init__.py +23 -0
- runbooks/inventory/utils/aws_helpers.py +510 -0
- runbooks/inventory/utils/threading_utils.py +493 -0
- runbooks/inventory/utils/validation.py +682 -0
- runbooks/inventory/verify_ec2_security_groups.py +1430 -0
- runbooks/main.py +785 -0
- runbooks/organizations/__init__.py +12 -0
- runbooks/organizations/manager.py +374 -0
- runbooks/security_baseline/README.md +324 -0
- runbooks/security_baseline/checklist/alternate_contacts.py +8 -1
- runbooks/security_baseline/checklist/bucket_public_access.py +4 -1
- runbooks/security_baseline/checklist/cloudwatch_alarm_configuration.py +9 -2
- runbooks/security_baseline/checklist/guardduty_enabled.py +9 -2
- runbooks/security_baseline/checklist/multi_region_instance_usage.py +5 -1
- runbooks/security_baseline/checklist/root_access_key.py +6 -1
- runbooks/security_baseline/config-origin.json +1 -1
- runbooks/security_baseline/config.json +1 -1
- runbooks/security_baseline/permission.json +1 -1
- runbooks/security_baseline/report_generator.py +10 -2
- runbooks/security_baseline/report_template_en.html +8 -8
- runbooks/security_baseline/report_template_jp.html +8 -8
- runbooks/security_baseline/report_template_kr.html +13 -13
- runbooks/security_baseline/report_template_vn.html +8 -8
- runbooks/security_baseline/requirements.txt +7 -0
- runbooks/security_baseline/run_script.py +8 -2
- runbooks/security_baseline/security_baseline_tester.py +10 -2
- runbooks/security_baseline/utils/common.py +5 -1
- runbooks/utils/__init__.py +204 -0
- runbooks-0.6.1.dist-info/METADATA +373 -0
- runbooks-0.6.1.dist-info/RECORD +237 -0
- {runbooks-0.2.3.dist-info → runbooks-0.6.1.dist-info}/WHEEL +1 -1
- runbooks-0.6.1.dist-info/entry_points.txt +7 -0
- runbooks-0.6.1.dist-info/licenses/LICENSE +201 -0
- runbooks-0.6.1.dist-info/top_level.txt +3 -0
- runbooks/python101/calculator.py +0 -34
- runbooks/python101/config.py +0 -1
- runbooks/python101/exceptions.py +0 -16
- runbooks/python101/file_manager.py +0 -218
- runbooks/python101/toolkit.py +0 -153
- runbooks-0.2.3.dist-info/METADATA +0 -435
- runbooks-0.2.3.dist-info/RECORD +0 -61
- runbooks-0.2.3.dist-info/entry_points.txt +0 -3
- runbooks-0.2.3.dist-info/top_level.txt +0 -1
@@ -0,0 +1,340 @@
|
|
1
|
+
"""
|
2
|
+
CLI tests for Cloud Foundations Assessment Tool.
|
3
|
+
|
4
|
+
Tests command-line interface argument parsing, validation, and integration
|
5
|
+
with the assessment engine. These tests focus on CLI behavior separately
|
6
|
+
from AWS API interactions.
|
7
|
+
"""
|
8
|
+
|
9
|
+
import os
|
10
|
+
import tempfile
|
11
|
+
from unittest.mock import MagicMock, patch
|
12
|
+
|
13
|
+
import pytest
|
14
|
+
from click.testing import CliRunner
|
15
|
+
|
16
|
+
from runbooks.cfat.tests import create_sample_assessment_report
|
17
|
+
from runbooks.main import assess, cfat, main
|
18
|
+
|
19
|
+
|
20
|
+
@pytest.mark.cli
|
21
|
+
class TestCFATCLI:
|
22
|
+
"""Test CFAT CLI functionality."""
|
23
|
+
|
24
|
+
def setup_method(self):
|
25
|
+
"""Set up test environment."""
|
26
|
+
self.runner = CliRunner()
|
27
|
+
self.sample_report = create_sample_assessment_report()
|
28
|
+
|
29
|
+
def test_cfat_help_command(self):
|
30
|
+
"""Test CFAT help command."""
|
31
|
+
result = self.runner.invoke(main, ["cfat", "--help"])
|
32
|
+
|
33
|
+
assert result.exit_code == 0
|
34
|
+
assert "Cloud Foundations Assessment Tool" in result.output
|
35
|
+
assert "assess" in result.output
|
36
|
+
|
37
|
+
def test_assess_help_command(self):
|
38
|
+
"""Test assess command help."""
|
39
|
+
result = self.runner.invoke(main, ["cfat", "assess", "--help"])
|
40
|
+
|
41
|
+
assert result.exit_code == 0
|
42
|
+
assert "Run enhanced Cloud Foundations assessment" in result.output
|
43
|
+
assert "--output" in result.output
|
44
|
+
assert "--severity" in result.output
|
45
|
+
assert "--parallel" in result.output
|
46
|
+
assert "--export-jira" in result.output
|
47
|
+
|
48
|
+
def test_assess_basic_arguments(self):
|
49
|
+
"""Test basic assess command arguments."""
|
50
|
+
with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
|
51
|
+
mock_instance = MagicMock()
|
52
|
+
mock_instance.run_assessment.return_value = self.sample_report
|
53
|
+
mock_runner.return_value = mock_instance
|
54
|
+
|
55
|
+
result = self.runner.invoke(main, ["cfat", "assess", "--output", "console", "--severity", "WARNING"])
|
56
|
+
|
57
|
+
# Should complete successfully
|
58
|
+
assert result.exit_code == 0
|
59
|
+
|
60
|
+
# Verify runner was initialized and configured
|
61
|
+
mock_runner.assert_called_once()
|
62
|
+
mock_instance.set_min_severity.assert_called_with("WARNING")
|
63
|
+
mock_instance.run_assessment.assert_called_once()
|
64
|
+
|
65
|
+
def test_assess_output_formats(self):
|
66
|
+
"""Test different output formats."""
|
67
|
+
formats = ["console", "html", "csv", "json", "markdown"]
|
68
|
+
|
69
|
+
for format_type in formats:
|
70
|
+
with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
|
71
|
+
mock_instance = MagicMock()
|
72
|
+
mock_instance.run_assessment.return_value = self.sample_report
|
73
|
+
mock_runner.return_value = mock_instance
|
74
|
+
|
75
|
+
result = self.runner.invoke(main, ["cfat", "assess", "--output", format_type])
|
76
|
+
|
77
|
+
assert result.exit_code == 0, f"Failed for format: {format_type}"
|
78
|
+
|
79
|
+
def test_assess_all_output_format(self):
|
80
|
+
"""Test 'all' output format generates multiple files."""
|
81
|
+
with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
|
82
|
+
mock_instance = MagicMock()
|
83
|
+
mock_instance.run_assessment.return_value = self.sample_report
|
84
|
+
mock_runner.return_value = mock_instance
|
85
|
+
|
86
|
+
with tempfile.TemporaryDirectory() as temp_dir:
|
87
|
+
original_cwd = os.getcwd()
|
88
|
+
try:
|
89
|
+
os.chdir(temp_dir)
|
90
|
+
|
91
|
+
result = self.runner.invoke(main, ["cfat", "assess", "--output", "all"])
|
92
|
+
|
93
|
+
assert result.exit_code == 0
|
94
|
+
assert "Generated files:" in result.output
|
95
|
+
|
96
|
+
finally:
|
97
|
+
os.chdir(original_cwd)
|
98
|
+
|
99
|
+
def test_assess_specific_checks(self):
|
100
|
+
"""Test specifying specific checks to run."""
|
101
|
+
with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
|
102
|
+
mock_instance = MagicMock()
|
103
|
+
mock_instance.run_assessment.return_value = self.sample_report
|
104
|
+
mock_runner.return_value = mock_instance
|
105
|
+
|
106
|
+
result = self.runner.invoke(
|
107
|
+
main, ["cfat", "assess", "--checks", "iam_root_mfa", "--checks", "cloudtrail_enabled"]
|
108
|
+
)
|
109
|
+
|
110
|
+
assert result.exit_code == 0
|
111
|
+
mock_instance.set_checks.assert_called_with(["iam_root_mfa", "cloudtrail_enabled"])
|
112
|
+
|
113
|
+
def test_assess_skip_checks(self):
|
114
|
+
"""Test skipping specific checks."""
|
115
|
+
with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
|
116
|
+
mock_instance = MagicMock()
|
117
|
+
mock_instance.run_assessment.return_value = self.sample_report
|
118
|
+
mock_runner.return_value = mock_instance
|
119
|
+
|
120
|
+
result = self.runner.invoke(
|
121
|
+
main, ["cfat", "assess", "--skip-checks", "ec2_instances", "--skip-checks", "rds_instances"]
|
122
|
+
)
|
123
|
+
|
124
|
+
assert result.exit_code == 0
|
125
|
+
mock_instance.skip_checks.assert_called_with(["ec2_instances", "rds_instances"])
|
126
|
+
|
127
|
+
def test_assess_categories(self):
|
128
|
+
"""Test specifying assessment categories."""
|
129
|
+
with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
|
130
|
+
mock_instance = MagicMock()
|
131
|
+
mock_instance.run_assessment.return_value = self.sample_report
|
132
|
+
mock_instance.assessment_config = MagicMock()
|
133
|
+
mock_runner.return_value = mock_instance
|
134
|
+
|
135
|
+
result = self.runner.invoke(
|
136
|
+
main, ["cfat", "assess", "--categories", "iam", "--categories", "vpc", "--skip-categories", "ec2"]
|
137
|
+
)
|
138
|
+
|
139
|
+
assert result.exit_code == 0
|
140
|
+
|
141
|
+
# Verify categories were set
|
142
|
+
assert mock_instance.assessment_config.included_categories == ["iam", "vpc"]
|
143
|
+
assert mock_instance.assessment_config.excluded_categories == ["ec2"]
|
144
|
+
|
145
|
+
def test_assess_parallel_execution_options(self):
|
146
|
+
"""Test parallel execution configuration."""
|
147
|
+
with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
|
148
|
+
mock_instance = MagicMock()
|
149
|
+
mock_instance.run_assessment.return_value = self.sample_report
|
150
|
+
mock_instance.assessment_config = MagicMock()
|
151
|
+
mock_runner.return_value = mock_instance
|
152
|
+
|
153
|
+
# Test parallel execution
|
154
|
+
result = self.runner.invoke(main, ["cfat", "assess", "--parallel", "--max-workers", "5"])
|
155
|
+
|
156
|
+
assert result.exit_code == 0
|
157
|
+
assert mock_instance.assessment_config.parallel_execution is True
|
158
|
+
assert mock_instance.assessment_config.max_workers == 5
|
159
|
+
|
160
|
+
# Test sequential execution
|
161
|
+
result = self.runner.invoke(main, ["cfat", "assess", "--sequential"])
|
162
|
+
|
163
|
+
assert result.exit_code == 0
|
164
|
+
assert mock_instance.assessment_config.parallel_execution is False
|
165
|
+
|
166
|
+
def test_assess_compliance_framework(self):
|
167
|
+
"""Test compliance framework specification."""
|
168
|
+
with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
|
169
|
+
mock_instance = MagicMock()
|
170
|
+
mock_instance.run_assessment.return_value = self.sample_report
|
171
|
+
mock_instance.assessment_config = MagicMock()
|
172
|
+
mock_runner.return_value = mock_instance
|
173
|
+
|
174
|
+
result = self.runner.invoke(main, ["cfat", "assess", "--compliance-framework", "SOC2"])
|
175
|
+
|
176
|
+
assert result.exit_code == 0
|
177
|
+
assert mock_instance.assessment_config.compliance_framework == "SOC2"
|
178
|
+
|
179
|
+
def test_assess_export_integrations(self):
|
180
|
+
"""Test export to project management tools."""
|
181
|
+
with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
|
182
|
+
with patch("runbooks.cfat.reporting.exporters.JiraExporter") as mock_jira:
|
183
|
+
with patch("runbooks.cfat.reporting.exporters.AsanaExporter") as mock_asana:
|
184
|
+
mock_instance = MagicMock()
|
185
|
+
mock_instance.run_assessment.return_value = self.sample_report
|
186
|
+
mock_runner.return_value = mock_instance
|
187
|
+
|
188
|
+
mock_jira_instance = MagicMock()
|
189
|
+
mock_asana_instance = MagicMock()
|
190
|
+
mock_jira.return_value = mock_jira_instance
|
191
|
+
mock_asana.return_value = mock_asana_instance
|
192
|
+
|
193
|
+
with tempfile.TemporaryDirectory() as temp_dir:
|
194
|
+
jira_file = os.path.join(temp_dir, "jira.csv")
|
195
|
+
asana_file = os.path.join(temp_dir, "asana.csv")
|
196
|
+
|
197
|
+
result = self.runner.invoke(
|
198
|
+
main, ["cfat", "assess", "--export-jira", jira_file, "--export-asana", asana_file]
|
199
|
+
)
|
200
|
+
|
201
|
+
assert result.exit_code == 0
|
202
|
+
|
203
|
+
# Verify exporters were called
|
204
|
+
mock_jira_instance.export.assert_called_once_with(self.sample_report, jira_file)
|
205
|
+
mock_asana_instance.export.assert_called_once_with(self.sample_report, asana_file)
|
206
|
+
|
207
|
+
def test_assess_web_server_option(self):
|
208
|
+
"""Test web server option (without actually starting server)."""
|
209
|
+
with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
|
210
|
+
with patch("runbooks.main.start_web_server") as mock_web_server:
|
211
|
+
mock_instance = MagicMock()
|
212
|
+
mock_instance.run_assessment.return_value = self.sample_report
|
213
|
+
mock_runner.return_value = mock_instance
|
214
|
+
|
215
|
+
result = self.runner.invoke(main, ["cfat", "assess", "--serve-web", "--web-port", "9000"])
|
216
|
+
|
217
|
+
assert result.exit_code == 0
|
218
|
+
mock_web_server.assert_called_once_with(self.sample_report, 9000)
|
219
|
+
|
220
|
+
def test_assess_output_file_specification(self):
|
221
|
+
"""Test specifying custom output file."""
|
222
|
+
with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
|
223
|
+
mock_instance = MagicMock()
|
224
|
+
mock_instance.run_assessment.return_value = self.sample_report
|
225
|
+
mock_runner.return_value = mock_instance
|
226
|
+
|
227
|
+
with tempfile.TemporaryDirectory() as temp_dir:
|
228
|
+
output_file = os.path.join(temp_dir, "custom_report.html")
|
229
|
+
|
230
|
+
result = self.runner.invoke(main, ["cfat", "assess", "--output", "html", "--output-file", output_file])
|
231
|
+
|
232
|
+
assert result.exit_code == 0
|
233
|
+
|
234
|
+
def test_assess_invalid_arguments(self):
|
235
|
+
"""Test handling of invalid arguments."""
|
236
|
+
# Test invalid output format
|
237
|
+
result = self.runner.invoke(main, ["cfat", "assess", "--output", "invalid_format"])
|
238
|
+
assert result.exit_code != 0
|
239
|
+
|
240
|
+
# Test invalid severity
|
241
|
+
result = self.runner.invoke(main, ["cfat", "assess", "--severity", "INVALID"])
|
242
|
+
assert result.exit_code != 0
|
243
|
+
|
244
|
+
# Test invalid max-workers
|
245
|
+
result = self.runner.invoke(main, ["cfat", "assess", "--max-workers", "0"])
|
246
|
+
# This should succeed at CLI level but may fail in assessment config validation
|
247
|
+
assert result.exit_code in [0, 1] # Either CLI rejects or assessment validates
|
248
|
+
|
249
|
+
def test_assess_error_handling(self):
|
250
|
+
"""Test CLI error handling when assessment fails."""
|
251
|
+
with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
|
252
|
+
mock_instance = MagicMock()
|
253
|
+
mock_instance.run_assessment.side_effect = Exception("Assessment failed")
|
254
|
+
mock_runner.return_value = mock_instance
|
255
|
+
|
256
|
+
result = self.runner.invoke(main, ["cfat", "assess"])
|
257
|
+
|
258
|
+
assert result.exit_code == 1
|
259
|
+
assert "Assessment failed" in result.output
|
260
|
+
|
261
|
+
def test_profile_and_region_options(self):
|
262
|
+
"""Test AWS profile and region options."""
|
263
|
+
with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
|
264
|
+
mock_instance = MagicMock()
|
265
|
+
mock_instance.run_assessment.return_value = self.sample_report
|
266
|
+
mock_runner.return_value = mock_instance
|
267
|
+
|
268
|
+
result = self.runner.invoke(main, ["--profile", "test-profile", "--region", "eu-west-1", "cfat", "assess"])
|
269
|
+
|
270
|
+
assert result.exit_code == 0
|
271
|
+
|
272
|
+
# Verify runner was initialized with correct profile and region
|
273
|
+
mock_runner.assert_called_once()
|
274
|
+
call_args = mock_runner.call_args
|
275
|
+
assert call_args[1]["profile"] == "test-profile"
|
276
|
+
assert call_args[1]["region"] == "eu-west-1"
|
277
|
+
|
278
|
+
def test_debug_mode(self):
|
279
|
+
"""Test debug mode activation."""
|
280
|
+
with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
|
281
|
+
mock_instance = MagicMock()
|
282
|
+
mock_instance.run_assessment.return_value = self.sample_report
|
283
|
+
mock_runner.return_value = mock_instance
|
284
|
+
|
285
|
+
result = self.runner.invoke(main, ["--debug", "cfat", "assess"])
|
286
|
+
|
287
|
+
assert result.exit_code == 0
|
288
|
+
|
289
|
+
def test_comprehensive_cli_workflow(self):
|
290
|
+
"""Test comprehensive CLI workflow with multiple options."""
|
291
|
+
with patch("runbooks.cfat.runner.AssessmentRunner") as mock_runner:
|
292
|
+
with patch("runbooks.cfat.reporting.exporters.JiraExporter") as mock_jira:
|
293
|
+
mock_instance = MagicMock()
|
294
|
+
mock_instance.run_assessment.return_value = self.sample_report
|
295
|
+
mock_instance.assessment_config = MagicMock()
|
296
|
+
mock_runner.return_value = mock_instance
|
297
|
+
|
298
|
+
mock_jira_instance = MagicMock()
|
299
|
+
mock_jira.return_value = mock_jira_instance
|
300
|
+
|
301
|
+
with tempfile.TemporaryDirectory() as temp_dir:
|
302
|
+
jira_file = os.path.join(temp_dir, "jira_export.csv")
|
303
|
+
|
304
|
+
result = self.runner.invoke(
|
305
|
+
main,
|
306
|
+
[
|
307
|
+
"--profile",
|
308
|
+
"production",
|
309
|
+
"--region",
|
310
|
+
"us-west-2",
|
311
|
+
"--debug",
|
312
|
+
"cfat",
|
313
|
+
"assess",
|
314
|
+
"--output",
|
315
|
+
"all",
|
316
|
+
"--categories",
|
317
|
+
"iam",
|
318
|
+
"cloudtrail",
|
319
|
+
"--skip-checks",
|
320
|
+
"ec2_unused_instances",
|
321
|
+
"--severity",
|
322
|
+
"CRITICAL",
|
323
|
+
"--parallel",
|
324
|
+
"--max-workers",
|
325
|
+
"8",
|
326
|
+
"--compliance-framework",
|
327
|
+
"SOC2",
|
328
|
+
"--export-jira",
|
329
|
+
jira_file,
|
330
|
+
],
|
331
|
+
)
|
332
|
+
|
333
|
+
assert result.exit_code == 0
|
334
|
+
|
335
|
+
# Verify all configurations were applied
|
336
|
+
assert mock_instance.assessment_config.included_categories == ["iam", "cloudtrail"]
|
337
|
+
assert mock_instance.assessment_config.excluded_checks == ["ec2_unused_instances"]
|
338
|
+
assert mock_instance.assessment_config.parallel_execution is True
|
339
|
+
assert mock_instance.assessment_config.max_workers == 8
|
340
|
+
assert mock_instance.assessment_config.compliance_framework == "SOC2"
|
@@ -0,0 +1,290 @@
|
|
1
|
+
"""
|
2
|
+
Integration tests for Cloud Foundations Assessment Tool.
|
3
|
+
|
4
|
+
Tests the complete CFAT workflow using moto for AWS service mocking,
|
5
|
+
ensuring the assessment engine works correctly with AWS APIs without
|
6
|
+
requiring real AWS credentials or making actual API calls.
|
7
|
+
|
8
|
+
These tests focus on integration patterns and real-world usage scenarios
|
9
|
+
while maintaining fast execution and reliability.
|
10
|
+
"""
|
11
|
+
|
12
|
+
from unittest.mock import MagicMock, patch
|
13
|
+
|
14
|
+
import boto3
|
15
|
+
import pytest
|
16
|
+
from moto import mock_aws
|
17
|
+
|
18
|
+
from runbooks.cfat.assessment.runner import CloudFoundationsAssessment
|
19
|
+
from runbooks.cfat.models import AssessmentConfig, CheckStatus, Severity
|
20
|
+
from runbooks.cfat.tests import TEST_ACCOUNT_ID, TEST_PROFILE, TEST_REGION
|
21
|
+
|
22
|
+
|
23
|
+
@pytest.mark.integration
|
24
|
+
class TestCFATIntegrationWithMoto:
|
25
|
+
"""Integration tests using moto for AWS service mocking."""
|
26
|
+
|
27
|
+
@mock_aws
|
28
|
+
def test_iam_assessment_with_mock_services(self):
|
29
|
+
"""Test IAM assessment using moto-mocked AWS services."""
|
30
|
+
# Create mock IAM resources
|
31
|
+
iam_client = boto3.client("iam", region_name=TEST_REGION)
|
32
|
+
|
33
|
+
# Create test user
|
34
|
+
iam_client.create_user(UserName="test-user")
|
35
|
+
|
36
|
+
# Create test role
|
37
|
+
assume_role_policy = {
|
38
|
+
"Version": "2012-10-17",
|
39
|
+
"Statement": [
|
40
|
+
{"Effect": "Allow", "Principal": {"Service": "ec2.amazonaws.com"}, "Action": "sts:AssumeRole"}
|
41
|
+
],
|
42
|
+
}
|
43
|
+
iam_client.create_role(RoleName="test-role", AssumeRolePolicyDocument=str(assume_role_policy))
|
44
|
+
|
45
|
+
# Create test policy
|
46
|
+
policy_document = {
|
47
|
+
"Version": "2012-10-17",
|
48
|
+
"Statement": [{"Effect": "Allow", "Action": "s3:GetObject", "Resource": "*"}],
|
49
|
+
}
|
50
|
+
iam_client.create_policy(PolicyName="test-policy", PolicyDocument=str(policy_document))
|
51
|
+
|
52
|
+
# Mock the assessment runner to use our mocked services
|
53
|
+
with patch("runbooks.cfat.assessment.runner.CloudFoundationsAssessment.get_account_id") as mock_account:
|
54
|
+
mock_account.return_value = TEST_ACCOUNT_ID
|
55
|
+
|
56
|
+
# Initialize assessment
|
57
|
+
assessment = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
|
58
|
+
|
59
|
+
# Configure for IAM-only assessment
|
60
|
+
assessment.assessment_config.included_categories = ["iam"]
|
61
|
+
assessment.assessment_config.parallel_execution = False # Easier to debug
|
62
|
+
|
63
|
+
# Run assessment
|
64
|
+
report = assessment.run_assessment()
|
65
|
+
|
66
|
+
# Validate results
|
67
|
+
assert report is not None
|
68
|
+
assert report.account_id == TEST_ACCOUNT_ID
|
69
|
+
assert report.region == TEST_REGION
|
70
|
+
assert len(report.results) > 0
|
71
|
+
|
72
|
+
# Should have IAM-related results
|
73
|
+
iam_results = report.get_results_by_category("iam")
|
74
|
+
assert len(iam_results) > 0
|
75
|
+
|
76
|
+
# Verify result structure
|
77
|
+
for result in iam_results:
|
78
|
+
assert result.finding_id is not None
|
79
|
+
assert result.check_name is not None
|
80
|
+
assert result.check_category == "iam"
|
81
|
+
assert result.status in [CheckStatus.PASS, CheckStatus.FAIL, CheckStatus.ERROR]
|
82
|
+
assert result.severity in [Severity.INFO, Severity.WARNING, Severity.CRITICAL]
|
83
|
+
assert result.execution_time >= 0
|
84
|
+
|
85
|
+
@mock_aws
|
86
|
+
def test_vpc_assessment_with_mock_services(self):
|
87
|
+
"""Test VPC assessment using moto-mocked EC2 services."""
|
88
|
+
# Create mock VPC resources
|
89
|
+
ec2_client = boto3.client("ec2", region_name=TEST_REGION)
|
90
|
+
|
91
|
+
# Create VPC
|
92
|
+
vpc_response = ec2_client.create_vpc(CidrBlock="10.0.0.0/16")
|
93
|
+
vpc_id = vpc_response["Vpc"]["VpcId"]
|
94
|
+
|
95
|
+
# Create subnet
|
96
|
+
ec2_client.create_subnet(VpcId=vpc_id, CidrBlock="10.0.1.0/24")
|
97
|
+
|
98
|
+
# Create security group
|
99
|
+
ec2_client.create_security_group(GroupName="test-sg", Description="Test security group", VpcId=vpc_id)
|
100
|
+
|
101
|
+
# Mock assessment execution
|
102
|
+
with patch("runbooks.cfat.assessment.runner.CloudFoundationsAssessment.get_account_id") as mock_account:
|
103
|
+
mock_account.return_value = TEST_ACCOUNT_ID
|
104
|
+
|
105
|
+
assessment = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
|
106
|
+
|
107
|
+
# Configure for VPC-only assessment
|
108
|
+
assessment.assessment_config.included_categories = ["vpc"]
|
109
|
+
assessment.assessment_config.parallel_execution = False
|
110
|
+
|
111
|
+
report = assessment.run_assessment()
|
112
|
+
|
113
|
+
# Validate VPC assessment results
|
114
|
+
assert report is not None
|
115
|
+
vpc_results = report.get_results_by_category("vpc")
|
116
|
+
|
117
|
+
# Should have some VPC-related checks
|
118
|
+
assert len(vpc_results) >= 0 # May be 0 if no VPC checks implemented yet
|
119
|
+
|
120
|
+
@mock_aws
|
121
|
+
def test_cloudtrail_assessment_with_mock_services(self):
|
122
|
+
"""Test CloudTrail assessment using moto-mocked services."""
|
123
|
+
# Create mock CloudTrail
|
124
|
+
cloudtrail_client = boto3.client("cloudtrail", region_name=TEST_REGION)
|
125
|
+
|
126
|
+
# Create trail
|
127
|
+
trail_name = "test-trail"
|
128
|
+
s3_bucket = "test-cloudtrail-bucket"
|
129
|
+
|
130
|
+
cloudtrail_client.create_trail(Name=trail_name, S3BucketName=s3_bucket)
|
131
|
+
|
132
|
+
# Start logging
|
133
|
+
cloudtrail_client.start_logging(Name=trail_name)
|
134
|
+
|
135
|
+
# Mock assessment
|
136
|
+
with patch("runbooks.cfat.assessment.runner.CloudFoundationsAssessment.get_account_id") as mock_account:
|
137
|
+
mock_account.return_value = TEST_ACCOUNT_ID
|
138
|
+
|
139
|
+
assessment = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
|
140
|
+
|
141
|
+
assessment.assessment_config.included_categories = ["cloudtrail"]
|
142
|
+
assessment.assessment_config.parallel_execution = False
|
143
|
+
|
144
|
+
report = assessment.run_assessment()
|
145
|
+
|
146
|
+
assert report is not None
|
147
|
+
cloudtrail_results = report.get_results_by_category("cloudtrail")
|
148
|
+
assert len(cloudtrail_results) >= 0
|
149
|
+
|
150
|
+
def test_assessment_configuration_integration(self):
|
151
|
+
"""Test assessment configuration integration."""
|
152
|
+
# Test custom configuration
|
153
|
+
config = AssessmentConfig(
|
154
|
+
included_categories=["iam", "vpc"],
|
155
|
+
excluded_checks=["iam_unused_credentials"],
|
156
|
+
parallel_execution=True,
|
157
|
+
max_workers=5,
|
158
|
+
severity_threshold=Severity.WARNING,
|
159
|
+
compliance_framework="SOC2",
|
160
|
+
)
|
161
|
+
|
162
|
+
with patch("runbooks.cfat.assessment.runner.CloudFoundationsAssessment.get_account_id") as mock_account:
|
163
|
+
mock_account.return_value = TEST_ACCOUNT_ID
|
164
|
+
|
165
|
+
assessment = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
|
166
|
+
assessment.assessment_config = config
|
167
|
+
|
168
|
+
# Should be able to run without errors
|
169
|
+
report = assessment.run_assessment()
|
170
|
+
assert report is not None
|
171
|
+
assert report.metadata.get("max_workers") == 5
|
172
|
+
|
173
|
+
def test_parallel_vs_sequential_execution(self):
|
174
|
+
"""Test parallel vs sequential execution modes."""
|
175
|
+
with patch("runbooks.cfat.assessment.runner.CloudFoundationsAssessment.get_account_id") as mock_account:
|
176
|
+
mock_account.return_value = TEST_ACCOUNT_ID
|
177
|
+
|
178
|
+
# Test parallel execution
|
179
|
+
assessment_parallel = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
|
180
|
+
assessment_parallel.assessment_config.parallel_execution = True
|
181
|
+
assessment_parallel.assessment_config.max_workers = 3
|
182
|
+
|
183
|
+
report_parallel = assessment_parallel.run_assessment()
|
184
|
+
|
185
|
+
# Test sequential execution
|
186
|
+
assessment_sequential = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
|
187
|
+
assessment_sequential.assessment_config.parallel_execution = False
|
188
|
+
|
189
|
+
report_sequential = assessment_sequential.run_assessment()
|
190
|
+
|
191
|
+
# Both should produce valid reports
|
192
|
+
assert report_parallel is not None
|
193
|
+
assert report_sequential is not None
|
194
|
+
|
195
|
+
# Results should be similar (may vary due to mock timing)
|
196
|
+
assert len(report_parallel.results) > 0
|
197
|
+
assert len(report_sequential.results) > 0
|
198
|
+
|
199
|
+
def test_error_handling_in_assessment(self):
|
200
|
+
"""Test error handling during assessment execution."""
|
201
|
+
with patch("runbooks.cfat.assessment.runner.CloudFoundationsAssessment.get_account_id") as mock_account:
|
202
|
+
mock_account.return_value = TEST_ACCOUNT_ID
|
203
|
+
|
204
|
+
assessment = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
|
205
|
+
|
206
|
+
# Mock a check that raises an exception
|
207
|
+
original_execute_single_check = assessment._execute_single_check
|
208
|
+
|
209
|
+
def mock_execute_single_check(check_name):
|
210
|
+
if check_name == "failing_check":
|
211
|
+
raise Exception("Simulated check failure")
|
212
|
+
return original_execute_single_check(check_name)
|
213
|
+
|
214
|
+
assessment._execute_single_check = mock_execute_single_check
|
215
|
+
|
216
|
+
# Override available checks to include our failing check
|
217
|
+
assessment._available_checks = {"passing_check": "PassingCheck", "failing_check": "FailingCheck"}
|
218
|
+
|
219
|
+
# Run assessment
|
220
|
+
report = assessment.run_assessment()
|
221
|
+
|
222
|
+
# Should handle errors gracefully
|
223
|
+
assert report is not None
|
224
|
+
|
225
|
+
# Should have error results for failed checks
|
226
|
+
error_results = [r for r in report.results if r.status == CheckStatus.ERROR]
|
227
|
+
assert len(error_results) > 0
|
228
|
+
|
229
|
+
def test_report_generation_all_formats(self):
|
230
|
+
"""Test report generation in all supported formats."""
|
231
|
+
with patch("runbooks.cfat.assessment.runner.CloudFoundationsAssessment.get_account_id") as mock_account:
|
232
|
+
mock_account.return_value = TEST_ACCOUNT_ID
|
233
|
+
|
234
|
+
assessment = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
|
235
|
+
|
236
|
+
# Run assessment
|
237
|
+
report = assessment.run_assessment()
|
238
|
+
|
239
|
+
# Test all export formats (without actually writing files)
|
240
|
+
assert hasattr(report, "to_json")
|
241
|
+
assert hasattr(report, "to_csv")
|
242
|
+
assert hasattr(report, "to_html")
|
243
|
+
assert hasattr(report, "to_markdown")
|
244
|
+
|
245
|
+
# Test methods return without error
|
246
|
+
assert callable(report.to_json)
|
247
|
+
assert callable(report.to_csv)
|
248
|
+
assert callable(report.to_html)
|
249
|
+
assert callable(report.to_markdown)
|
250
|
+
|
251
|
+
def test_compliance_framework_integration(self):
|
252
|
+
"""Test compliance framework-specific assessments."""
|
253
|
+
frameworks = ["SOC2", "PCI-DSS", "HIPAA"]
|
254
|
+
|
255
|
+
for framework in frameworks:
|
256
|
+
with patch("runbooks.cfat.assessment.runner.CloudFoundationsAssessment.get_account_id") as mock_account:
|
257
|
+
mock_account.return_value = TEST_ACCOUNT_ID
|
258
|
+
|
259
|
+
assessment = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
|
260
|
+
assessment.assessment_config.compliance_framework = framework
|
261
|
+
|
262
|
+
report = assessment.run_assessment()
|
263
|
+
|
264
|
+
assert report is not None
|
265
|
+
assert (
|
266
|
+
report.metadata.get("compliance_framework") is None
|
267
|
+
or report.metadata.get("compliance_framework") == framework
|
268
|
+
)
|
269
|
+
|
270
|
+
@pytest.mark.slow
|
271
|
+
def test_large_scale_assessment(self):
|
272
|
+
"""Test assessment with many checks (performance test)."""
|
273
|
+
with patch("runbooks.cfat.assessment.runner.CloudFoundationsAssessment.get_account_id") as mock_account:
|
274
|
+
mock_account.return_value = TEST_ACCOUNT_ID
|
275
|
+
|
276
|
+
assessment = CloudFoundationsAssessment(profile=TEST_PROFILE, region=TEST_REGION)
|
277
|
+
|
278
|
+
# Override with many mock checks
|
279
|
+
many_checks = {f"check_{i}": f"Check{i}" for i in range(50)}
|
280
|
+
assessment._available_checks = many_checks
|
281
|
+
|
282
|
+
assessment.assessment_config.parallel_execution = True
|
283
|
+
assessment.assessment_config.max_workers = 10
|
284
|
+
|
285
|
+
report = assessment.run_assessment()
|
286
|
+
|
287
|
+
assert report is not None
|
288
|
+
# Should complete in reasonable time with parallel execution
|
289
|
+
assert report.summary.total_execution_time < 60 # Should finish within 60 seconds
|
290
|
+
assert len(report.results) == 50
|