runbooks 0.2.3__py3-none-any.whl → 0.6.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- conftest.py +26 -0
- jupyter-agent/.env.template +2 -0
- jupyter-agent/.gitattributes +35 -0
- jupyter-agent/README.md +16 -0
- jupyter-agent/app.py +256 -0
- jupyter-agent/cloudops-agent.png +0 -0
- jupyter-agent/ds-system-prompt.txt +154 -0
- jupyter-agent/jupyter-agent.png +0 -0
- jupyter-agent/llama3_template.jinja +123 -0
- jupyter-agent/requirements.txt +9 -0
- jupyter-agent/utils.py +409 -0
- runbooks/__init__.py +71 -3
- runbooks/__main__.py +13 -0
- runbooks/aws/ec2_describe_instances.py +1 -1
- runbooks/aws/ec2_run_instances.py +8 -2
- runbooks/aws/ec2_start_stop_instances.py +17 -4
- runbooks/aws/ec2_unused_volumes.py +5 -1
- runbooks/aws/s3_create_bucket.py +4 -2
- runbooks/aws/s3_list_objects.py +6 -1
- runbooks/aws/tagging_lambda_handler.py +13 -2
- runbooks/aws/tags.json +12 -0
- runbooks/base.py +353 -0
- runbooks/cfat/README.md +49 -0
- runbooks/cfat/__init__.py +74 -0
- runbooks/cfat/app.ts +644 -0
- runbooks/cfat/assessment/__init__.py +40 -0
- runbooks/cfat/assessment/asana-import.csv +39 -0
- runbooks/cfat/assessment/cfat-checks.csv +31 -0
- runbooks/cfat/assessment/cfat.txt +520 -0
- runbooks/cfat/assessment/collectors.py +200 -0
- runbooks/cfat/assessment/jira-import.csv +39 -0
- runbooks/cfat/assessment/runner.py +387 -0
- runbooks/cfat/assessment/validators.py +290 -0
- runbooks/cfat/cli.py +103 -0
- runbooks/cfat/docs/asana-import.csv +24 -0
- runbooks/cfat/docs/cfat-checks.csv +31 -0
- runbooks/cfat/docs/cfat.txt +335 -0
- runbooks/cfat/docs/checks-output.png +0 -0
- runbooks/cfat/docs/cloudshell-console-run.png +0 -0
- runbooks/cfat/docs/cloudshell-download.png +0 -0
- runbooks/cfat/docs/cloudshell-output.png +0 -0
- runbooks/cfat/docs/downloadfile.png +0 -0
- runbooks/cfat/docs/jira-import.csv +24 -0
- runbooks/cfat/docs/open-cloudshell.png +0 -0
- runbooks/cfat/docs/report-header.png +0 -0
- runbooks/cfat/models.py +1026 -0
- runbooks/cfat/package-lock.json +5116 -0
- runbooks/cfat/package.json +38 -0
- runbooks/cfat/report.py +496 -0
- runbooks/cfat/reporting/__init__.py +46 -0
- runbooks/cfat/reporting/exporters.py +337 -0
- runbooks/cfat/reporting/formatters.py +496 -0
- runbooks/cfat/reporting/templates.py +135 -0
- runbooks/cfat/run-assessment.sh +23 -0
- runbooks/cfat/runner.py +69 -0
- runbooks/cfat/src/actions/check-cloudtrail-existence.ts +43 -0
- runbooks/cfat/src/actions/check-config-existence.ts +37 -0
- runbooks/cfat/src/actions/check-control-tower.ts +37 -0
- runbooks/cfat/src/actions/check-ec2-existence.ts +46 -0
- runbooks/cfat/src/actions/check-iam-users.ts +50 -0
- runbooks/cfat/src/actions/check-legacy-cur.ts +30 -0
- runbooks/cfat/src/actions/check-org-cloudformation.ts +30 -0
- runbooks/cfat/src/actions/check-vpc-existence.ts +43 -0
- runbooks/cfat/src/actions/create-asanaimport.ts +14 -0
- runbooks/cfat/src/actions/create-backlog.ts +372 -0
- runbooks/cfat/src/actions/create-jiraimport.ts +15 -0
- runbooks/cfat/src/actions/create-report.ts +616 -0
- runbooks/cfat/src/actions/define-account-type.ts +51 -0
- runbooks/cfat/src/actions/get-enabled-org-policy-types.ts +40 -0
- runbooks/cfat/src/actions/get-enabled-org-services.ts +26 -0
- runbooks/cfat/src/actions/get-idc-info.ts +34 -0
- runbooks/cfat/src/actions/get-org-da-accounts.ts +34 -0
- runbooks/cfat/src/actions/get-org-details.ts +35 -0
- runbooks/cfat/src/actions/get-org-member-accounts.ts +44 -0
- runbooks/cfat/src/actions/get-org-ous.ts +35 -0
- runbooks/cfat/src/actions/get-regions.ts +22 -0
- runbooks/cfat/src/actions/zip-assessment.ts +27 -0
- runbooks/cfat/src/types/index.d.ts +147 -0
- runbooks/cfat/tests/__init__.py +141 -0
- runbooks/cfat/tests/test_cli.py +340 -0
- runbooks/cfat/tests/test_integration.py +290 -0
- runbooks/cfat/tests/test_models.py +505 -0
- runbooks/cfat/tests/test_reporting.py +354 -0
- runbooks/cfat/tsconfig.json +16 -0
- runbooks/cfat/webpack.config.cjs +27 -0
- runbooks/config.py +260 -0
- runbooks/finops/__init__.py +88 -0
- runbooks/finops/aws_client.py +245 -0
- runbooks/finops/cli.py +151 -0
- runbooks/finops/cost_processor.py +410 -0
- runbooks/finops/dashboard_runner.py +448 -0
- runbooks/finops/helpers.py +355 -0
- runbooks/finops/main.py +14 -0
- runbooks/finops/profile_processor.py +174 -0
- runbooks/finops/types.py +66 -0
- runbooks/finops/visualisations.py +80 -0
- runbooks/inventory/.gitignore +354 -0
- runbooks/inventory/ArgumentsClass.py +261 -0
- runbooks/inventory/Inventory_Modules.py +6130 -0
- runbooks/inventory/LandingZone/delete_lz.py +1075 -0
- runbooks/inventory/README.md +1320 -0
- runbooks/inventory/__init__.py +62 -0
- runbooks/inventory/account_class.py +532 -0
- runbooks/inventory/all_my_instances_wrapper.py +123 -0
- runbooks/inventory/aws_decorators.py +201 -0
- runbooks/inventory/cfn_move_stack_instances.py +1526 -0
- runbooks/inventory/check_cloudtrail_compliance.py +614 -0
- runbooks/inventory/check_controltower_readiness.py +1107 -0
- runbooks/inventory/check_landingzone_readiness.py +711 -0
- runbooks/inventory/cloudtrail.md +727 -0
- runbooks/inventory/collectors/__init__.py +20 -0
- runbooks/inventory/collectors/aws_compute.py +518 -0
- runbooks/inventory/collectors/aws_networking.py +275 -0
- runbooks/inventory/collectors/base.py +222 -0
- runbooks/inventory/core/__init__.py +19 -0
- runbooks/inventory/core/collector.py +303 -0
- runbooks/inventory/core/formatter.py +296 -0
- runbooks/inventory/delete_s3_buckets_objects.py +169 -0
- runbooks/inventory/discovery.md +81 -0
- runbooks/inventory/draw_org_structure.py +748 -0
- runbooks/inventory/ec2_vpc_utils.py +341 -0
- runbooks/inventory/find_cfn_drift_detection.py +272 -0
- runbooks/inventory/find_cfn_orphaned_stacks.py +719 -0
- runbooks/inventory/find_cfn_stackset_drift.py +733 -0
- runbooks/inventory/find_ec2_security_groups.py +669 -0
- runbooks/inventory/find_landingzone_versions.py +201 -0
- runbooks/inventory/find_vpc_flow_logs.py +1221 -0
- runbooks/inventory/inventory.sh +659 -0
- runbooks/inventory/list_cfn_stacks.py +558 -0
- runbooks/inventory/list_cfn_stackset_operation_results.py +252 -0
- runbooks/inventory/list_cfn_stackset_operations.py +734 -0
- runbooks/inventory/list_cfn_stacksets.py +453 -0
- runbooks/inventory/list_config_recorders_delivery_channels.py +681 -0
- runbooks/inventory/list_ds_directories.py +354 -0
- runbooks/inventory/list_ec2_availability_zones.py +286 -0
- runbooks/inventory/list_ec2_ebs_volumes.py +244 -0
- runbooks/inventory/list_ec2_instances.py +425 -0
- runbooks/inventory/list_ecs_clusters_and_tasks.py +562 -0
- runbooks/inventory/list_elbs_load_balancers.py +411 -0
- runbooks/inventory/list_enis_network_interfaces.py +526 -0
- runbooks/inventory/list_guardduty_detectors.py +568 -0
- runbooks/inventory/list_iam_policies.py +404 -0
- runbooks/inventory/list_iam_roles.py +518 -0
- runbooks/inventory/list_iam_saml_providers.py +359 -0
- runbooks/inventory/list_lambda_functions.py +882 -0
- runbooks/inventory/list_org_accounts.py +446 -0
- runbooks/inventory/list_org_accounts_users.py +354 -0
- runbooks/inventory/list_rds_db_instances.py +406 -0
- runbooks/inventory/list_route53_hosted_zones.py +318 -0
- runbooks/inventory/list_servicecatalog_provisioned_products.py +575 -0
- runbooks/inventory/list_sns_topics.py +360 -0
- runbooks/inventory/list_ssm_parameters.py +402 -0
- runbooks/inventory/list_vpc_subnets.py +433 -0
- runbooks/inventory/list_vpcs.py +422 -0
- runbooks/inventory/lockdown_cfn_stackset_role.py +224 -0
- runbooks/inventory/models/__init__.py +24 -0
- runbooks/inventory/models/account.py +192 -0
- runbooks/inventory/models/inventory.py +309 -0
- runbooks/inventory/models/resource.py +247 -0
- runbooks/inventory/recover_cfn_stack_ids.py +205 -0
- runbooks/inventory/requirements.txt +12 -0
- runbooks/inventory/run_on_multi_accounts.py +211 -0
- runbooks/inventory/tests/common_test_data.py +3661 -0
- runbooks/inventory/tests/common_test_functions.py +204 -0
- runbooks/inventory/tests/script_test_data.py +0 -0
- runbooks/inventory/tests/setup.py +24 -0
- runbooks/inventory/tests/src.py +18 -0
- runbooks/inventory/tests/test_cfn_describe_stacks.py +208 -0
- runbooks/inventory/tests/test_ec2_describe_instances.py +162 -0
- runbooks/inventory/tests/test_inventory_modules.py +55 -0
- runbooks/inventory/tests/test_lambda_list_functions.py +86 -0
- runbooks/inventory/tests/test_moto_integration_example.py +273 -0
- runbooks/inventory/tests/test_org_list_accounts.py +49 -0
- runbooks/inventory/update_aws_actions.py +173 -0
- runbooks/inventory/update_cfn_stacksets.py +1215 -0
- runbooks/inventory/update_cloudwatch_logs_retention_policy.py +294 -0
- runbooks/inventory/update_iam_roles_cross_accounts.py +478 -0
- runbooks/inventory/update_s3_public_access_block.py +539 -0
- runbooks/inventory/utils/__init__.py +23 -0
- runbooks/inventory/utils/aws_helpers.py +510 -0
- runbooks/inventory/utils/threading_utils.py +493 -0
- runbooks/inventory/utils/validation.py +682 -0
- runbooks/inventory/verify_ec2_security_groups.py +1430 -0
- runbooks/main.py +785 -0
- runbooks/organizations/__init__.py +12 -0
- runbooks/organizations/manager.py +374 -0
- runbooks/security_baseline/README.md +324 -0
- runbooks/security_baseline/checklist/alternate_contacts.py +8 -1
- runbooks/security_baseline/checklist/bucket_public_access.py +4 -1
- runbooks/security_baseline/checklist/cloudwatch_alarm_configuration.py +9 -2
- runbooks/security_baseline/checklist/guardduty_enabled.py +9 -2
- runbooks/security_baseline/checklist/multi_region_instance_usage.py +5 -1
- runbooks/security_baseline/checklist/root_access_key.py +6 -1
- runbooks/security_baseline/config-origin.json +1 -1
- runbooks/security_baseline/config.json +1 -1
- runbooks/security_baseline/permission.json +1 -1
- runbooks/security_baseline/report_generator.py +10 -2
- runbooks/security_baseline/report_template_en.html +8 -8
- runbooks/security_baseline/report_template_jp.html +8 -8
- runbooks/security_baseline/report_template_kr.html +13 -13
- runbooks/security_baseline/report_template_vn.html +8 -8
- runbooks/security_baseline/requirements.txt +7 -0
- runbooks/security_baseline/run_script.py +8 -2
- runbooks/security_baseline/security_baseline_tester.py +10 -2
- runbooks/security_baseline/utils/common.py +5 -1
- runbooks/utils/__init__.py +204 -0
- runbooks-0.6.1.dist-info/METADATA +373 -0
- runbooks-0.6.1.dist-info/RECORD +237 -0
- {runbooks-0.2.3.dist-info → runbooks-0.6.1.dist-info}/WHEEL +1 -1
- runbooks-0.6.1.dist-info/entry_points.txt +7 -0
- runbooks-0.6.1.dist-info/licenses/LICENSE +201 -0
- runbooks-0.6.1.dist-info/top_level.txt +3 -0
- runbooks/python101/calculator.py +0 -34
- runbooks/python101/config.py +0 -1
- runbooks/python101/exceptions.py +0 -16
- runbooks/python101/file_manager.py +0 -218
- runbooks/python101/toolkit.py +0 -153
- runbooks-0.2.3.dist-info/METADATA +0 -435
- runbooks-0.2.3.dist-info/RECORD +0 -61
- runbooks-0.2.3.dist-info/entry_points.txt +0 -3
- runbooks-0.2.3.dist-info/top_level.txt +0 -1
@@ -0,0 +1,244 @@
|
|
1
|
+
#!/usr/bin/env python3
|
2
|
+
import logging
|
3
|
+
import sys
|
4
|
+
from queue import Queue
|
5
|
+
from threading import Thread
|
6
|
+
from time import time
|
7
|
+
|
8
|
+
from ArgumentsClass import CommonArguments
|
9
|
+
from botocore.exceptions import ClientError
|
10
|
+
from colorama import Fore, init
|
11
|
+
from Inventory_Modules import display_results, find_account_volumes2, get_all_credentials
|
12
|
+
from tqdm.auto import tqdm
|
13
|
+
|
14
|
+
init()
|
15
|
+
__version__ = "2024.05.31"
|
16
|
+
|
17
|
+
# ANSI escape code for clearing current line (progress bar cleanup)
|
18
|
+
ERASE_LINE = "\x1b[2K"
|
19
|
+
|
20
|
+
|
21
|
+
##################
|
22
|
+
def parse_args(f_arguments):
|
23
|
+
"""
|
24
|
+
Description: Parses the arguments passed into the script
|
25
|
+
@param f_arguments: args represents the list of arguments passed in
|
26
|
+
@return: returns an object namespace that contains the individualized parameters passed in
|
27
|
+
"""
|
28
|
+
|
29
|
+
parser = CommonArguments()
|
30
|
+
parser.multiprofile()
|
31
|
+
parser.multiregion()
|
32
|
+
parser.extendedargs()
|
33
|
+
parser.fragment()
|
34
|
+
parser.rootOnly()
|
35
|
+
parser.save_to_file()
|
36
|
+
parser.timing()
|
37
|
+
parser.verbosity()
|
38
|
+
parser.version(__version__)
|
39
|
+
return parser.my_parser.parse_args(f_arguments)
|
40
|
+
|
41
|
+
|
42
|
+
def present_results(fVolumesFound: list):
|
43
|
+
"""
|
44
|
+
Display comprehensive results of EBS volume discovery with analysis.
|
45
|
+
|
46
|
+
This function processes the discovered volumes, removes duplicates,
|
47
|
+
sorts them logically, and presents them in a formatted output with
|
48
|
+
summary statistics and operational insights.
|
49
|
+
|
50
|
+
Args:
|
51
|
+
fVolumesFound (list): List of discovered EBS volumes with metadata
|
52
|
+
"""
|
53
|
+
|
54
|
+
display_dict = {
|
55
|
+
"VolumeId": {"DisplayOrder": 1, "Heading": "Volume ID"},
|
56
|
+
"Size": {"DisplayOrder": 2, "Heading": "Size (GB)"},
|
57
|
+
"VolumeType": {"DisplayOrder": 3, "Heading": "Volume Type"},
|
58
|
+
"VolumeName": {"DisplayOrder": 4, "Heading": "Volume Name"},
|
59
|
+
"InstanceId": {"DisplayOrder": 5, "Heading": "Instance ID"},
|
60
|
+
"Encrypted": {"DisplayOrder": 6, "Heading": "Encrypted"},
|
61
|
+
"State": {"DisplayOrder": 7, "Heading": "State"},
|
62
|
+
"AccountId": {"DisplayOrder": 8, "Heading": "Account"},
|
63
|
+
"Region": {"DisplayOrder": 9, "Heading": "Region"},
|
64
|
+
}
|
65
|
+
|
66
|
+
# Phase 1: Data deduplication and preparation
|
67
|
+
de_dupe_VolumesFound = []
|
68
|
+
AccountsFound = set()
|
69
|
+
RegionsFound = set()
|
70
|
+
|
71
|
+
# Remove duplicate volumes based on VolumeId and collect unique accounts/regions
|
72
|
+
seen = set()
|
73
|
+
for volume in fVolumesFound:
|
74
|
+
key = volume["VolumeId"]
|
75
|
+
if key not in seen:
|
76
|
+
seen.add(key)
|
77
|
+
de_dupe_VolumesFound.append(volume)
|
78
|
+
AccountsFound.add(volume.get("AccountId"))
|
79
|
+
RegionsFound.add(volume.get("Region"))
|
80
|
+
|
81
|
+
sorted_Volumes_Found = sorted(
|
82
|
+
de_dupe_VolumesFound, key=lambda x: (x["MgmtAccount"], x["AccountId"], x["Region"], x["VolumeName"], x["Size"])
|
83
|
+
)
|
84
|
+
display_results(sorted_Volumes_Found, display_dict, "None", None)
|
85
|
+
|
86
|
+
print()
|
87
|
+
print(
|
88
|
+
f"Found {len(de_dupe_VolumesFound)} volumes across {len(AccountsFound)} account{'' if len(AccountsFound) == 1 else 's'} "
|
89
|
+
f"across {len(RegionsFound)} region{'' if len(RegionsFound) == 1 else 's'}"
|
90
|
+
)
|
91
|
+
|
92
|
+
# Calculate and display orphaned volumes
|
93
|
+
orphaned_volumes = [vol for vol in de_dupe_VolumesFound if vol.get("State") == "available"]
|
94
|
+
if orphaned_volumes:
|
95
|
+
print(f"{Fore.YELLOW}Warning: {len(orphaned_volumes)} orphaned (unattached) volumes found{Fore.RESET}")
|
96
|
+
total_orphaned_size = sum(vol.get("Size", 0) for vol in orphaned_volumes)
|
97
|
+
print(f"Total orphaned storage: {total_orphaned_size} GB")
|
98
|
+
|
99
|
+
print()
|
100
|
+
|
101
|
+
|
102
|
+
def check_accounts_for_ebs_volumes(f_CredentialList, f_fragment_list=None):
|
103
|
+
"""
|
104
|
+
Note that this function takes a list of Credentials and checks for EBS Volumes in every account it has creds for
|
105
|
+
@param f_CredentialList: List of credentials for all accounts to check
|
106
|
+
@param f_fragment_list: List of name tag fragments to limit the searching to
|
107
|
+
@return:
|
108
|
+
"""
|
109
|
+
|
110
|
+
class FindVolumes(Thread):
|
111
|
+
def __init__(self, queue):
|
112
|
+
Thread.__init__(self)
|
113
|
+
self.queue = queue
|
114
|
+
|
115
|
+
def run(self):
|
116
|
+
while True:
|
117
|
+
# Get the work from the queue and expand the tuple
|
118
|
+
# c_account_credentials, c_region, c_text_to_find, c_PlacesToLook, c_PlaceCount = self.queue.get()
|
119
|
+
c_account_credentials, c_region, c_fragment = self.queue.get()
|
120
|
+
logging.info(f"De-queued info for account {c_account_credentials['AccountId']}")
|
121
|
+
try:
|
122
|
+
logging.info(f"Attempting to connect to {c_account_credentials['AccountId']}")
|
123
|
+
# account_volumes = find_account_volumes2(c_account_credentials, c_text_to_find)
|
124
|
+
account_volumes = find_account_volumes2(c_account_credentials)
|
125
|
+
logging.info(f"Successfully connected to account {c_account_credentials['AccountId']}")
|
126
|
+
for _ in range(len(account_volumes)):
|
127
|
+
account_volumes[_]["MgmtAccount"] = c_account_credentials["MgmtAccount"]
|
128
|
+
AllVolumes.extend(account_volumes)
|
129
|
+
except KeyError as my_Error:
|
130
|
+
logging.error(f"Account Access failed - trying to access {c_account_credentials['AccountId']}")
|
131
|
+
logging.info(f"Actual Error: {my_Error}")
|
132
|
+
pass
|
133
|
+
except AttributeError as my_Error:
|
134
|
+
logging.error(f"Error: Likely that one of the supplied profiles {pProfiles} was wrong")
|
135
|
+
logging.warning(my_Error)
|
136
|
+
continue
|
137
|
+
finally:
|
138
|
+
logging.info(
|
139
|
+
f"{ERASE_LINE}Finished finding EBS volumes in account {c_account_credentials['AccountId']} in region {c_account_credentials['Region']}"
|
140
|
+
)
|
141
|
+
pbar.update()
|
142
|
+
self.queue.task_done()
|
143
|
+
|
144
|
+
if f_fragment_list is None:
|
145
|
+
f_fragment_list = []
|
146
|
+
AllVolumes = []
|
147
|
+
WorkerThreads = min(len(f_CredentialList), 50)
|
148
|
+
|
149
|
+
checkqueue = Queue()
|
150
|
+
|
151
|
+
pbar = tqdm(
|
152
|
+
desc=f"Finding ebs volumes from {len(f_CredentialList)} accounts and regions",
|
153
|
+
total=len(f_CredentialList),
|
154
|
+
unit=" accounts & regions",
|
155
|
+
)
|
156
|
+
|
157
|
+
for x in range(WorkerThreads):
|
158
|
+
worker = FindVolumes(checkqueue)
|
159
|
+
# Setting daemon to True will let the main thread exit even though the workers are blocking
|
160
|
+
worker.daemon = True
|
161
|
+
worker.start()
|
162
|
+
|
163
|
+
for credential in f_CredentialList:
|
164
|
+
logging.info(f"Connecting to account {credential['AccountId']}")
|
165
|
+
try:
|
166
|
+
# print(f"{ERASE_LINE}Queuing account {credential['AccountId']} in region {region}", end='\r')
|
167
|
+
checkqueue.put((credential, credential["Region"], f_fragment_list))
|
168
|
+
except ClientError as my_Error:
|
169
|
+
if "AuthFailure" in str(my_Error):
|
170
|
+
logging.error(
|
171
|
+
f"Authorization Failure accessing account {credential['AccountId']} in '{credential['Region']}' region"
|
172
|
+
)
|
173
|
+
logging.warning(f"It's possible that the region '{credential['Region']}' hasn't been opted-into")
|
174
|
+
pass
|
175
|
+
checkqueue.join()
|
176
|
+
pbar.close()
|
177
|
+
return AllVolumes
|
178
|
+
|
179
|
+
|
180
|
+
##################
|
181
|
+
|
182
|
+
|
183
|
+
def main():
|
184
|
+
"""
|
185
|
+
Main execution function for EBS volume inventory discovery.
|
186
|
+
Orchestrates argument parsing, credential discovery, volume enumeration, and result presentation.
|
187
|
+
"""
|
188
|
+
import sys
|
189
|
+
|
190
|
+
args = parse_args(sys.argv[1:])
|
191
|
+
pProfiles = args.Profiles
|
192
|
+
pRegionList = args.Regions
|
193
|
+
pAccounts = args.Accounts
|
194
|
+
pFragments = args.Fragments
|
195
|
+
pSkipAccounts = args.SkipAccounts
|
196
|
+
pSkipProfiles = args.SkipProfiles
|
197
|
+
pRootOnly = args.RootOnly
|
198
|
+
pFilename = args.Filename
|
199
|
+
pTiming = args.Time
|
200
|
+
verbose = args.loglevel
|
201
|
+
|
202
|
+
# Setup logging levels
|
203
|
+
logging.basicConfig(level=verbose, format="[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
|
204
|
+
logging.getLogger("boto3").setLevel(logging.CRITICAL)
|
205
|
+
logging.getLogger("botocore").setLevel(logging.CRITICAL)
|
206
|
+
logging.getLogger("s3transfer").setLevel(logging.CRITICAL)
|
207
|
+
logging.getLogger("urllib3").setLevel(logging.CRITICAL)
|
208
|
+
|
209
|
+
# ANSI escape code for clearing current line (progress bar cleanup)
|
210
|
+
ERASE_LINE = "\x1b[2K"
|
211
|
+
|
212
|
+
# Start execution timing
|
213
|
+
begin_time = time()
|
214
|
+
|
215
|
+
# Display startup information
|
216
|
+
print()
|
217
|
+
print(f"Checking for EBS Volumes... ")
|
218
|
+
logging.info(f"Profiles: {pProfiles}")
|
219
|
+
print()
|
220
|
+
|
221
|
+
# Phase 1: Gather credentials for all target accounts and regions
|
222
|
+
CredentialList = get_all_credentials(
|
223
|
+
pProfiles, pTiming, pSkipProfiles, pSkipAccounts, pRootOnly, pAccounts, pRegionList
|
224
|
+
)
|
225
|
+
|
226
|
+
# Phase 2: Execute multi-threaded EBS volume discovery
|
227
|
+
VolumesFound = check_accounts_for_ebs_volumes(CredentialList, pFragments)
|
228
|
+
|
229
|
+
# Phase 3: Present results with analysis and recommendations
|
230
|
+
present_results(VolumesFound)
|
231
|
+
|
232
|
+
# Display execution timing if requested
|
233
|
+
if pTiming:
|
234
|
+
print(ERASE_LINE)
|
235
|
+
print(f"{Fore.GREEN}This script completed in {time() - begin_time:.2f} seconds{Fore.RESET}")
|
236
|
+
|
237
|
+
|
238
|
+
if __name__ == "__main__":
|
239
|
+
main()
|
240
|
+
|
241
|
+
# Display completion message
|
242
|
+
print()
|
243
|
+
print("Thank you for using this script")
|
244
|
+
print()
|
@@ -0,0 +1,425 @@
|
|
1
|
+
#!/usr/bin/env python3
|
2
|
+
"""
|
3
|
+
AWS EC2 Instance Inventory Collection
|
4
|
+
|
5
|
+
A comprehensive EC2 instance discovery tool that operates across multiple AWS accounts
|
6
|
+
and regions using the AWS Cloud Foundations framework. This script provides detailed
|
7
|
+
inventory reporting with support for multi-account Organizations access patterns.
|
8
|
+
|
9
|
+
**AWS API Mapping**: `boto3.client('ec2').describe_instances()`
|
10
|
+
|
11
|
+
Features:
|
12
|
+
- Multi-account EC2 instance discovery via AWS Organizations
|
13
|
+
- Parallel region scanning with threading optimization
|
14
|
+
- Flexible filtering by instance state (running/stopped)
|
15
|
+
- Multiple output formats (table, JSON, CSV)
|
16
|
+
- Tag-based metadata collection
|
17
|
+
- VPC and network information gathering
|
18
|
+
|
19
|
+
Compatibility:
|
20
|
+
- AWS Organizations with cross-account roles
|
21
|
+
- AWS Control Tower managed accounts
|
22
|
+
- Standalone AWS accounts
|
23
|
+
- All AWS regions including opt-in regions
|
24
|
+
|
25
|
+
Example:
|
26
|
+
Basic usage across all regions:
|
27
|
+
```bash
|
28
|
+
python ec2_describe_instances.py --profile my-org-profile
|
29
|
+
```
|
30
|
+
|
31
|
+
Filter running instances in specific regions:
|
32
|
+
```bash
|
33
|
+
python ec2_describe_instances.py --profile my-profile \\
|
34
|
+
--regions us-east-1 us-west-2 --status running
|
35
|
+
```
|
36
|
+
|
37
|
+
Export to JSON for analysis:
|
38
|
+
```bash
|
39
|
+
python ec2_describe_instances.py --profile my-profile \\
|
40
|
+
--output json --save instances_inventory.json
|
41
|
+
```
|
42
|
+
|
43
|
+
Requirements:
|
44
|
+
- IAM permissions: `ec2:DescribeInstances`, `sts:AssumeRole`
|
45
|
+
- AWS Organizations access (for multi-account scanning)
|
46
|
+
- Python 3.8+ with required dependencies
|
47
|
+
|
48
|
+
Author:
|
49
|
+
AWS Cloud Foundations Team
|
50
|
+
|
51
|
+
Version:
|
52
|
+
2025.04.09
|
53
|
+
"""
|
54
|
+
|
55
|
+
import logging
|
56
|
+
import sys
|
57
|
+
from os.path import split
|
58
|
+
from queue import Queue
|
59
|
+
from threading import Thread
|
60
|
+
from time import time
|
61
|
+
|
62
|
+
import Inventory_Modules
|
63
|
+
from ArgumentsClass import CommonArguments
|
64
|
+
from botocore.exceptions import ClientError
|
65
|
+
from colorama import Fore, init
|
66
|
+
from Inventory_Modules import display_results, get_all_credentials
|
67
|
+
from tqdm.auto import tqdm
|
68
|
+
|
69
|
+
init()
|
70
|
+
__version__ = "2025.04.09"
|
71
|
+
ERASE_LINE = "\x1b[2K"
|
72
|
+
begin_time = time()
|
73
|
+
|
74
|
+
|
75
|
+
##################
|
76
|
+
# Functions
|
77
|
+
##################
|
78
|
+
|
79
|
+
|
80
|
+
def parse_args(f_arguments):
|
81
|
+
"""Parse command-line arguments for EC2 instance inventory collection.
|
82
|
+
|
83
|
+
Configures argument parser with standard AWS Cloud Foundations parameters
|
84
|
+
plus EC2-specific options for filtering instance states.
|
85
|
+
|
86
|
+
Args:
|
87
|
+
f_arguments (list): Command-line arguments to parse
|
88
|
+
|
89
|
+
Returns:
|
90
|
+
argparse.Namespace: Parsed arguments containing:
|
91
|
+
- pProfiles: List of AWS profiles to use
|
92
|
+
- pRegions: List of AWS regions to scan
|
93
|
+
- pStatus: Instance state filter ('running', 'stopped', or None)
|
94
|
+
- Verbose: Logging verbosity level
|
95
|
+
- Other standard AWS Cloud Foundations parameters
|
96
|
+
|
97
|
+
Example:
|
98
|
+
>>> args = parse_args(['--profile', 'my-profile', '--status', 'running'])
|
99
|
+
>>> print(args.pStatus)
|
100
|
+
'running'
|
101
|
+
"""
|
102
|
+
script_path, script_name = split(sys.argv[0])
|
103
|
+
parser = CommonArguments()
|
104
|
+
parser.my_parser.description = (
|
105
|
+
"We're going to find all instances within any of the accounts we have access to, given the profile(s) provided."
|
106
|
+
)
|
107
|
+
parser.multiprofile()
|
108
|
+
parser.multiregion()
|
109
|
+
parser.extendedargs()
|
110
|
+
parser.rolestouse()
|
111
|
+
parser.rootOnly()
|
112
|
+
parser.save_to_file()
|
113
|
+
parser.timing()
|
114
|
+
parser.verbosity()
|
115
|
+
parser.version(__version__)
|
116
|
+
local = parser.my_parser.add_argument_group(script_name, "Parameters specific to this script")
|
117
|
+
local.add_argument(
|
118
|
+
"-s",
|
119
|
+
"--status",
|
120
|
+
dest="pStatus",
|
121
|
+
choices=["running", "stopped"],
|
122
|
+
type=str,
|
123
|
+
default=None,
|
124
|
+
help="Whether you want to limit the instances returned to either 'running', 'stopped'. Default is both",
|
125
|
+
)
|
126
|
+
return parser.my_parser.parse_args(f_arguments)
|
127
|
+
|
128
|
+
|
129
|
+
def find_all_instances(fAllCredentials: list, fStatus: str = None) -> list:
|
130
|
+
"""Discover EC2 instances across multiple AWS accounts and regions.
|
131
|
+
|
132
|
+
Performs parallel discovery of EC2 instances using multi-threading for optimal
|
133
|
+
performance across large AWS Organizations. Supports optional state filtering
|
134
|
+
to focus on specific instance lifecycle phases.
|
135
|
+
|
136
|
+
Args:
|
137
|
+
fAllCredentials (list): List of credential dictionaries containing:
|
138
|
+
- AccountId: AWS account identifier
|
139
|
+
- Region: AWS region name
|
140
|
+
- AccessKey, SecretKey, SessionToken: AWS credentials
|
141
|
+
- MgmtAccount: Management account flag
|
142
|
+
fStatus (str, optional): Instance state filter. Valid values:
|
143
|
+
- 'running': Only return running instances
|
144
|
+
- 'stopped': Only return stopped instances
|
145
|
+
- None: Return instances in all states
|
146
|
+
|
147
|
+
Returns:
|
148
|
+
list: Collection of EC2 instance dictionaries with attributes:
|
149
|
+
- InstanceId: EC2 instance identifier
|
150
|
+
- InstanceType: EC2 instance type (e.g., 't3.micro')
|
151
|
+
- State: Current instance state
|
152
|
+
- VpcId: Associated VPC identifier
|
153
|
+
- SubnetId: Associated subnet identifier
|
154
|
+
- PrivateIpAddress: Private IP address
|
155
|
+
- PublicIpAddress: Public IP address (if assigned)
|
156
|
+
- Tags: Instance tags as key-value pairs
|
157
|
+
- LaunchTime: Instance launch timestamp
|
158
|
+
- AccountId: Source AWS account
|
159
|
+
- Region: Source AWS region
|
160
|
+
|
161
|
+
Raises:
|
162
|
+
ClientError: When AWS API calls fail due to permissions or service issues
|
163
|
+
|
164
|
+
Example:
|
165
|
+
>>> credentials = get_all_credentials(['my-profile'])
|
166
|
+
>>> running_instances = find_all_instances(credentials, 'running')
|
167
|
+
>>> print(f"Found {len(running_instances)} running instances")
|
168
|
+
|
169
|
+
Note:
|
170
|
+
Uses ThreadPool for concurrent execution across regions/accounts.
|
171
|
+
Progress tracking provided via tqdm when verbose logging enabled.
|
172
|
+
"""
|
173
|
+
|
174
|
+
class FindInstances(Thread):
|
175
|
+
"""
|
176
|
+
Worker thread for concurrent EC2 instance discovery across AWS accounts.
|
177
|
+
|
178
|
+
Each worker thread processes credential sets from the shared queue,
|
179
|
+
calls AWS EC2 APIs to discover compute instances, and performs detailed
|
180
|
+
metadata extraction including networking, tagging, and state analysis.
|
181
|
+
|
182
|
+
Compute Discovery Capabilities:
|
183
|
+
- EC2 instance enumeration with comprehensive metadata
|
184
|
+
- Instance state and lifecycle analysis
|
185
|
+
- Network configuration discovery (VPC, subnet, IP addresses)
|
186
|
+
- Tag-based instance categorization and naming
|
187
|
+
- Multi-account compute inventory aggregation
|
188
|
+
"""
|
189
|
+
|
190
|
+
def __init__(self, queue):
|
191
|
+
"""
|
192
|
+
Initialize worker thread with reference to shared work queue.
|
193
|
+
|
194
|
+
Args:
|
195
|
+
queue (Queue): Thread-safe queue containing EC2 discovery work items
|
196
|
+
"""
|
197
|
+
Thread.__init__(self)
|
198
|
+
self.queue = queue
|
199
|
+
|
200
|
+
def run(self):
|
201
|
+
"""
|
202
|
+
Main worker thread execution loop for EC2 instance discovery.
|
203
|
+
|
204
|
+
Continuously processes credential sets from queue, performs instance
|
205
|
+
discovery via AWS EC2 APIs, and aggregates compute infrastructure data
|
206
|
+
with comprehensive metadata extraction and state filtering.
|
207
|
+
"""
|
208
|
+
while True:
|
209
|
+
# Get EC2 discovery work item from thread-safe queue
|
210
|
+
c_account_credentials = self.queue.get()
|
211
|
+
logging.info(f"De-queued info for account number {c_account_credentials['AccountId']}")
|
212
|
+
|
213
|
+
try:
|
214
|
+
# Call AWS EC2 API to discover instances in this account/region
|
215
|
+
# find_account_instances2() handles DescribeInstances API with pagination
|
216
|
+
# This is the most time-intensive operation in the discovery process
|
217
|
+
Instances = Inventory_Modules.find_account_instances2(c_account_credentials)
|
218
|
+
logging.info(
|
219
|
+
f"Account: {c_account_credentials['AccountId']} Region: {c_account_credentials['Region']} | Found {len(Instances['Reservations'])} reservations"
|
220
|
+
)
|
221
|
+
|
222
|
+
# Initialize instance metadata variables with defaults
|
223
|
+
State = InstanceType = InstanceId = PublicDnsName = Name = ""
|
224
|
+
|
225
|
+
# Process discovered EC2 instances with comprehensive metadata extraction
|
226
|
+
# AWS returns instances grouped by reservation (launch request)
|
227
|
+
if "Reservations" in Instances.keys():
|
228
|
+
for y in range(len(Instances["Reservations"])):
|
229
|
+
# Each reservation can contain multiple instances
|
230
|
+
for z in range(len(Instances["Reservations"][y]["Instances"])):
|
231
|
+
# Extract core instance attributes
|
232
|
+
instance_data = Instances["Reservations"][y]["Instances"][z]
|
233
|
+
|
234
|
+
InstanceType = instance_data["InstanceType"]
|
235
|
+
InstanceId = instance_data["InstanceId"]
|
236
|
+
State = instance_data["State"]["Name"]
|
237
|
+
|
238
|
+
# Handle optional public DNS name (depends on network configuration)
|
239
|
+
PublicDnsName = (
|
240
|
+
instance_data["PublicDnsName"]
|
241
|
+
if "PublicDnsName" in instance_data
|
242
|
+
else "No Public DNS Name"
|
243
|
+
)
|
244
|
+
|
245
|
+
# Initialize name with fallback for untagged instances
|
246
|
+
Name = "No Name Tag"
|
247
|
+
# Extract instance name from tags for resource identification
|
248
|
+
# Proper instance naming is essential for operational visibility
|
249
|
+
try:
|
250
|
+
if "Tags" in instance_data:
|
251
|
+
for tag in instance_data["Tags"]:
|
252
|
+
if tag["Key"] == "Name":
|
253
|
+
Name = tag["Value"]
|
254
|
+
break
|
255
|
+
except KeyError as my_Error:
|
256
|
+
# Handle cases where Tags key is missing from API response
|
257
|
+
# This can occur with instances launched without tags
|
258
|
+
logging.info(f"No tags found for instance {InstanceId}: {my_Error}")
|
259
|
+
pass
|
260
|
+
# Apply state filtering if specified
|
261
|
+
# Common filters: 'running' for active workloads, 'stopped' for cost analysis
|
262
|
+
if fStatus is None or fStatus == State:
|
263
|
+
# Create comprehensive instance record for inventory
|
264
|
+
instance_record = {
|
265
|
+
# Organizational context
|
266
|
+
"MgmtAccount": c_account_credentials["MgmtAccount"],
|
267
|
+
"AccountId": c_account_credentials["AccountId"],
|
268
|
+
"Region": c_account_credentials["Region"],
|
269
|
+
"ParentProfile": c_account_credentials["ParentProfile"],
|
270
|
+
# Instance identification and metadata
|
271
|
+
"InstanceId": InstanceId,
|
272
|
+
"Name": Name,
|
273
|
+
# Compute and operational attributes
|
274
|
+
"InstanceType": InstanceType,
|
275
|
+
"State": State,
|
276
|
+
# Network configuration
|
277
|
+
"PublicDNSName": PublicDnsName,
|
278
|
+
}
|
279
|
+
|
280
|
+
# Add to global inventory collection
|
281
|
+
AllInstances.append(instance_record)
|
282
|
+
else:
|
283
|
+
# Skip instances that don't match state filter
|
284
|
+
continue
|
285
|
+
except KeyError as my_Error:
|
286
|
+
# Handle credential or account access configuration errors
|
287
|
+
logging.error(f"Account Access failed - trying to access {c_account_credentials['AccountId']}")
|
288
|
+
logging.info(f"Actual Error: {my_Error}")
|
289
|
+
# Continue processing other accounts despite this failure
|
290
|
+
pass
|
291
|
+
|
292
|
+
except AttributeError as my_Error:
|
293
|
+
# Handle profile configuration or credential format errors
|
294
|
+
logging.error(f"Error: Likely that one of the supplied profiles was wrong")
|
295
|
+
logging.warning(my_Error)
|
296
|
+
continue
|
297
|
+
|
298
|
+
except ClientError as my_Error:
|
299
|
+
# Handle AWS API authentication and authorization errors
|
300
|
+
if "AuthFailure" in str(my_Error):
|
301
|
+
logging.error(
|
302
|
+
f"Authorization Failure accessing account {c_account_credentials['AccountId']} in {c_account_credentials['Region']} region"
|
303
|
+
)
|
304
|
+
logging.warning(
|
305
|
+
f"It's possible that the region {c_account_credentials['Region']} hasn't been opted-into"
|
306
|
+
)
|
307
|
+
continue
|
308
|
+
else:
|
309
|
+
# Handle API throttling and other service errors
|
310
|
+
logging.error(f"Error: Likely throttling errors from too much activity")
|
311
|
+
logging.warning(my_Error)
|
312
|
+
continue
|
313
|
+
|
314
|
+
finally:
|
315
|
+
# Always mark work item as complete for queue management
|
316
|
+
self.queue.task_done()
|
317
|
+
|
318
|
+
###########
|
319
|
+
|
320
|
+
checkqueue = Queue()
|
321
|
+
|
322
|
+
AllInstances = []
|
323
|
+
WorkerThreads = min(len(fAllCredentials), 25)
|
324
|
+
|
325
|
+
pbar = tqdm(
|
326
|
+
desc=f"Finding instances from {len(fAllCredentials)} locations", total=len(fAllCredentials), unit=" locations"
|
327
|
+
)
|
328
|
+
|
329
|
+
for x in range(WorkerThreads):
|
330
|
+
worker = FindInstances(checkqueue)
|
331
|
+
# Setting daemon to True will let the main thread exit even though the workers are blocking
|
332
|
+
worker.daemon = True
|
333
|
+
worker.start()
|
334
|
+
|
335
|
+
for credential in fAllCredentials:
|
336
|
+
logging.info(f"Beginning to queue data - starting with {credential['AccountId']}")
|
337
|
+
try:
|
338
|
+
# I don't know why - but double parens are necessary below. If you remove them, only the first parameter is queued.
|
339
|
+
checkqueue.put((credential))
|
340
|
+
except ClientError as my_Error:
|
341
|
+
if "AuthFailure" in str(my_Error):
|
342
|
+
logging.error(
|
343
|
+
f"Authorization Failure accessing account {credential['AccountId']} in {credential['Region']} region"
|
344
|
+
)
|
345
|
+
logging.warning(f"It's possible that the region {credential['Region']} hasn't been opted-into")
|
346
|
+
pass
|
347
|
+
checkqueue.join()
|
348
|
+
pbar.close()
|
349
|
+
return AllInstances
|
350
|
+
|
351
|
+
|
352
|
+
##################
|
353
|
+
# Main
|
354
|
+
##################
|
355
|
+
|
356
|
+
if __name__ == "__main__":
|
357
|
+
args = parse_args(sys.argv[1:])
|
358
|
+
pProfiles = args.Profiles
|
359
|
+
pRegionList = args.Regions
|
360
|
+
pAccounts = args.Accounts
|
361
|
+
pSkipAccounts = args.SkipAccounts
|
362
|
+
pSkipProfiles = args.SkipProfiles
|
363
|
+
pAccessRoles = args.AccessRoles
|
364
|
+
pStatus = args.pStatus
|
365
|
+
pRootOnly = args.RootOnly
|
366
|
+
pFilename = args.Filename
|
367
|
+
pTiming = args.Time
|
368
|
+
verbose = args.loglevel
|
369
|
+
# Setup logging levels
|
370
|
+
logging.basicConfig(level=verbose, format="[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
|
371
|
+
logging.getLogger("boto3").setLevel(logging.CRITICAL)
|
372
|
+
logging.getLogger("botocore").setLevel(logging.CRITICAL)
|
373
|
+
logging.getLogger("s3transfer").setLevel(logging.CRITICAL)
|
374
|
+
logging.getLogger("urllib3").setLevel(logging.CRITICAL)
|
375
|
+
|
376
|
+
print()
|
377
|
+
print(f"Checking for instances... ")
|
378
|
+
print()
|
379
|
+
|
380
|
+
# Find credentials for all Child Accounts
|
381
|
+
# CredentialList = get_credentials(pProfiles, pRegionList, pSkipProfiles, pSkipAccounts, pRootOnly, pAccounts, pAccessRoles, pTiming)
|
382
|
+
CredentialList = get_all_credentials(
|
383
|
+
pProfiles, pTiming, pSkipProfiles, pSkipAccounts, pRootOnly, pAccounts, pRegionList, pAccessRoles
|
384
|
+
)
|
385
|
+
AccountNum = len(set([acct["AccountId"] for acct in CredentialList]))
|
386
|
+
RegionNum = len(set([acct["Region"] for acct in CredentialList]))
|
387
|
+
print()
|
388
|
+
print(f"Searching total of {AccountNum} accounts and {RegionNum} regions")
|
389
|
+
if pTiming:
|
390
|
+
print()
|
391
|
+
milestone_time1 = time()
|
392
|
+
print(
|
393
|
+
f"{Fore.GREEN}\t\tFiguring out what regions are available to your accounts, and capturing credentials for all accounts in those regions took: {(milestone_time1 - begin_time):.3f} seconds{Fore.RESET}"
|
394
|
+
)
|
395
|
+
print()
|
396
|
+
print(f"Now running through all accounts and regions identified to find resources...")
|
397
|
+
# Collect all the instances from the credentials found
|
398
|
+
AllInstances = find_all_instances(CredentialList, pStatus)
|
399
|
+
# Display the information we've found thus far
|
400
|
+
display_dict = {
|
401
|
+
"ParentProfile": {"DisplayOrder": 1, "Heading": "Parent Profile"},
|
402
|
+
"MgmtAccount": {"DisplayOrder": 2, "Heading": "Mgmt Acct"},
|
403
|
+
"AccountId": {"DisplayOrder": 3, "Heading": "Acct Number"},
|
404
|
+
"Region": {"DisplayOrder": 4, "Heading": "Region"},
|
405
|
+
"InstanceType": {"DisplayOrder": 5, "Heading": "Instance Type"},
|
406
|
+
"Name": {"DisplayOrder": 6, "Heading": "Name"},
|
407
|
+
"InstanceId": {"DisplayOrder": 7, "Heading": "Instance ID"},
|
408
|
+
"PublicDNSName": {"DisplayOrder": 8, "Heading": "Public Name"},
|
409
|
+
"State": {"DisplayOrder": 9, "Heading": "State", "Condition": ["running"]},
|
410
|
+
}
|
411
|
+
|
412
|
+
sorted_all_instances = sorted(
|
413
|
+
AllInstances, key=lambda d: (d["ParentProfile"], d["MgmtAccount"], d["Region"], d["AccountId"])
|
414
|
+
)
|
415
|
+
display_results(sorted_all_instances, display_dict, None, pFilename)
|
416
|
+
|
417
|
+
if pTiming:
|
418
|
+
print(ERASE_LINE)
|
419
|
+
print(f"{Fore.GREEN}This script took {time() - begin_time:.2f} seconds{Fore.RESET}")
|
420
|
+
print(ERASE_LINE)
|
421
|
+
|
422
|
+
print(f"Found {len(AllInstances)} instances across {AccountNum} accounts across {RegionNum} regions")
|
423
|
+
print()
|
424
|
+
print("Thank you for using this script")
|
425
|
+
print()
|