runbooks 0.2.3__py3-none-any.whl → 0.6.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. conftest.py +26 -0
  2. jupyter-agent/.env.template +2 -0
  3. jupyter-agent/.gitattributes +35 -0
  4. jupyter-agent/README.md +16 -0
  5. jupyter-agent/app.py +256 -0
  6. jupyter-agent/cloudops-agent.png +0 -0
  7. jupyter-agent/ds-system-prompt.txt +154 -0
  8. jupyter-agent/jupyter-agent.png +0 -0
  9. jupyter-agent/llama3_template.jinja +123 -0
  10. jupyter-agent/requirements.txt +9 -0
  11. jupyter-agent/utils.py +409 -0
  12. runbooks/__init__.py +71 -3
  13. runbooks/__main__.py +13 -0
  14. runbooks/aws/ec2_describe_instances.py +1 -1
  15. runbooks/aws/ec2_run_instances.py +8 -2
  16. runbooks/aws/ec2_start_stop_instances.py +17 -4
  17. runbooks/aws/ec2_unused_volumes.py +5 -1
  18. runbooks/aws/s3_create_bucket.py +4 -2
  19. runbooks/aws/s3_list_objects.py +6 -1
  20. runbooks/aws/tagging_lambda_handler.py +13 -2
  21. runbooks/aws/tags.json +12 -0
  22. runbooks/base.py +353 -0
  23. runbooks/cfat/README.md +49 -0
  24. runbooks/cfat/__init__.py +74 -0
  25. runbooks/cfat/app.ts +644 -0
  26. runbooks/cfat/assessment/__init__.py +40 -0
  27. runbooks/cfat/assessment/asana-import.csv +39 -0
  28. runbooks/cfat/assessment/cfat-checks.csv +31 -0
  29. runbooks/cfat/assessment/cfat.txt +520 -0
  30. runbooks/cfat/assessment/collectors.py +200 -0
  31. runbooks/cfat/assessment/jira-import.csv +39 -0
  32. runbooks/cfat/assessment/runner.py +387 -0
  33. runbooks/cfat/assessment/validators.py +290 -0
  34. runbooks/cfat/cli.py +103 -0
  35. runbooks/cfat/docs/asana-import.csv +24 -0
  36. runbooks/cfat/docs/cfat-checks.csv +31 -0
  37. runbooks/cfat/docs/cfat.txt +335 -0
  38. runbooks/cfat/docs/checks-output.png +0 -0
  39. runbooks/cfat/docs/cloudshell-console-run.png +0 -0
  40. runbooks/cfat/docs/cloudshell-download.png +0 -0
  41. runbooks/cfat/docs/cloudshell-output.png +0 -0
  42. runbooks/cfat/docs/downloadfile.png +0 -0
  43. runbooks/cfat/docs/jira-import.csv +24 -0
  44. runbooks/cfat/docs/open-cloudshell.png +0 -0
  45. runbooks/cfat/docs/report-header.png +0 -0
  46. runbooks/cfat/models.py +1026 -0
  47. runbooks/cfat/package-lock.json +5116 -0
  48. runbooks/cfat/package.json +38 -0
  49. runbooks/cfat/report.py +496 -0
  50. runbooks/cfat/reporting/__init__.py +46 -0
  51. runbooks/cfat/reporting/exporters.py +337 -0
  52. runbooks/cfat/reporting/formatters.py +496 -0
  53. runbooks/cfat/reporting/templates.py +135 -0
  54. runbooks/cfat/run-assessment.sh +23 -0
  55. runbooks/cfat/runner.py +69 -0
  56. runbooks/cfat/src/actions/check-cloudtrail-existence.ts +43 -0
  57. runbooks/cfat/src/actions/check-config-existence.ts +37 -0
  58. runbooks/cfat/src/actions/check-control-tower.ts +37 -0
  59. runbooks/cfat/src/actions/check-ec2-existence.ts +46 -0
  60. runbooks/cfat/src/actions/check-iam-users.ts +50 -0
  61. runbooks/cfat/src/actions/check-legacy-cur.ts +30 -0
  62. runbooks/cfat/src/actions/check-org-cloudformation.ts +30 -0
  63. runbooks/cfat/src/actions/check-vpc-existence.ts +43 -0
  64. runbooks/cfat/src/actions/create-asanaimport.ts +14 -0
  65. runbooks/cfat/src/actions/create-backlog.ts +372 -0
  66. runbooks/cfat/src/actions/create-jiraimport.ts +15 -0
  67. runbooks/cfat/src/actions/create-report.ts +616 -0
  68. runbooks/cfat/src/actions/define-account-type.ts +51 -0
  69. runbooks/cfat/src/actions/get-enabled-org-policy-types.ts +40 -0
  70. runbooks/cfat/src/actions/get-enabled-org-services.ts +26 -0
  71. runbooks/cfat/src/actions/get-idc-info.ts +34 -0
  72. runbooks/cfat/src/actions/get-org-da-accounts.ts +34 -0
  73. runbooks/cfat/src/actions/get-org-details.ts +35 -0
  74. runbooks/cfat/src/actions/get-org-member-accounts.ts +44 -0
  75. runbooks/cfat/src/actions/get-org-ous.ts +35 -0
  76. runbooks/cfat/src/actions/get-regions.ts +22 -0
  77. runbooks/cfat/src/actions/zip-assessment.ts +27 -0
  78. runbooks/cfat/src/types/index.d.ts +147 -0
  79. runbooks/cfat/tests/__init__.py +141 -0
  80. runbooks/cfat/tests/test_cli.py +340 -0
  81. runbooks/cfat/tests/test_integration.py +290 -0
  82. runbooks/cfat/tests/test_models.py +505 -0
  83. runbooks/cfat/tests/test_reporting.py +354 -0
  84. runbooks/cfat/tsconfig.json +16 -0
  85. runbooks/cfat/webpack.config.cjs +27 -0
  86. runbooks/config.py +260 -0
  87. runbooks/finops/__init__.py +88 -0
  88. runbooks/finops/aws_client.py +245 -0
  89. runbooks/finops/cli.py +151 -0
  90. runbooks/finops/cost_processor.py +410 -0
  91. runbooks/finops/dashboard_runner.py +448 -0
  92. runbooks/finops/helpers.py +355 -0
  93. runbooks/finops/main.py +14 -0
  94. runbooks/finops/profile_processor.py +174 -0
  95. runbooks/finops/types.py +66 -0
  96. runbooks/finops/visualisations.py +80 -0
  97. runbooks/inventory/.gitignore +354 -0
  98. runbooks/inventory/ArgumentsClass.py +261 -0
  99. runbooks/inventory/Inventory_Modules.py +6130 -0
  100. runbooks/inventory/LandingZone/delete_lz.py +1075 -0
  101. runbooks/inventory/README.md +1320 -0
  102. runbooks/inventory/__init__.py +62 -0
  103. runbooks/inventory/account_class.py +532 -0
  104. runbooks/inventory/all_my_instances_wrapper.py +123 -0
  105. runbooks/inventory/aws_decorators.py +201 -0
  106. runbooks/inventory/cfn_move_stack_instances.py +1526 -0
  107. runbooks/inventory/check_cloudtrail_compliance.py +614 -0
  108. runbooks/inventory/check_controltower_readiness.py +1107 -0
  109. runbooks/inventory/check_landingzone_readiness.py +711 -0
  110. runbooks/inventory/cloudtrail.md +727 -0
  111. runbooks/inventory/collectors/__init__.py +20 -0
  112. runbooks/inventory/collectors/aws_compute.py +518 -0
  113. runbooks/inventory/collectors/aws_networking.py +275 -0
  114. runbooks/inventory/collectors/base.py +222 -0
  115. runbooks/inventory/core/__init__.py +19 -0
  116. runbooks/inventory/core/collector.py +303 -0
  117. runbooks/inventory/core/formatter.py +296 -0
  118. runbooks/inventory/delete_s3_buckets_objects.py +169 -0
  119. runbooks/inventory/discovery.md +81 -0
  120. runbooks/inventory/draw_org_structure.py +748 -0
  121. runbooks/inventory/ec2_vpc_utils.py +341 -0
  122. runbooks/inventory/find_cfn_drift_detection.py +272 -0
  123. runbooks/inventory/find_cfn_orphaned_stacks.py +719 -0
  124. runbooks/inventory/find_cfn_stackset_drift.py +733 -0
  125. runbooks/inventory/find_ec2_security_groups.py +669 -0
  126. runbooks/inventory/find_landingzone_versions.py +201 -0
  127. runbooks/inventory/find_vpc_flow_logs.py +1221 -0
  128. runbooks/inventory/inventory.sh +659 -0
  129. runbooks/inventory/list_cfn_stacks.py +558 -0
  130. runbooks/inventory/list_cfn_stackset_operation_results.py +252 -0
  131. runbooks/inventory/list_cfn_stackset_operations.py +734 -0
  132. runbooks/inventory/list_cfn_stacksets.py +453 -0
  133. runbooks/inventory/list_config_recorders_delivery_channels.py +681 -0
  134. runbooks/inventory/list_ds_directories.py +354 -0
  135. runbooks/inventory/list_ec2_availability_zones.py +286 -0
  136. runbooks/inventory/list_ec2_ebs_volumes.py +244 -0
  137. runbooks/inventory/list_ec2_instances.py +425 -0
  138. runbooks/inventory/list_ecs_clusters_and_tasks.py +562 -0
  139. runbooks/inventory/list_elbs_load_balancers.py +411 -0
  140. runbooks/inventory/list_enis_network_interfaces.py +526 -0
  141. runbooks/inventory/list_guardduty_detectors.py +568 -0
  142. runbooks/inventory/list_iam_policies.py +404 -0
  143. runbooks/inventory/list_iam_roles.py +518 -0
  144. runbooks/inventory/list_iam_saml_providers.py +359 -0
  145. runbooks/inventory/list_lambda_functions.py +882 -0
  146. runbooks/inventory/list_org_accounts.py +446 -0
  147. runbooks/inventory/list_org_accounts_users.py +354 -0
  148. runbooks/inventory/list_rds_db_instances.py +406 -0
  149. runbooks/inventory/list_route53_hosted_zones.py +318 -0
  150. runbooks/inventory/list_servicecatalog_provisioned_products.py +575 -0
  151. runbooks/inventory/list_sns_topics.py +360 -0
  152. runbooks/inventory/list_ssm_parameters.py +402 -0
  153. runbooks/inventory/list_vpc_subnets.py +433 -0
  154. runbooks/inventory/list_vpcs.py +422 -0
  155. runbooks/inventory/lockdown_cfn_stackset_role.py +224 -0
  156. runbooks/inventory/models/__init__.py +24 -0
  157. runbooks/inventory/models/account.py +192 -0
  158. runbooks/inventory/models/inventory.py +309 -0
  159. runbooks/inventory/models/resource.py +247 -0
  160. runbooks/inventory/recover_cfn_stack_ids.py +205 -0
  161. runbooks/inventory/requirements.txt +12 -0
  162. runbooks/inventory/run_on_multi_accounts.py +211 -0
  163. runbooks/inventory/tests/common_test_data.py +3661 -0
  164. runbooks/inventory/tests/common_test_functions.py +204 -0
  165. runbooks/inventory/tests/script_test_data.py +0 -0
  166. runbooks/inventory/tests/setup.py +24 -0
  167. runbooks/inventory/tests/src.py +18 -0
  168. runbooks/inventory/tests/test_cfn_describe_stacks.py +208 -0
  169. runbooks/inventory/tests/test_ec2_describe_instances.py +162 -0
  170. runbooks/inventory/tests/test_inventory_modules.py +55 -0
  171. runbooks/inventory/tests/test_lambda_list_functions.py +86 -0
  172. runbooks/inventory/tests/test_moto_integration_example.py +273 -0
  173. runbooks/inventory/tests/test_org_list_accounts.py +49 -0
  174. runbooks/inventory/update_aws_actions.py +173 -0
  175. runbooks/inventory/update_cfn_stacksets.py +1215 -0
  176. runbooks/inventory/update_cloudwatch_logs_retention_policy.py +294 -0
  177. runbooks/inventory/update_iam_roles_cross_accounts.py +478 -0
  178. runbooks/inventory/update_s3_public_access_block.py +539 -0
  179. runbooks/inventory/utils/__init__.py +23 -0
  180. runbooks/inventory/utils/aws_helpers.py +510 -0
  181. runbooks/inventory/utils/threading_utils.py +493 -0
  182. runbooks/inventory/utils/validation.py +682 -0
  183. runbooks/inventory/verify_ec2_security_groups.py +1430 -0
  184. runbooks/main.py +785 -0
  185. runbooks/organizations/__init__.py +12 -0
  186. runbooks/organizations/manager.py +374 -0
  187. runbooks/security_baseline/README.md +324 -0
  188. runbooks/security_baseline/checklist/alternate_contacts.py +8 -1
  189. runbooks/security_baseline/checklist/bucket_public_access.py +4 -1
  190. runbooks/security_baseline/checklist/cloudwatch_alarm_configuration.py +9 -2
  191. runbooks/security_baseline/checklist/guardduty_enabled.py +9 -2
  192. runbooks/security_baseline/checklist/multi_region_instance_usage.py +5 -1
  193. runbooks/security_baseline/checklist/root_access_key.py +6 -1
  194. runbooks/security_baseline/config-origin.json +1 -1
  195. runbooks/security_baseline/config.json +1 -1
  196. runbooks/security_baseline/permission.json +1 -1
  197. runbooks/security_baseline/report_generator.py +10 -2
  198. runbooks/security_baseline/report_template_en.html +8 -8
  199. runbooks/security_baseline/report_template_jp.html +8 -8
  200. runbooks/security_baseline/report_template_kr.html +13 -13
  201. runbooks/security_baseline/report_template_vn.html +8 -8
  202. runbooks/security_baseline/requirements.txt +7 -0
  203. runbooks/security_baseline/run_script.py +8 -2
  204. runbooks/security_baseline/security_baseline_tester.py +10 -2
  205. runbooks/security_baseline/utils/common.py +5 -1
  206. runbooks/utils/__init__.py +204 -0
  207. runbooks-0.6.1.dist-info/METADATA +373 -0
  208. runbooks-0.6.1.dist-info/RECORD +237 -0
  209. {runbooks-0.2.3.dist-info → runbooks-0.6.1.dist-info}/WHEEL +1 -1
  210. runbooks-0.6.1.dist-info/entry_points.txt +7 -0
  211. runbooks-0.6.1.dist-info/licenses/LICENSE +201 -0
  212. runbooks-0.6.1.dist-info/top_level.txt +3 -0
  213. runbooks/python101/calculator.py +0 -34
  214. runbooks/python101/config.py +0 -1
  215. runbooks/python101/exceptions.py +0 -16
  216. runbooks/python101/file_manager.py +0 -218
  217. runbooks/python101/toolkit.py +0 -153
  218. runbooks-0.2.3.dist-info/METADATA +0 -435
  219. runbooks-0.2.3.dist-info/RECORD +0 -61
  220. runbooks-0.2.3.dist-info/entry_points.txt +0 -3
  221. runbooks-0.2.3.dist-info/top_level.txt +0 -1
@@ -0,0 +1,244 @@
1
+ #!/usr/bin/env python3
2
+ import logging
3
+ import sys
4
+ from queue import Queue
5
+ from threading import Thread
6
+ from time import time
7
+
8
+ from ArgumentsClass import CommonArguments
9
+ from botocore.exceptions import ClientError
10
+ from colorama import Fore, init
11
+ from Inventory_Modules import display_results, find_account_volumes2, get_all_credentials
12
+ from tqdm.auto import tqdm
13
+
14
+ init()
15
+ __version__ = "2024.05.31"
16
+
17
+ # ANSI escape code for clearing current line (progress bar cleanup)
18
+ ERASE_LINE = "\x1b[2K"
19
+
20
+
21
+ ##################
22
+ def parse_args(f_arguments):
23
+ """
24
+ Description: Parses the arguments passed into the script
25
+ @param f_arguments: args represents the list of arguments passed in
26
+ @return: returns an object namespace that contains the individualized parameters passed in
27
+ """
28
+
29
+ parser = CommonArguments()
30
+ parser.multiprofile()
31
+ parser.multiregion()
32
+ parser.extendedargs()
33
+ parser.fragment()
34
+ parser.rootOnly()
35
+ parser.save_to_file()
36
+ parser.timing()
37
+ parser.verbosity()
38
+ parser.version(__version__)
39
+ return parser.my_parser.parse_args(f_arguments)
40
+
41
+
42
+ def present_results(fVolumesFound: list):
43
+ """
44
+ Display comprehensive results of EBS volume discovery with analysis.
45
+
46
+ This function processes the discovered volumes, removes duplicates,
47
+ sorts them logically, and presents them in a formatted output with
48
+ summary statistics and operational insights.
49
+
50
+ Args:
51
+ fVolumesFound (list): List of discovered EBS volumes with metadata
52
+ """
53
+
54
+ display_dict = {
55
+ "VolumeId": {"DisplayOrder": 1, "Heading": "Volume ID"},
56
+ "Size": {"DisplayOrder": 2, "Heading": "Size (GB)"},
57
+ "VolumeType": {"DisplayOrder": 3, "Heading": "Volume Type"},
58
+ "VolumeName": {"DisplayOrder": 4, "Heading": "Volume Name"},
59
+ "InstanceId": {"DisplayOrder": 5, "Heading": "Instance ID"},
60
+ "Encrypted": {"DisplayOrder": 6, "Heading": "Encrypted"},
61
+ "State": {"DisplayOrder": 7, "Heading": "State"},
62
+ "AccountId": {"DisplayOrder": 8, "Heading": "Account"},
63
+ "Region": {"DisplayOrder": 9, "Heading": "Region"},
64
+ }
65
+
66
+ # Phase 1: Data deduplication and preparation
67
+ de_dupe_VolumesFound = []
68
+ AccountsFound = set()
69
+ RegionsFound = set()
70
+
71
+ # Remove duplicate volumes based on VolumeId and collect unique accounts/regions
72
+ seen = set()
73
+ for volume in fVolumesFound:
74
+ key = volume["VolumeId"]
75
+ if key not in seen:
76
+ seen.add(key)
77
+ de_dupe_VolumesFound.append(volume)
78
+ AccountsFound.add(volume.get("AccountId"))
79
+ RegionsFound.add(volume.get("Region"))
80
+
81
+ sorted_Volumes_Found = sorted(
82
+ de_dupe_VolumesFound, key=lambda x: (x["MgmtAccount"], x["AccountId"], x["Region"], x["VolumeName"], x["Size"])
83
+ )
84
+ display_results(sorted_Volumes_Found, display_dict, "None", None)
85
+
86
+ print()
87
+ print(
88
+ f"Found {len(de_dupe_VolumesFound)} volumes across {len(AccountsFound)} account{'' if len(AccountsFound) == 1 else 's'} "
89
+ f"across {len(RegionsFound)} region{'' if len(RegionsFound) == 1 else 's'}"
90
+ )
91
+
92
+ # Calculate and display orphaned volumes
93
+ orphaned_volumes = [vol for vol in de_dupe_VolumesFound if vol.get("State") == "available"]
94
+ if orphaned_volumes:
95
+ print(f"{Fore.YELLOW}Warning: {len(orphaned_volumes)} orphaned (unattached) volumes found{Fore.RESET}")
96
+ total_orphaned_size = sum(vol.get("Size", 0) for vol in orphaned_volumes)
97
+ print(f"Total orphaned storage: {total_orphaned_size} GB")
98
+
99
+ print()
100
+
101
+
102
+ def check_accounts_for_ebs_volumes(f_CredentialList, f_fragment_list=None):
103
+ """
104
+ Note that this function takes a list of Credentials and checks for EBS Volumes in every account it has creds for
105
+ @param f_CredentialList: List of credentials for all accounts to check
106
+ @param f_fragment_list: List of name tag fragments to limit the searching to
107
+ @return:
108
+ """
109
+
110
+ class FindVolumes(Thread):
111
+ def __init__(self, queue):
112
+ Thread.__init__(self)
113
+ self.queue = queue
114
+
115
+ def run(self):
116
+ while True:
117
+ # Get the work from the queue and expand the tuple
118
+ # c_account_credentials, c_region, c_text_to_find, c_PlacesToLook, c_PlaceCount = self.queue.get()
119
+ c_account_credentials, c_region, c_fragment = self.queue.get()
120
+ logging.info(f"De-queued info for account {c_account_credentials['AccountId']}")
121
+ try:
122
+ logging.info(f"Attempting to connect to {c_account_credentials['AccountId']}")
123
+ # account_volumes = find_account_volumes2(c_account_credentials, c_text_to_find)
124
+ account_volumes = find_account_volumes2(c_account_credentials)
125
+ logging.info(f"Successfully connected to account {c_account_credentials['AccountId']}")
126
+ for _ in range(len(account_volumes)):
127
+ account_volumes[_]["MgmtAccount"] = c_account_credentials["MgmtAccount"]
128
+ AllVolumes.extend(account_volumes)
129
+ except KeyError as my_Error:
130
+ logging.error(f"Account Access failed - trying to access {c_account_credentials['AccountId']}")
131
+ logging.info(f"Actual Error: {my_Error}")
132
+ pass
133
+ except AttributeError as my_Error:
134
+ logging.error(f"Error: Likely that one of the supplied profiles {pProfiles} was wrong")
135
+ logging.warning(my_Error)
136
+ continue
137
+ finally:
138
+ logging.info(
139
+ f"{ERASE_LINE}Finished finding EBS volumes in account {c_account_credentials['AccountId']} in region {c_account_credentials['Region']}"
140
+ )
141
+ pbar.update()
142
+ self.queue.task_done()
143
+
144
+ if f_fragment_list is None:
145
+ f_fragment_list = []
146
+ AllVolumes = []
147
+ WorkerThreads = min(len(f_CredentialList), 50)
148
+
149
+ checkqueue = Queue()
150
+
151
+ pbar = tqdm(
152
+ desc=f"Finding ebs volumes from {len(f_CredentialList)} accounts and regions",
153
+ total=len(f_CredentialList),
154
+ unit=" accounts & regions",
155
+ )
156
+
157
+ for x in range(WorkerThreads):
158
+ worker = FindVolumes(checkqueue)
159
+ # Setting daemon to True will let the main thread exit even though the workers are blocking
160
+ worker.daemon = True
161
+ worker.start()
162
+
163
+ for credential in f_CredentialList:
164
+ logging.info(f"Connecting to account {credential['AccountId']}")
165
+ try:
166
+ # print(f"{ERASE_LINE}Queuing account {credential['AccountId']} in region {region}", end='\r')
167
+ checkqueue.put((credential, credential["Region"], f_fragment_list))
168
+ except ClientError as my_Error:
169
+ if "AuthFailure" in str(my_Error):
170
+ logging.error(
171
+ f"Authorization Failure accessing account {credential['AccountId']} in '{credential['Region']}' region"
172
+ )
173
+ logging.warning(f"It's possible that the region '{credential['Region']}' hasn't been opted-into")
174
+ pass
175
+ checkqueue.join()
176
+ pbar.close()
177
+ return AllVolumes
178
+
179
+
180
+ ##################
181
+
182
+
183
+ def main():
184
+ """
185
+ Main execution function for EBS volume inventory discovery.
186
+ Orchestrates argument parsing, credential discovery, volume enumeration, and result presentation.
187
+ """
188
+ import sys
189
+
190
+ args = parse_args(sys.argv[1:])
191
+ pProfiles = args.Profiles
192
+ pRegionList = args.Regions
193
+ pAccounts = args.Accounts
194
+ pFragments = args.Fragments
195
+ pSkipAccounts = args.SkipAccounts
196
+ pSkipProfiles = args.SkipProfiles
197
+ pRootOnly = args.RootOnly
198
+ pFilename = args.Filename
199
+ pTiming = args.Time
200
+ verbose = args.loglevel
201
+
202
+ # Setup logging levels
203
+ logging.basicConfig(level=verbose, format="[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
204
+ logging.getLogger("boto3").setLevel(logging.CRITICAL)
205
+ logging.getLogger("botocore").setLevel(logging.CRITICAL)
206
+ logging.getLogger("s3transfer").setLevel(logging.CRITICAL)
207
+ logging.getLogger("urllib3").setLevel(logging.CRITICAL)
208
+
209
+ # ANSI escape code for clearing current line (progress bar cleanup)
210
+ ERASE_LINE = "\x1b[2K"
211
+
212
+ # Start execution timing
213
+ begin_time = time()
214
+
215
+ # Display startup information
216
+ print()
217
+ print(f"Checking for EBS Volumes... ")
218
+ logging.info(f"Profiles: {pProfiles}")
219
+ print()
220
+
221
+ # Phase 1: Gather credentials for all target accounts and regions
222
+ CredentialList = get_all_credentials(
223
+ pProfiles, pTiming, pSkipProfiles, pSkipAccounts, pRootOnly, pAccounts, pRegionList
224
+ )
225
+
226
+ # Phase 2: Execute multi-threaded EBS volume discovery
227
+ VolumesFound = check_accounts_for_ebs_volumes(CredentialList, pFragments)
228
+
229
+ # Phase 3: Present results with analysis and recommendations
230
+ present_results(VolumesFound)
231
+
232
+ # Display execution timing if requested
233
+ if pTiming:
234
+ print(ERASE_LINE)
235
+ print(f"{Fore.GREEN}This script completed in {time() - begin_time:.2f} seconds{Fore.RESET}")
236
+
237
+
238
+ if __name__ == "__main__":
239
+ main()
240
+
241
+ # Display completion message
242
+ print()
243
+ print("Thank you for using this script")
244
+ print()
@@ -0,0 +1,425 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ AWS EC2 Instance Inventory Collection
4
+
5
+ A comprehensive EC2 instance discovery tool that operates across multiple AWS accounts
6
+ and regions using the AWS Cloud Foundations framework. This script provides detailed
7
+ inventory reporting with support for multi-account Organizations access patterns.
8
+
9
+ **AWS API Mapping**: `boto3.client('ec2').describe_instances()`
10
+
11
+ Features:
12
+ - Multi-account EC2 instance discovery via AWS Organizations
13
+ - Parallel region scanning with threading optimization
14
+ - Flexible filtering by instance state (running/stopped)
15
+ - Multiple output formats (table, JSON, CSV)
16
+ - Tag-based metadata collection
17
+ - VPC and network information gathering
18
+
19
+ Compatibility:
20
+ - AWS Organizations with cross-account roles
21
+ - AWS Control Tower managed accounts
22
+ - Standalone AWS accounts
23
+ - All AWS regions including opt-in regions
24
+
25
+ Example:
26
+ Basic usage across all regions:
27
+ ```bash
28
+ python ec2_describe_instances.py --profile my-org-profile
29
+ ```
30
+
31
+ Filter running instances in specific regions:
32
+ ```bash
33
+ python ec2_describe_instances.py --profile my-profile \\
34
+ --regions us-east-1 us-west-2 --status running
35
+ ```
36
+
37
+ Export to JSON for analysis:
38
+ ```bash
39
+ python ec2_describe_instances.py --profile my-profile \\
40
+ --output json --save instances_inventory.json
41
+ ```
42
+
43
+ Requirements:
44
+ - IAM permissions: `ec2:DescribeInstances`, `sts:AssumeRole`
45
+ - AWS Organizations access (for multi-account scanning)
46
+ - Python 3.8+ with required dependencies
47
+
48
+ Author:
49
+ AWS Cloud Foundations Team
50
+
51
+ Version:
52
+ 2025.04.09
53
+ """
54
+
55
+ import logging
56
+ import sys
57
+ from os.path import split
58
+ from queue import Queue
59
+ from threading import Thread
60
+ from time import time
61
+
62
+ import Inventory_Modules
63
+ from ArgumentsClass import CommonArguments
64
+ from botocore.exceptions import ClientError
65
+ from colorama import Fore, init
66
+ from Inventory_Modules import display_results, get_all_credentials
67
+ from tqdm.auto import tqdm
68
+
69
+ init()
70
+ __version__ = "2025.04.09"
71
+ ERASE_LINE = "\x1b[2K"
72
+ begin_time = time()
73
+
74
+
75
+ ##################
76
+ # Functions
77
+ ##################
78
+
79
+
80
+ def parse_args(f_arguments):
81
+ """Parse command-line arguments for EC2 instance inventory collection.
82
+
83
+ Configures argument parser with standard AWS Cloud Foundations parameters
84
+ plus EC2-specific options for filtering instance states.
85
+
86
+ Args:
87
+ f_arguments (list): Command-line arguments to parse
88
+
89
+ Returns:
90
+ argparse.Namespace: Parsed arguments containing:
91
+ - pProfiles: List of AWS profiles to use
92
+ - pRegions: List of AWS regions to scan
93
+ - pStatus: Instance state filter ('running', 'stopped', or None)
94
+ - Verbose: Logging verbosity level
95
+ - Other standard AWS Cloud Foundations parameters
96
+
97
+ Example:
98
+ >>> args = parse_args(['--profile', 'my-profile', '--status', 'running'])
99
+ >>> print(args.pStatus)
100
+ 'running'
101
+ """
102
+ script_path, script_name = split(sys.argv[0])
103
+ parser = CommonArguments()
104
+ parser.my_parser.description = (
105
+ "We're going to find all instances within any of the accounts we have access to, given the profile(s) provided."
106
+ )
107
+ parser.multiprofile()
108
+ parser.multiregion()
109
+ parser.extendedargs()
110
+ parser.rolestouse()
111
+ parser.rootOnly()
112
+ parser.save_to_file()
113
+ parser.timing()
114
+ parser.verbosity()
115
+ parser.version(__version__)
116
+ local = parser.my_parser.add_argument_group(script_name, "Parameters specific to this script")
117
+ local.add_argument(
118
+ "-s",
119
+ "--status",
120
+ dest="pStatus",
121
+ choices=["running", "stopped"],
122
+ type=str,
123
+ default=None,
124
+ help="Whether you want to limit the instances returned to either 'running', 'stopped'. Default is both",
125
+ )
126
+ return parser.my_parser.parse_args(f_arguments)
127
+
128
+
129
+ def find_all_instances(fAllCredentials: list, fStatus: str = None) -> list:
130
+ """Discover EC2 instances across multiple AWS accounts and regions.
131
+
132
+ Performs parallel discovery of EC2 instances using multi-threading for optimal
133
+ performance across large AWS Organizations. Supports optional state filtering
134
+ to focus on specific instance lifecycle phases.
135
+
136
+ Args:
137
+ fAllCredentials (list): List of credential dictionaries containing:
138
+ - AccountId: AWS account identifier
139
+ - Region: AWS region name
140
+ - AccessKey, SecretKey, SessionToken: AWS credentials
141
+ - MgmtAccount: Management account flag
142
+ fStatus (str, optional): Instance state filter. Valid values:
143
+ - 'running': Only return running instances
144
+ - 'stopped': Only return stopped instances
145
+ - None: Return instances in all states
146
+
147
+ Returns:
148
+ list: Collection of EC2 instance dictionaries with attributes:
149
+ - InstanceId: EC2 instance identifier
150
+ - InstanceType: EC2 instance type (e.g., 't3.micro')
151
+ - State: Current instance state
152
+ - VpcId: Associated VPC identifier
153
+ - SubnetId: Associated subnet identifier
154
+ - PrivateIpAddress: Private IP address
155
+ - PublicIpAddress: Public IP address (if assigned)
156
+ - Tags: Instance tags as key-value pairs
157
+ - LaunchTime: Instance launch timestamp
158
+ - AccountId: Source AWS account
159
+ - Region: Source AWS region
160
+
161
+ Raises:
162
+ ClientError: When AWS API calls fail due to permissions or service issues
163
+
164
+ Example:
165
+ >>> credentials = get_all_credentials(['my-profile'])
166
+ >>> running_instances = find_all_instances(credentials, 'running')
167
+ >>> print(f"Found {len(running_instances)} running instances")
168
+
169
+ Note:
170
+ Uses ThreadPool for concurrent execution across regions/accounts.
171
+ Progress tracking provided via tqdm when verbose logging enabled.
172
+ """
173
+
174
+ class FindInstances(Thread):
175
+ """
176
+ Worker thread for concurrent EC2 instance discovery across AWS accounts.
177
+
178
+ Each worker thread processes credential sets from the shared queue,
179
+ calls AWS EC2 APIs to discover compute instances, and performs detailed
180
+ metadata extraction including networking, tagging, and state analysis.
181
+
182
+ Compute Discovery Capabilities:
183
+ - EC2 instance enumeration with comprehensive metadata
184
+ - Instance state and lifecycle analysis
185
+ - Network configuration discovery (VPC, subnet, IP addresses)
186
+ - Tag-based instance categorization and naming
187
+ - Multi-account compute inventory aggregation
188
+ """
189
+
190
+ def __init__(self, queue):
191
+ """
192
+ Initialize worker thread with reference to shared work queue.
193
+
194
+ Args:
195
+ queue (Queue): Thread-safe queue containing EC2 discovery work items
196
+ """
197
+ Thread.__init__(self)
198
+ self.queue = queue
199
+
200
+ def run(self):
201
+ """
202
+ Main worker thread execution loop for EC2 instance discovery.
203
+
204
+ Continuously processes credential sets from queue, performs instance
205
+ discovery via AWS EC2 APIs, and aggregates compute infrastructure data
206
+ with comprehensive metadata extraction and state filtering.
207
+ """
208
+ while True:
209
+ # Get EC2 discovery work item from thread-safe queue
210
+ c_account_credentials = self.queue.get()
211
+ logging.info(f"De-queued info for account number {c_account_credentials['AccountId']}")
212
+
213
+ try:
214
+ # Call AWS EC2 API to discover instances in this account/region
215
+ # find_account_instances2() handles DescribeInstances API with pagination
216
+ # This is the most time-intensive operation in the discovery process
217
+ Instances = Inventory_Modules.find_account_instances2(c_account_credentials)
218
+ logging.info(
219
+ f"Account: {c_account_credentials['AccountId']} Region: {c_account_credentials['Region']} | Found {len(Instances['Reservations'])} reservations"
220
+ )
221
+
222
+ # Initialize instance metadata variables with defaults
223
+ State = InstanceType = InstanceId = PublicDnsName = Name = ""
224
+
225
+ # Process discovered EC2 instances with comprehensive metadata extraction
226
+ # AWS returns instances grouped by reservation (launch request)
227
+ if "Reservations" in Instances.keys():
228
+ for y in range(len(Instances["Reservations"])):
229
+ # Each reservation can contain multiple instances
230
+ for z in range(len(Instances["Reservations"][y]["Instances"])):
231
+ # Extract core instance attributes
232
+ instance_data = Instances["Reservations"][y]["Instances"][z]
233
+
234
+ InstanceType = instance_data["InstanceType"]
235
+ InstanceId = instance_data["InstanceId"]
236
+ State = instance_data["State"]["Name"]
237
+
238
+ # Handle optional public DNS name (depends on network configuration)
239
+ PublicDnsName = (
240
+ instance_data["PublicDnsName"]
241
+ if "PublicDnsName" in instance_data
242
+ else "No Public DNS Name"
243
+ )
244
+
245
+ # Initialize name with fallback for untagged instances
246
+ Name = "No Name Tag"
247
+ # Extract instance name from tags for resource identification
248
+ # Proper instance naming is essential for operational visibility
249
+ try:
250
+ if "Tags" in instance_data:
251
+ for tag in instance_data["Tags"]:
252
+ if tag["Key"] == "Name":
253
+ Name = tag["Value"]
254
+ break
255
+ except KeyError as my_Error:
256
+ # Handle cases where Tags key is missing from API response
257
+ # This can occur with instances launched without tags
258
+ logging.info(f"No tags found for instance {InstanceId}: {my_Error}")
259
+ pass
260
+ # Apply state filtering if specified
261
+ # Common filters: 'running' for active workloads, 'stopped' for cost analysis
262
+ if fStatus is None or fStatus == State:
263
+ # Create comprehensive instance record for inventory
264
+ instance_record = {
265
+ # Organizational context
266
+ "MgmtAccount": c_account_credentials["MgmtAccount"],
267
+ "AccountId": c_account_credentials["AccountId"],
268
+ "Region": c_account_credentials["Region"],
269
+ "ParentProfile": c_account_credentials["ParentProfile"],
270
+ # Instance identification and metadata
271
+ "InstanceId": InstanceId,
272
+ "Name": Name,
273
+ # Compute and operational attributes
274
+ "InstanceType": InstanceType,
275
+ "State": State,
276
+ # Network configuration
277
+ "PublicDNSName": PublicDnsName,
278
+ }
279
+
280
+ # Add to global inventory collection
281
+ AllInstances.append(instance_record)
282
+ else:
283
+ # Skip instances that don't match state filter
284
+ continue
285
+ except KeyError as my_Error:
286
+ # Handle credential or account access configuration errors
287
+ logging.error(f"Account Access failed - trying to access {c_account_credentials['AccountId']}")
288
+ logging.info(f"Actual Error: {my_Error}")
289
+ # Continue processing other accounts despite this failure
290
+ pass
291
+
292
+ except AttributeError as my_Error:
293
+ # Handle profile configuration or credential format errors
294
+ logging.error(f"Error: Likely that one of the supplied profiles was wrong")
295
+ logging.warning(my_Error)
296
+ continue
297
+
298
+ except ClientError as my_Error:
299
+ # Handle AWS API authentication and authorization errors
300
+ if "AuthFailure" in str(my_Error):
301
+ logging.error(
302
+ f"Authorization Failure accessing account {c_account_credentials['AccountId']} in {c_account_credentials['Region']} region"
303
+ )
304
+ logging.warning(
305
+ f"It's possible that the region {c_account_credentials['Region']} hasn't been opted-into"
306
+ )
307
+ continue
308
+ else:
309
+ # Handle API throttling and other service errors
310
+ logging.error(f"Error: Likely throttling errors from too much activity")
311
+ logging.warning(my_Error)
312
+ continue
313
+
314
+ finally:
315
+ # Always mark work item as complete for queue management
316
+ self.queue.task_done()
317
+
318
+ ###########
319
+
320
+ checkqueue = Queue()
321
+
322
+ AllInstances = []
323
+ WorkerThreads = min(len(fAllCredentials), 25)
324
+
325
+ pbar = tqdm(
326
+ desc=f"Finding instances from {len(fAllCredentials)} locations", total=len(fAllCredentials), unit=" locations"
327
+ )
328
+
329
+ for x in range(WorkerThreads):
330
+ worker = FindInstances(checkqueue)
331
+ # Setting daemon to True will let the main thread exit even though the workers are blocking
332
+ worker.daemon = True
333
+ worker.start()
334
+
335
+ for credential in fAllCredentials:
336
+ logging.info(f"Beginning to queue data - starting with {credential['AccountId']}")
337
+ try:
338
+ # I don't know why - but double parens are necessary below. If you remove them, only the first parameter is queued.
339
+ checkqueue.put((credential))
340
+ except ClientError as my_Error:
341
+ if "AuthFailure" in str(my_Error):
342
+ logging.error(
343
+ f"Authorization Failure accessing account {credential['AccountId']} in {credential['Region']} region"
344
+ )
345
+ logging.warning(f"It's possible that the region {credential['Region']} hasn't been opted-into")
346
+ pass
347
+ checkqueue.join()
348
+ pbar.close()
349
+ return AllInstances
350
+
351
+
352
+ ##################
353
+ # Main
354
+ ##################
355
+
356
+ if __name__ == "__main__":
357
+ args = parse_args(sys.argv[1:])
358
+ pProfiles = args.Profiles
359
+ pRegionList = args.Regions
360
+ pAccounts = args.Accounts
361
+ pSkipAccounts = args.SkipAccounts
362
+ pSkipProfiles = args.SkipProfiles
363
+ pAccessRoles = args.AccessRoles
364
+ pStatus = args.pStatus
365
+ pRootOnly = args.RootOnly
366
+ pFilename = args.Filename
367
+ pTiming = args.Time
368
+ verbose = args.loglevel
369
+ # Setup logging levels
370
+ logging.basicConfig(level=verbose, format="[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
371
+ logging.getLogger("boto3").setLevel(logging.CRITICAL)
372
+ logging.getLogger("botocore").setLevel(logging.CRITICAL)
373
+ logging.getLogger("s3transfer").setLevel(logging.CRITICAL)
374
+ logging.getLogger("urllib3").setLevel(logging.CRITICAL)
375
+
376
+ print()
377
+ print(f"Checking for instances... ")
378
+ print()
379
+
380
+ # Find credentials for all Child Accounts
381
+ # CredentialList = get_credentials(pProfiles, pRegionList, pSkipProfiles, pSkipAccounts, pRootOnly, pAccounts, pAccessRoles, pTiming)
382
+ CredentialList = get_all_credentials(
383
+ pProfiles, pTiming, pSkipProfiles, pSkipAccounts, pRootOnly, pAccounts, pRegionList, pAccessRoles
384
+ )
385
+ AccountNum = len(set([acct["AccountId"] for acct in CredentialList]))
386
+ RegionNum = len(set([acct["Region"] for acct in CredentialList]))
387
+ print()
388
+ print(f"Searching total of {AccountNum} accounts and {RegionNum} regions")
389
+ if pTiming:
390
+ print()
391
+ milestone_time1 = time()
392
+ print(
393
+ f"{Fore.GREEN}\t\tFiguring out what regions are available to your accounts, and capturing credentials for all accounts in those regions took: {(milestone_time1 - begin_time):.3f} seconds{Fore.RESET}"
394
+ )
395
+ print()
396
+ print(f"Now running through all accounts and regions identified to find resources...")
397
+ # Collect all the instances from the credentials found
398
+ AllInstances = find_all_instances(CredentialList, pStatus)
399
+ # Display the information we've found thus far
400
+ display_dict = {
401
+ "ParentProfile": {"DisplayOrder": 1, "Heading": "Parent Profile"},
402
+ "MgmtAccount": {"DisplayOrder": 2, "Heading": "Mgmt Acct"},
403
+ "AccountId": {"DisplayOrder": 3, "Heading": "Acct Number"},
404
+ "Region": {"DisplayOrder": 4, "Heading": "Region"},
405
+ "InstanceType": {"DisplayOrder": 5, "Heading": "Instance Type"},
406
+ "Name": {"DisplayOrder": 6, "Heading": "Name"},
407
+ "InstanceId": {"DisplayOrder": 7, "Heading": "Instance ID"},
408
+ "PublicDNSName": {"DisplayOrder": 8, "Heading": "Public Name"},
409
+ "State": {"DisplayOrder": 9, "Heading": "State", "Condition": ["running"]},
410
+ }
411
+
412
+ sorted_all_instances = sorted(
413
+ AllInstances, key=lambda d: (d["ParentProfile"], d["MgmtAccount"], d["Region"], d["AccountId"])
414
+ )
415
+ display_results(sorted_all_instances, display_dict, None, pFilename)
416
+
417
+ if pTiming:
418
+ print(ERASE_LINE)
419
+ print(f"{Fore.GREEN}This script took {time() - begin_time:.2f} seconds{Fore.RESET}")
420
+ print(ERASE_LINE)
421
+
422
+ print(f"Found {len(AllInstances)} instances across {AccountNum} accounts across {RegionNum} regions")
423
+ print()
424
+ print("Thank you for using this script")
425
+ print()