runbooks 0.2.3__py3-none-any.whl → 0.6.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. conftest.py +26 -0
  2. jupyter-agent/.env.template +2 -0
  3. jupyter-agent/.gitattributes +35 -0
  4. jupyter-agent/README.md +16 -0
  5. jupyter-agent/app.py +256 -0
  6. jupyter-agent/cloudops-agent.png +0 -0
  7. jupyter-agent/ds-system-prompt.txt +154 -0
  8. jupyter-agent/jupyter-agent.png +0 -0
  9. jupyter-agent/llama3_template.jinja +123 -0
  10. jupyter-agent/requirements.txt +9 -0
  11. jupyter-agent/utils.py +409 -0
  12. runbooks/__init__.py +71 -3
  13. runbooks/__main__.py +13 -0
  14. runbooks/aws/ec2_describe_instances.py +1 -1
  15. runbooks/aws/ec2_run_instances.py +8 -2
  16. runbooks/aws/ec2_start_stop_instances.py +17 -4
  17. runbooks/aws/ec2_unused_volumes.py +5 -1
  18. runbooks/aws/s3_create_bucket.py +4 -2
  19. runbooks/aws/s3_list_objects.py +6 -1
  20. runbooks/aws/tagging_lambda_handler.py +13 -2
  21. runbooks/aws/tags.json +12 -0
  22. runbooks/base.py +353 -0
  23. runbooks/cfat/README.md +49 -0
  24. runbooks/cfat/__init__.py +74 -0
  25. runbooks/cfat/app.ts +644 -0
  26. runbooks/cfat/assessment/__init__.py +40 -0
  27. runbooks/cfat/assessment/asana-import.csv +39 -0
  28. runbooks/cfat/assessment/cfat-checks.csv +31 -0
  29. runbooks/cfat/assessment/cfat.txt +520 -0
  30. runbooks/cfat/assessment/collectors.py +200 -0
  31. runbooks/cfat/assessment/jira-import.csv +39 -0
  32. runbooks/cfat/assessment/runner.py +387 -0
  33. runbooks/cfat/assessment/validators.py +290 -0
  34. runbooks/cfat/cli.py +103 -0
  35. runbooks/cfat/docs/asana-import.csv +24 -0
  36. runbooks/cfat/docs/cfat-checks.csv +31 -0
  37. runbooks/cfat/docs/cfat.txt +335 -0
  38. runbooks/cfat/docs/checks-output.png +0 -0
  39. runbooks/cfat/docs/cloudshell-console-run.png +0 -0
  40. runbooks/cfat/docs/cloudshell-download.png +0 -0
  41. runbooks/cfat/docs/cloudshell-output.png +0 -0
  42. runbooks/cfat/docs/downloadfile.png +0 -0
  43. runbooks/cfat/docs/jira-import.csv +24 -0
  44. runbooks/cfat/docs/open-cloudshell.png +0 -0
  45. runbooks/cfat/docs/report-header.png +0 -0
  46. runbooks/cfat/models.py +1026 -0
  47. runbooks/cfat/package-lock.json +5116 -0
  48. runbooks/cfat/package.json +38 -0
  49. runbooks/cfat/report.py +496 -0
  50. runbooks/cfat/reporting/__init__.py +46 -0
  51. runbooks/cfat/reporting/exporters.py +337 -0
  52. runbooks/cfat/reporting/formatters.py +496 -0
  53. runbooks/cfat/reporting/templates.py +135 -0
  54. runbooks/cfat/run-assessment.sh +23 -0
  55. runbooks/cfat/runner.py +69 -0
  56. runbooks/cfat/src/actions/check-cloudtrail-existence.ts +43 -0
  57. runbooks/cfat/src/actions/check-config-existence.ts +37 -0
  58. runbooks/cfat/src/actions/check-control-tower.ts +37 -0
  59. runbooks/cfat/src/actions/check-ec2-existence.ts +46 -0
  60. runbooks/cfat/src/actions/check-iam-users.ts +50 -0
  61. runbooks/cfat/src/actions/check-legacy-cur.ts +30 -0
  62. runbooks/cfat/src/actions/check-org-cloudformation.ts +30 -0
  63. runbooks/cfat/src/actions/check-vpc-existence.ts +43 -0
  64. runbooks/cfat/src/actions/create-asanaimport.ts +14 -0
  65. runbooks/cfat/src/actions/create-backlog.ts +372 -0
  66. runbooks/cfat/src/actions/create-jiraimport.ts +15 -0
  67. runbooks/cfat/src/actions/create-report.ts +616 -0
  68. runbooks/cfat/src/actions/define-account-type.ts +51 -0
  69. runbooks/cfat/src/actions/get-enabled-org-policy-types.ts +40 -0
  70. runbooks/cfat/src/actions/get-enabled-org-services.ts +26 -0
  71. runbooks/cfat/src/actions/get-idc-info.ts +34 -0
  72. runbooks/cfat/src/actions/get-org-da-accounts.ts +34 -0
  73. runbooks/cfat/src/actions/get-org-details.ts +35 -0
  74. runbooks/cfat/src/actions/get-org-member-accounts.ts +44 -0
  75. runbooks/cfat/src/actions/get-org-ous.ts +35 -0
  76. runbooks/cfat/src/actions/get-regions.ts +22 -0
  77. runbooks/cfat/src/actions/zip-assessment.ts +27 -0
  78. runbooks/cfat/src/types/index.d.ts +147 -0
  79. runbooks/cfat/tests/__init__.py +141 -0
  80. runbooks/cfat/tests/test_cli.py +340 -0
  81. runbooks/cfat/tests/test_integration.py +290 -0
  82. runbooks/cfat/tests/test_models.py +505 -0
  83. runbooks/cfat/tests/test_reporting.py +354 -0
  84. runbooks/cfat/tsconfig.json +16 -0
  85. runbooks/cfat/webpack.config.cjs +27 -0
  86. runbooks/config.py +260 -0
  87. runbooks/finops/__init__.py +88 -0
  88. runbooks/finops/aws_client.py +245 -0
  89. runbooks/finops/cli.py +151 -0
  90. runbooks/finops/cost_processor.py +410 -0
  91. runbooks/finops/dashboard_runner.py +448 -0
  92. runbooks/finops/helpers.py +355 -0
  93. runbooks/finops/main.py +14 -0
  94. runbooks/finops/profile_processor.py +174 -0
  95. runbooks/finops/types.py +66 -0
  96. runbooks/finops/visualisations.py +80 -0
  97. runbooks/inventory/.gitignore +354 -0
  98. runbooks/inventory/ArgumentsClass.py +261 -0
  99. runbooks/inventory/Inventory_Modules.py +6130 -0
  100. runbooks/inventory/LandingZone/delete_lz.py +1075 -0
  101. runbooks/inventory/README.md +1320 -0
  102. runbooks/inventory/__init__.py +62 -0
  103. runbooks/inventory/account_class.py +532 -0
  104. runbooks/inventory/all_my_instances_wrapper.py +123 -0
  105. runbooks/inventory/aws_decorators.py +201 -0
  106. runbooks/inventory/cfn_move_stack_instances.py +1526 -0
  107. runbooks/inventory/check_cloudtrail_compliance.py +614 -0
  108. runbooks/inventory/check_controltower_readiness.py +1107 -0
  109. runbooks/inventory/check_landingzone_readiness.py +711 -0
  110. runbooks/inventory/cloudtrail.md +727 -0
  111. runbooks/inventory/collectors/__init__.py +20 -0
  112. runbooks/inventory/collectors/aws_compute.py +518 -0
  113. runbooks/inventory/collectors/aws_networking.py +275 -0
  114. runbooks/inventory/collectors/base.py +222 -0
  115. runbooks/inventory/core/__init__.py +19 -0
  116. runbooks/inventory/core/collector.py +303 -0
  117. runbooks/inventory/core/formatter.py +296 -0
  118. runbooks/inventory/delete_s3_buckets_objects.py +169 -0
  119. runbooks/inventory/discovery.md +81 -0
  120. runbooks/inventory/draw_org_structure.py +748 -0
  121. runbooks/inventory/ec2_vpc_utils.py +341 -0
  122. runbooks/inventory/find_cfn_drift_detection.py +272 -0
  123. runbooks/inventory/find_cfn_orphaned_stacks.py +719 -0
  124. runbooks/inventory/find_cfn_stackset_drift.py +733 -0
  125. runbooks/inventory/find_ec2_security_groups.py +669 -0
  126. runbooks/inventory/find_landingzone_versions.py +201 -0
  127. runbooks/inventory/find_vpc_flow_logs.py +1221 -0
  128. runbooks/inventory/inventory.sh +659 -0
  129. runbooks/inventory/list_cfn_stacks.py +558 -0
  130. runbooks/inventory/list_cfn_stackset_operation_results.py +252 -0
  131. runbooks/inventory/list_cfn_stackset_operations.py +734 -0
  132. runbooks/inventory/list_cfn_stacksets.py +453 -0
  133. runbooks/inventory/list_config_recorders_delivery_channels.py +681 -0
  134. runbooks/inventory/list_ds_directories.py +354 -0
  135. runbooks/inventory/list_ec2_availability_zones.py +286 -0
  136. runbooks/inventory/list_ec2_ebs_volumes.py +244 -0
  137. runbooks/inventory/list_ec2_instances.py +425 -0
  138. runbooks/inventory/list_ecs_clusters_and_tasks.py +562 -0
  139. runbooks/inventory/list_elbs_load_balancers.py +411 -0
  140. runbooks/inventory/list_enis_network_interfaces.py +526 -0
  141. runbooks/inventory/list_guardduty_detectors.py +568 -0
  142. runbooks/inventory/list_iam_policies.py +404 -0
  143. runbooks/inventory/list_iam_roles.py +518 -0
  144. runbooks/inventory/list_iam_saml_providers.py +359 -0
  145. runbooks/inventory/list_lambda_functions.py +882 -0
  146. runbooks/inventory/list_org_accounts.py +446 -0
  147. runbooks/inventory/list_org_accounts_users.py +354 -0
  148. runbooks/inventory/list_rds_db_instances.py +406 -0
  149. runbooks/inventory/list_route53_hosted_zones.py +318 -0
  150. runbooks/inventory/list_servicecatalog_provisioned_products.py +575 -0
  151. runbooks/inventory/list_sns_topics.py +360 -0
  152. runbooks/inventory/list_ssm_parameters.py +402 -0
  153. runbooks/inventory/list_vpc_subnets.py +433 -0
  154. runbooks/inventory/list_vpcs.py +422 -0
  155. runbooks/inventory/lockdown_cfn_stackset_role.py +224 -0
  156. runbooks/inventory/models/__init__.py +24 -0
  157. runbooks/inventory/models/account.py +192 -0
  158. runbooks/inventory/models/inventory.py +309 -0
  159. runbooks/inventory/models/resource.py +247 -0
  160. runbooks/inventory/recover_cfn_stack_ids.py +205 -0
  161. runbooks/inventory/requirements.txt +12 -0
  162. runbooks/inventory/run_on_multi_accounts.py +211 -0
  163. runbooks/inventory/tests/common_test_data.py +3661 -0
  164. runbooks/inventory/tests/common_test_functions.py +204 -0
  165. runbooks/inventory/tests/script_test_data.py +0 -0
  166. runbooks/inventory/tests/setup.py +24 -0
  167. runbooks/inventory/tests/src.py +18 -0
  168. runbooks/inventory/tests/test_cfn_describe_stacks.py +208 -0
  169. runbooks/inventory/tests/test_ec2_describe_instances.py +162 -0
  170. runbooks/inventory/tests/test_inventory_modules.py +55 -0
  171. runbooks/inventory/tests/test_lambda_list_functions.py +86 -0
  172. runbooks/inventory/tests/test_moto_integration_example.py +273 -0
  173. runbooks/inventory/tests/test_org_list_accounts.py +49 -0
  174. runbooks/inventory/update_aws_actions.py +173 -0
  175. runbooks/inventory/update_cfn_stacksets.py +1215 -0
  176. runbooks/inventory/update_cloudwatch_logs_retention_policy.py +294 -0
  177. runbooks/inventory/update_iam_roles_cross_accounts.py +478 -0
  178. runbooks/inventory/update_s3_public_access_block.py +539 -0
  179. runbooks/inventory/utils/__init__.py +23 -0
  180. runbooks/inventory/utils/aws_helpers.py +510 -0
  181. runbooks/inventory/utils/threading_utils.py +493 -0
  182. runbooks/inventory/utils/validation.py +682 -0
  183. runbooks/inventory/verify_ec2_security_groups.py +1430 -0
  184. runbooks/main.py +785 -0
  185. runbooks/organizations/__init__.py +12 -0
  186. runbooks/organizations/manager.py +374 -0
  187. runbooks/security_baseline/README.md +324 -0
  188. runbooks/security_baseline/checklist/alternate_contacts.py +8 -1
  189. runbooks/security_baseline/checklist/bucket_public_access.py +4 -1
  190. runbooks/security_baseline/checklist/cloudwatch_alarm_configuration.py +9 -2
  191. runbooks/security_baseline/checklist/guardduty_enabled.py +9 -2
  192. runbooks/security_baseline/checklist/multi_region_instance_usage.py +5 -1
  193. runbooks/security_baseline/checklist/root_access_key.py +6 -1
  194. runbooks/security_baseline/config-origin.json +1 -1
  195. runbooks/security_baseline/config.json +1 -1
  196. runbooks/security_baseline/permission.json +1 -1
  197. runbooks/security_baseline/report_generator.py +10 -2
  198. runbooks/security_baseline/report_template_en.html +8 -8
  199. runbooks/security_baseline/report_template_jp.html +8 -8
  200. runbooks/security_baseline/report_template_kr.html +13 -13
  201. runbooks/security_baseline/report_template_vn.html +8 -8
  202. runbooks/security_baseline/requirements.txt +7 -0
  203. runbooks/security_baseline/run_script.py +8 -2
  204. runbooks/security_baseline/security_baseline_tester.py +10 -2
  205. runbooks/security_baseline/utils/common.py +5 -1
  206. runbooks/utils/__init__.py +204 -0
  207. runbooks-0.6.1.dist-info/METADATA +373 -0
  208. runbooks-0.6.1.dist-info/RECORD +237 -0
  209. {runbooks-0.2.3.dist-info → runbooks-0.6.1.dist-info}/WHEEL +1 -1
  210. runbooks-0.6.1.dist-info/entry_points.txt +7 -0
  211. runbooks-0.6.1.dist-info/licenses/LICENSE +201 -0
  212. runbooks-0.6.1.dist-info/top_level.txt +3 -0
  213. runbooks/python101/calculator.py +0 -34
  214. runbooks/python101/config.py +0 -1
  215. runbooks/python101/exceptions.py +0 -16
  216. runbooks/python101/file_manager.py +0 -218
  217. runbooks/python101/toolkit.py +0 -153
  218. runbooks-0.2.3.dist-info/METADATA +0 -435
  219. runbooks-0.2.3.dist-info/RECORD +0 -61
  220. runbooks-0.2.3.dist-info/entry_points.txt +0 -3
  221. runbooks-0.2.3.dist-info/top_level.txt +0 -1
@@ -0,0 +1,669 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ AWS EC2 Security Group Discovery and Analysis Tool
4
+
5
+ CRITICAL SECURITY WARNING: This script analyzes security group configurations across
6
+ multiple AWS accounts. Security groups are fundamental network access controls that
7
+ directly impact the security posture of AWS workloads.
8
+
9
+ Purpose:
10
+ Discovers, analyzes, and inventories EC2 security groups across AWS Organizations
11
+ with advanced filtering, reference tracking, and rule analysis capabilities.
12
+ Essential for security auditing, compliance validation, and network governance.
13
+
14
+ AWS API Operations:
15
+ - ec2.describe_security_groups(): Primary security group discovery
16
+ - ec2.describe_instances(): Instance-to-security-group associations
17
+ - ec2.describe_network_interfaces(): ENI-to-security-group mappings
18
+ - ec2.describe_load_balancers(): ELB security group references
19
+ - Additional service APIs for comprehensive reference tracking
20
+
21
+ Security Analysis Features:
22
+ - Default security group identification (critical for compliance)
23
+ - Unused security group detection (attack surface reduction)
24
+ - Security rule analysis and breakdown
25
+ - Cross-service reference tracking
26
+ - Fragment-based filtering for targeted analysis
27
+
28
+ Compliance Use Cases:
29
+ - PCI DSS network segmentation validation
30
+ - SOC2 access control auditing
31
+ - CIS benchmark security group compliance
32
+ - Zero Trust network architecture assessment
33
+ - Security group sprawl management
34
+
35
+ Performance Features:
36
+ - Multi-threaded discovery across accounts/regions
37
+ - Progress tracking for large environments
38
+ - Memory-efficient processing of complex relationships
39
+ - Configurable analysis depth (basic vs comprehensive)
40
+
41
+ Risk Mitigation:
42
+ - Read-only operations with minimal permissions
43
+ - Comprehensive audit logging
44
+ - No modification capabilities (analysis only)
45
+ - Safe handling of large security group inventories
46
+
47
+ Usage:
48
+ python find_ec2_security_groups.py -p <profile> --default
49
+ python find_ec2_security_groups.py -p <profile> --references --rules
50
+
51
+ Author: AWS Cloud Foundations Team
52
+ Version: 2024.09.24
53
+ Maintained: Network Security Team
54
+ """
55
+
56
+ import logging
57
+ import sys
58
+ from os.path import split
59
+ from queue import Queue
60
+ from threading import Thread
61
+ from time import time
62
+
63
+ from ArgumentsClass import CommonArguments
64
+ from botocore.exceptions import ClientError
65
+ from colorama import Fore, init
66
+ from Inventory_Modules import (
67
+ display_results,
68
+ find_references_to_security_groups2,
69
+ find_security_groups2,
70
+ get_all_credentials,
71
+ )
72
+ from tqdm.auto import tqdm
73
+
74
+ init()
75
+ __version__ = "2024.09.24"
76
+ ERASE_LINE = "\x1b[2K"
77
+ begin_time = time()
78
+
79
+
80
+ ##################
81
+ # Functions
82
+ ##################
83
+ def parse_args(f_arguments):
84
+ """
85
+ Parse and validate command-line arguments for security group analysis.
86
+
87
+ Configures the argument parser with security-specific options including
88
+ default security group detection, reference tracking, rule analysis,
89
+ and filtering capabilities for comprehensive security posture assessment.
90
+
91
+ Args:
92
+ f_arguments (list): Command-line arguments to parse (typically sys.argv[1:])
93
+
94
+ Returns:
95
+ argparse.Namespace: Parsed arguments containing:
96
+ - Profiles: AWS profiles for multi-account security analysis
97
+ - Regions: Target AWS regions for security group discovery
98
+ - Fragments: Security group name fragments for targeted analysis
99
+ - pDefault: Flag to focus on default security groups (compliance)
100
+ - pReferences: Enable cross-service reference tracking
101
+ - pNoEmpty: Filter out unused security groups
102
+ - pRules: Enable detailed security rule analysis
103
+ - Other standard framework arguments
104
+
105
+ Security-Specific Arguments:
106
+ --default: Critical for compliance auditing. Default security groups
107
+ often violate security policies and should be closely monitored.
108
+
109
+ --references: Enables comprehensive reference tracking across:
110
+ - EC2 instances and their ENIs
111
+ - Load balancers (ALB, NLB, CLB)
112
+ - RDS instances and clusters
113
+ - Lambda functions (VPC-enabled)
114
+ - Other AWS services using security groups
115
+
116
+ --noempty: Filters unused security groups that represent potential
117
+ attack surface. Useful for security group hygiene and
118
+ attack surface reduction initiatives.
119
+
120
+ --rules: Enables detailed ingress/egress rule analysis including:
121
+ - Protocol and port mappings
122
+ - Source/destination CIDR analysis
123
+ - Cross-security-group references
124
+ - Overly permissive rule identification
125
+
126
+ Use Cases:
127
+ - Compliance auditing: --default for policy violations
128
+ - Security hygiene: --noempty for unused resource cleanup
129
+ - Incident response: --references for blast radius analysis
130
+ - Security architecture: --rules for network segmentation review
131
+ """
132
+ script_path, script_name = split(sys.argv[0])
133
+ parser = CommonArguments()
134
+ parser.multiprofile()
135
+ parser.multiregion()
136
+ parser.extendedargs()
137
+ parser.fragment()
138
+ parser.rootOnly()
139
+ parser.timing()
140
+ parser.save_to_file()
141
+ parser.verbosity()
142
+ parser.version(__version__)
143
+ local = parser.my_parser.add_argument_group(script_name, "Parameters specific to this script")
144
+ local.add_argument(
145
+ "--default",
146
+ dest="pDefault",
147
+ action="store_true",
148
+ help="flag to determines if you're only looking for default security groups",
149
+ )
150
+ local.add_argument(
151
+ "--references",
152
+ dest="pReferences",
153
+ action="store_true",
154
+ help="flag to further get references to the security groups found",
155
+ )
156
+ local.add_argument(
157
+ "--noempty",
158
+ dest="pNoEmpty",
159
+ action="store_true",
160
+ help="flag to remove empty Security Groups (no references) before display",
161
+ )
162
+ local.add_argument(
163
+ "--rules",
164
+ dest="pRules",
165
+ action="store_true",
166
+ help="flag to further break out the rules within the security groups found",
167
+ )
168
+ return parser.my_parser.parse_args(f_arguments)
169
+
170
+
171
+ def check_accounts_for_security_groups(
172
+ fCredentialList,
173
+ fFragment: list = None,
174
+ fExact: bool = False,
175
+ fDefault: bool = False,
176
+ fReferences: bool = False,
177
+ fRules: bool = False,
178
+ ):
179
+ """
180
+ Execute multi-threaded security group discovery and analysis across AWS accounts.
181
+
182
+ This is the core orchestration function that performs concurrent security group
183
+ analysis across all provided AWS accounts and regions. Implements comprehensive
184
+ security group discovery with optional deep analysis capabilities.
185
+
186
+ Args:
187
+ fCredentialList (list): List of credential dictionaries containing:
188
+ - AccountId: AWS account identifier
189
+ - Region: AWS region name
190
+ - AccessKeyId, SecretAccessKey, SessionToken: AWS credentials
191
+ - Success: Boolean flag indicating credential validation status
192
+
193
+ fFragment (list, optional): Security group name fragments for filtering.
194
+ Enables targeted analysis of specific security groups by name pattern.
195
+
196
+ fExact (bool, optional): Use exact matching for fragments instead of
197
+ substring matching. Defaults to False (partial matching).
198
+
199
+ fDefault (bool, optional): Focus analysis on default security groups only.
200
+ Critical for compliance auditing as default SGs often violate policies.
201
+
202
+ fReferences (bool, optional): Enable comprehensive reference tracking.
203
+ Discovers all AWS resources associated with each security group.
204
+
205
+ fRules (bool, optional): Enable detailed security rule analysis.
206
+ Breaks down ingress/egress rules for granular security assessment.
207
+
208
+ Returns:
209
+ list: Comprehensive list of security group dictionaries with metadata:
210
+ - GroupId: Security group identifier
211
+ - GroupName: Security group name
212
+ - VpcId: Associated VPC identifier
213
+ - Description: Security group description
214
+ - AccountId: Source AWS account
215
+ - Region: Source AWS region
216
+ - Rules: Detailed rule analysis (if fRules=True)
217
+ - References: Associated resources (if fReferences=True)
218
+
219
+ Threading Architecture:
220
+ - Uses Queue for thread-safe work distribution
221
+ - Worker thread pool sized for optimal AWS API performance
222
+ - Progress tracking via tqdm for user feedback
223
+ - Comprehensive error handling for failed account access
224
+
225
+ Security Analysis Modes:
226
+ 1. Basic Discovery: Security group enumeration and metadata
227
+ 2. Reference Tracking: Cross-service resource associations
228
+ 3. Rule Analysis: Granular ingress/egress rule breakdown
229
+ 4. Default Focus: Compliance-oriented default security group audit
230
+
231
+ Performance Optimization:
232
+ - Concurrent processing across accounts/regions
233
+ - Efficient API pagination handling
234
+ - Memory-conscious processing for large environments
235
+ - Rate limiting respect for AWS API throttling
236
+
237
+ Error Handling:
238
+ - Account access failures: Logged and skipped gracefully
239
+ - API throttling: Handled through retry logic
240
+ - Permission errors: Detailed logging for troubleshooting
241
+ - Invalid credentials: Validation and error reporting
242
+
243
+ Security Considerations:
244
+ - Read-only operations only (no modifications)
245
+ - Minimal required permissions
246
+ - Comprehensive audit logging
247
+ - Safe handling of sensitive security configurations
248
+ """
249
+
250
+ class FindSecurityGroups(Thread):
251
+ """
252
+ Worker thread for concurrent security group discovery and analysis.
253
+
254
+ Each worker thread processes credential sets from the shared queue,
255
+ calls AWS EC2 APIs to discover security groups, and performs optional
256
+ deep analysis based on configured parameters.
257
+
258
+ Security Analysis Capabilities:
259
+ - Basic security group enumeration
260
+ - Default security group identification
261
+ - Cross-service reference tracking
262
+ - Detailed rule analysis and breakdown
263
+ - Fragment-based filtering
264
+ """
265
+
266
+ def __init__(self, queue):
267
+ """
268
+ Initialize worker thread with reference to shared work queue.
269
+
270
+ Args:
271
+ queue (Queue): Thread-safe queue containing credential work items
272
+ """
273
+ Thread.__init__(self)
274
+ self.queue = queue
275
+
276
+ def run(self):
277
+ """
278
+ Main worker thread execution loop for security group analysis.
279
+
280
+ Continuously processes credential sets from queue, performs security
281
+ group discovery, and aggregates results with optional deep analysis
282
+ based on configured parameters (references, rules, defaults).
283
+ """
284
+ while True:
285
+ # Get work item from thread-safe queue
286
+ c_account_credentials, c_fragments, c_exact, c_default = self.queue.get()
287
+ logging.info(f"De-queued info for account number {c_account_credentials['AccountId']}")
288
+ try:
289
+ # TODO:
290
+ # If I wanted to find the arns of the resources that belonged to the security groups,
291
+ # I'd have to get a listing of all the resources that could possibly have a security group attached
292
+ # and then use that list to reverse-match the enis we find to the enis attached to the resources,
293
+ # so I could figure out which resources were being represented by the enis.
294
+ # This seems like a lot of work, although I understand why it would be useful
295
+ # It's possible we could start with just EC2 instances, and eventually widen the scope
296
+ # Now go through each credential (account / region), and find all default security groups
297
+ SecurityGroups = find_security_groups2(c_account_credentials, c_fragments, c_exact, c_default)
298
+ """
299
+ instances = aws_acct.session.client('ec2').describe_instances()
300
+ for sg in SecurityGroups:
301
+ for instance in instances['Reservations']:
302
+ for inst in instance['Instances']:
303
+ for secgrp in inst['SecurityGroups']:
304
+ if sg['GroupName'] in secgrp['GroupName']:
305
+ print(inst['InstanceId'], inst['PrivateIpAddress'], inst['State']['Name'], inst['PrivateDnsName'], sg['GroupName'], sg['Description'])
306
+ """
307
+ logging.info(
308
+ f"Account: {c_account_credentials['AccountId']} | Region: {c_account_credentials['Region']} | Found {len(SecurityGroups)} groups"
309
+ )
310
+ # Checking whether the list is empty or not
311
+ if SecurityGroups:
312
+ for security_group in SecurityGroups:
313
+ if fReferences:
314
+ ResourcesReferencingSG = find_references_to_security_groups2(
315
+ c_account_credentials, security_group
316
+ )
317
+ if fRules:
318
+ for inbound_permission in security_group["IpPermissions"]:
319
+ inbound_permission["Protocol"] = (
320
+ "AllTraffic"
321
+ if inbound_permission["IpProtocol"] == "-1"
322
+ else inbound_permission["IpProtocol"]
323
+ )
324
+ if AnySource in inbound_permission["IpRanges"]:
325
+ inbound_permission["From"] = "Any"
326
+ elif inbound_permission["IpRanges"]:
327
+ inbound_permission["From"] = inbound_permission["IpRanges"]
328
+ elif inbound_permission["UserIdGroupPairs"]:
329
+ inbound_permission["From"] = inbound_permission["UserIdGroupPairs"]
330
+ if inbound_permission["From"][0]["GroupId"] == security_group["GroupId"]:
331
+ inbound_permission["From"] = "Myself"
332
+ elif inbound_permission["PrefixListIds"]:
333
+ inbound_permission["From"] = inbound_permission["PrefixListIds"]
334
+ else:
335
+ inbound_permission["From"] = None
336
+ for outbound_permission in security_group["IpPermissionsEgress"]:
337
+ outbound_permission["Protocol"] = (
338
+ "AllTraffic"
339
+ if outbound_permission["IpProtocol"] == "-1"
340
+ else outbound_permission["IpProtocol"]
341
+ )
342
+ if AnyDest in outbound_permission["IpRanges"]:
343
+ outbound_permission["To"] = "Any"
344
+ elif outbound_permission["IpRanges"]:
345
+ outbound_permission["To"] = outbound_permission["IpRanges"]
346
+ elif outbound_permission["UserIdGroupPairs"]:
347
+ outbound_permission["To"] = outbound_permission["UserIdGroupPairs"]
348
+ elif outbound_permission["PrefixListIds"]:
349
+ outbound_permission["To"] = outbound_permission["PrefixListIds"]
350
+ else:
351
+ outbound_permission["To"] = None
352
+ else:
353
+ logging.info(
354
+ f"No security groups found in account {c_account_credentials['AccountId']} in region {c_account_credentials['Region']}"
355
+ )
356
+
357
+ # Thread-safe aggregation of results
358
+ AllSecurityGroups.extend(security_groups)
359
+
360
+ except ClientError as my_Error:
361
+ # Handle AWS API authentication and authorization errors
362
+ if "AuthFailure" in str(my_Error):
363
+ logging.error(
364
+ f"Authorization Failure accessing account {c_account_credentials['AccountId']} in '{c_account_credentials['Region']}' region"
365
+ )
366
+ logging.warning(
367
+ f"It's possible that the region '{c_account_credentials['Region']}' hasn't been opted-into"
368
+ )
369
+ # Continue processing other accounts despite this failure
370
+ pass
371
+
372
+ except KeyError as my_Error:
373
+ # Handle credential or account access failures
374
+ logging.error(f"Account Access failed - trying to access {c_account_credentials['AccountId']}")
375
+ logging.info(f"Actual Error: {my_Error}")
376
+ # Continue processing other accounts
377
+ pass
378
+
379
+ except AttributeError as my_Error:
380
+ # Handle profile configuration errors
381
+ logging.error(f"Error: Likely that one of the supplied profiles {pProfiles} was wrong")
382
+ logging.warning(my_Error)
383
+ continue
384
+
385
+ finally:
386
+ # Always complete the work item and update progress
387
+ logging.info(
388
+ f"{ERASE_LINE}Finished finding security groups in account {c_account_credentials['AccountId']} in region {c_account_credentials['Region']}"
389
+ )
390
+ pbar.update()
391
+ self.queue.task_done()
392
+
393
+ ###########
394
+ AnyDest = {"CidrIp": "0.0.0.0/0"}
395
+ AnySource = {"CidrIp": "0.0.0.0/0"}
396
+
397
+ checkqueue = Queue()
398
+
399
+ # Initialize shared data structures for thread-safe result aggregation
400
+ AllSecurityGroups = []
401
+
402
+ # Calculate optimal thread pool size for AWS API performance
403
+ # Limit to 50 to respect AWS API rate limits and avoid overwhelming service
404
+ WorkerThreads = min(len(fCredentialList), 50)
405
+
406
+ # Initialize thread-safe work queue
407
+ checkqueue = Queue()
408
+
409
+ # Initialize progress bar for user feedback during long-running analysis
410
+ pbar = tqdm(
411
+ desc=f"Finding security groups from {len(fCredentialList)} locations",
412
+ total=len(fCredentialList),
413
+ unit=" locations",
414
+ )
415
+
416
+ # Start worker thread pool for concurrent security group analysis
417
+ for x in range(WorkerThreads):
418
+ worker = FindSecurityGroups(checkqueue)
419
+ # Daemon threads will terminate when main thread exits
420
+ # This prevents hanging if an exception occurs in main thread
421
+ worker.daemon = True
422
+ worker.start()
423
+
424
+ # Queue all credential work items for concurrent processing
425
+ for credential in fCredentialList:
426
+ logging.info(f"Connecting to account {credential['AccountId']}")
427
+ try:
428
+ # Add credential set and parameters to work queue for worker thread processing
429
+ checkqueue.put((credential, fFragment, fExact, fDefault))
430
+ except ClientError as my_Error:
431
+ # Handle credential validation errors during queue population
432
+ if "AuthFailure" in str(my_Error):
433
+ logging.error(
434
+ f"Authorization Failure accessing account {credential['AccountId']} in '{credential['Region']}' region"
435
+ )
436
+ logging.warning(f"It's possible that the region '{credential['Region']}' hasn't been opted-into")
437
+ # Continue queuing other credentials despite this failure
438
+ pass
439
+
440
+ # Wait for all queued work items to be processed by worker threads
441
+ # This blocks until all worker threads call task_done()
442
+ checkqueue.join()
443
+
444
+ # Clean up progress bar
445
+ pbar.close()
446
+
447
+ return AllSecurityGroups
448
+
449
+
450
+ # Find all security groups (Done)
451
+ # Determine whether these Security Groups are in use (Done)
452
+ # For each security group, find if any rules mention the security group found (either ENI or in other Security Groups) (Done)
453
+ # TODO:
454
+ # To find the arn of the resource using that security group, instead of just the ENI.
455
+ # To fix the use of a default security group:
456
+ # For each security group, find if any rules mention the security group found
457
+ # Once all the rules are found, create a new security group - cloning those rules
458
+ # Find all the ENIs (not just EC2 instances) that might use that security group
459
+ # Determine if there's a way to update those resources to use the new security group
460
+ # Present what we've found, and ask the user if they want to update those resources to use the new security group created
461
+
462
+
463
+ def save_data_to_file(
464
+ f_AllSecurityGroups: list,
465
+ f_Filename: str,
466
+ f_References: bool = False,
467
+ f_Rules: bool = False,
468
+ f_NoEmpty: bool = False,
469
+ ) -> str:
470
+ """
471
+ Description: Saves the data to a file
472
+ @param f_AllSecurityGroups: The security groups and associated data that were found
473
+ @param f_Filename: The file to save the data to
474
+ @param f_References: Whether to include the references to the security groups or not
475
+ @param f_Rules: Whether to include the rules within the security groups or not
476
+ @param f_NoEmpty: Whether to include non-referenced security groups or not
477
+ @return: The filename that was saved
478
+ """
479
+ # Save the header to the file
480
+ Heading = f"AccountId|Region|SG Group Name|SG Group ID|VPC ID|Default(T/F)|Description"
481
+ reference_Heading = (
482
+ f"|Resource Type|Resource ID|Resource Status|Attachment ID|Instance Name Tag|IP Address|Description"
483
+ )
484
+ rules_Heading = f"|InboundRule|Port From|Port To|From"
485
+ if f_References:
486
+ Heading += reference_Heading
487
+ if f_Rules:
488
+ Heading += rules_Heading
489
+ # Save the data to a file
490
+ with open(f_Filename, "w") as f:
491
+ f.write(Heading + "\n")
492
+ for sg in f_AllSecurityGroups:
493
+ sg_line = f"{sg['AccountId']}|{sg['Region']}|{sg['GroupName']}|{sg['GroupId']}|{sg['VpcId']}|{sg['Default']}|{sg['Description']}"
494
+ if pReferences:
495
+ if sg["NumOfReferences"] == 0 and f_NoEmpty:
496
+ # This means that the SG had no references, and the "NoEmpty" means we don't want non-referenced SGs, so it skips ahead
497
+ continue
498
+ elif sg["NumOfReferences"] == 0:
499
+ sg_line_with_references = sg_line + f"{'|None' * 7}"
500
+ # f.write(sg_line)
501
+ elif sg["NumOfReferences"] > 0:
502
+ for reference in sg["ReferencedResources"]:
503
+ reference_line = f"|{reference['ResourceType']}|{reference['Id']}|{reference['Status']}|{reference['AttachmentId']}|{reference['InstanceNameTag']}|{reference['IpAddress']}|{reference['Description']}"
504
+ sg_line_with_references = sg_line + reference_line
505
+ # f.write(sg_line + reference_line)
506
+ if pRules:
507
+ if sg["NumOfRules"] == 0:
508
+ sg_line_with_rules = sg_line + f"{'|None' * 4}\n"
509
+ # f.write(sg_line)
510
+ else:
511
+ for inbound_permission in sg["IpPermissions"]:
512
+ inbound_permission_line = f"|{inbound_permission['Protocol']}|{inbound_permission['FromPort']}|{inbound_permission['ToPort']}|{inbound_permission['From']}"
513
+ row = sg_line + inbound_permission_line + "\n"
514
+ f.write(row)
515
+ for outbound_permission in sg["IpPermissionsEgress"]:
516
+ outbound_permission_line = f"|{outbound_permission['Protocol']}|{outbound_permission['FromPort']}|{outbound_permission['ToPort']}|{outbound_permission['To']}"
517
+ row = sg_line + outbound_permission_line + "\n"
518
+ f.write(row)
519
+ elif not pReferences:
520
+ row = sg_line + "\n"
521
+ f.write(row)
522
+ elif pReferences:
523
+ row = sg_line_with_references + "\n"
524
+ f.write(row)
525
+ logging.info(f"Data saved to {f_Filename}")
526
+ return f_Filename
527
+
528
+
529
+ def find_resource_using_eni(f_eni: str, f_sg: dict, f_AllSecurityGroups: list) -> dict:
530
+ """
531
+ Description: Finds the resource using the ENI
532
+ @param f_eni: The ENI to find the resource for
533
+ @param f_sg: The security group to find the resource for
534
+ @param f_AllSecurityGroups: The list of all security groups and associated data
535
+ @return: The resource using the ENI
536
+ """
537
+ for resource in f_AllSecurityGroups:
538
+ if resource["GroupId"] == f_sg["GroupId"]:
539
+ for eni in resource["NetworkInterfaces"]:
540
+ if eni["NetworkInterfaceId"] == f_eni:
541
+ return resource
542
+ return None
543
+
544
+
545
+ ##################
546
+ # Main
547
+ ##################
548
+
549
+ if __name__ == "__main__":
550
+ args = parse_args(sys.argv[1:])
551
+ pProfiles = args.Profiles
552
+ pRegionList = args.Regions
553
+ pSkipAccounts = args.SkipAccounts
554
+ pSkipProfiles = args.SkipProfiles
555
+ pAccounts = args.Accounts
556
+ pRootOnly = args.RootOnly
557
+ pFragment = args.Fragments
558
+ pExact = args.Exact
559
+ pDefault = args.pDefault
560
+ pReferences = args.pReferences
561
+ pRules = args.pRules
562
+ pNoEmpty = args.pNoEmpty
563
+ pFilename = args.Filename
564
+ pTiming = args.Time
565
+ verbose = args.loglevel
566
+ # Setup logging levels
567
+ logging.basicConfig(level=verbose, format="[%(filename)s:%(lineno)s - %(funcName)20s() ] %(message)s")
568
+ logging.getLogger("boto3").setLevel(logging.CRITICAL)
569
+ logging.getLogger("botocore").setLevel(logging.CRITICAL)
570
+ logging.getLogger("s3transfer").setLevel(logging.CRITICAL)
571
+ logging.getLogger("urllib3").setLevel(logging.CRITICAL)
572
+
573
+ print()
574
+ print(f"Checking for Security Groups... ")
575
+ print()
576
+
577
+ logging.info(f"Profiles: {pProfiles}")
578
+
579
+ display_dict = {
580
+ "MgmtAccount": {"DisplayOrder": 1, "Heading": "Mgmt Acct"},
581
+ "AccountId": {"DisplayOrder": 2, "Heading": "Acct Number"},
582
+ "Region": {"DisplayOrder": 3, "Heading": "Region"},
583
+ "GroupName": {"DisplayOrder": 4, "Heading": "Group Name"},
584
+ "GroupId": {"DisplayOrder": 5, "Heading": "Group ID"},
585
+ "VpcId": {"DisplayOrder": 6, "Heading": "VPC ID"},
586
+ "Default": {"DisplayOrder": 7, "Heading": "Default", "Condition": [True]},
587
+ "Description": {"DisplayOrder": 10, "Heading": "Description"},
588
+ }
589
+ display_dict.update(
590
+ {
591
+ "NumOfReferences": {"DisplayOrder": 8, "Heading": "# Refs"},
592
+ "ReferencedResources": {
593
+ "DisplayOrder": 11,
594
+ "Heading": "References",
595
+ "SubDisplay": {
596
+ "ResourceType": {"DisplayOrder": 1, "Heading": "Resource Type"},
597
+ "Id": {"DisplayOrder": 2, "Heading": "ID"},
598
+ "Status": {"DisplayOrder": 3, "Heading": "Status"},
599
+ "AttachmentId": {"DisplayOrder": 4, "Heading": "Instance Id"},
600
+ "InstanceNameTag": {"DisplayOrder": 5, "Heading": "Name"},
601
+ "IpAddress": {"DisplayOrder": 6, "Heading": "Private IP"},
602
+ "Description": {"DisplayOrder": 7, "Heading": "Description"},
603
+ },
604
+ },
605
+ }
606
+ ) if pReferences else None
607
+ # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2/client/describe_security_groups.html
608
+ display_dict.update(
609
+ {
610
+ "NumOfRules": {"DisplayOrder": 9, "Heading": "# Rules"},
611
+ "IpPermissions": {
612
+ "DisplayOrder": 12,
613
+ "Heading": "Inbound Rules",
614
+ "SubDisplay": {
615
+ "Protocol": {"DisplayOrder": 1, "Heading": "In Protocol"},
616
+ "FromPort": {"DisplayOrder": 2, "Heading": "Port From", "Delimiter": False},
617
+ "ToPort": {"DisplayOrder": 3, "Heading": "Port To", "Delimiter": False},
618
+ "From": {"DisplayOrder": 4, "Heading": "From", "Condition": ["10.1.1.0/24"]},
619
+ # 'UserIdGroupPairs': {'DisplayOrder': 5, 'Heading': 'Group Pairs'},
620
+ # 'Description' : {'DisplayOrder': 6, 'Heading': 'Description'}
621
+ },
622
+ },
623
+ "IpPermissionsEgress": {
624
+ "DisplayOrder": 13,
625
+ "Heading": "Outbound Rules",
626
+ "SubDisplay": {
627
+ "Protocol": {"DisplayOrder": 1, "Heading": "Out Protocol"},
628
+ "FromPort": {"DisplayOrder": 2, "Heading": "Port From", "Delimiter": False},
629
+ "ToPort": {"DisplayOrder": 3, "Heading": "Port To", "Delimiter": False},
630
+ "To": {"DisplayOrder": 4, "Heading": "To"},
631
+ # 'UserIdGroupPairs': {'DisplayOrder': 5, 'Heading': 'Group Pairs'},
632
+ # 'Description' : {'DisplayOrder': 6, 'Heading': 'Description'}
633
+ },
634
+ },
635
+ }
636
+ ) if pRules else None
637
+
638
+ # Get credentials for all relevant children accounts
639
+
640
+ CredentialList = get_all_credentials(
641
+ pProfiles, pTiming, pSkipProfiles, pSkipAccounts, pRootOnly, pAccounts, pRegionList
642
+ )
643
+ AccountList = list(set([x["AccountId"] for x in CredentialList if x["Success"]]))
644
+ RegionList = list(set([x["Region"] for x in CredentialList if x["Success"]]))
645
+ # Find Security Groups across all children accounts
646
+ # This same function also does the references check, if you want it to...
647
+ AllSecurityGroups = check_accounts_for_security_groups(
648
+ CredentialList, pFragment, pExact, pDefault, pReferences, pRules
649
+ )
650
+ sorted_AllSecurityGroups = sorted(
651
+ AllSecurityGroups, key=lambda k: (k["MgmtAccount"], k["AccountId"], k["Region"], k["GroupName"])
652
+ )
653
+
654
+ # Display results
655
+ display_results(sorted_AllSecurityGroups, display_dict, None)
656
+
657
+ if pFilename:
658
+ saved_filename = save_data_to_file(sorted_AllSecurityGroups, pFilename, pReferences, pRules, pNoEmpty)
659
+ print(f"Data has been saved to {saved_filename}")
660
+ if pTiming:
661
+ print(ERASE_LINE)
662
+ print(f"{Fore.GREEN}This script took {time() - begin_time:.2f} seconds{Fore.RESET}")
663
+
664
+ print(
665
+ f"We found {len(AllSecurityGroups)} {'default ' if pDefault else ''}security group{'' if len(AllSecurityGroups) == 1 else 's'} across {len(AccountList)} accounts and {len(RegionList)} regions"
666
+ )
667
+ print()
668
+ print("Thank you for using this script")
669
+ print()