runbooks 1.1.4__py3-none-any.whl → 1.1.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (273) hide show
  1. runbooks/__init__.py +31 -2
  2. runbooks/__init___optimized.py +18 -4
  3. runbooks/_platform/__init__.py +1 -5
  4. runbooks/_platform/core/runbooks_wrapper.py +141 -138
  5. runbooks/aws2/accuracy_validator.py +812 -0
  6. runbooks/base.py +7 -0
  7. runbooks/cfat/assessment/compliance.py +1 -1
  8. runbooks/cfat/assessment/runner.py +1 -0
  9. runbooks/cfat/cloud_foundations_assessment.py +227 -239
  10. runbooks/cli/__init__.py +1 -1
  11. runbooks/cli/commands/cfat.py +64 -23
  12. runbooks/cli/commands/finops.py +1005 -54
  13. runbooks/cli/commands/inventory.py +135 -91
  14. runbooks/cli/commands/operate.py +9 -36
  15. runbooks/cli/commands/security.py +42 -18
  16. runbooks/cli/commands/validation.py +432 -18
  17. runbooks/cli/commands/vpc.py +81 -17
  18. runbooks/cli/registry.py +22 -10
  19. runbooks/cloudops/__init__.py +20 -27
  20. runbooks/cloudops/base.py +96 -107
  21. runbooks/cloudops/cost_optimizer.py +544 -542
  22. runbooks/cloudops/infrastructure_optimizer.py +5 -4
  23. runbooks/cloudops/interfaces.py +224 -225
  24. runbooks/cloudops/lifecycle_manager.py +5 -4
  25. runbooks/cloudops/mcp_cost_validation.py +252 -235
  26. runbooks/cloudops/models.py +78 -53
  27. runbooks/cloudops/monitoring_automation.py +5 -4
  28. runbooks/cloudops/notebook_framework.py +177 -213
  29. runbooks/cloudops/security_enforcer.py +125 -159
  30. runbooks/common/accuracy_validator.py +17 -12
  31. runbooks/common/aws_pricing.py +349 -326
  32. runbooks/common/aws_pricing_api.py +211 -212
  33. runbooks/common/aws_profile_manager.py +40 -36
  34. runbooks/common/aws_utils.py +74 -79
  35. runbooks/common/business_logic.py +126 -104
  36. runbooks/common/cli_decorators.py +36 -60
  37. runbooks/common/comprehensive_cost_explorer_integration.py +455 -463
  38. runbooks/common/cross_account_manager.py +197 -204
  39. runbooks/common/date_utils.py +27 -39
  40. runbooks/common/decorators.py +29 -19
  41. runbooks/common/dry_run_examples.py +173 -208
  42. runbooks/common/dry_run_framework.py +157 -155
  43. runbooks/common/enhanced_exception_handler.py +15 -4
  44. runbooks/common/enhanced_logging_example.py +50 -64
  45. runbooks/common/enhanced_logging_integration_example.py +65 -37
  46. runbooks/common/env_utils.py +16 -16
  47. runbooks/common/error_handling.py +40 -38
  48. runbooks/common/lazy_loader.py +41 -23
  49. runbooks/common/logging_integration_helper.py +79 -86
  50. runbooks/common/mcp_cost_explorer_integration.py +476 -493
  51. runbooks/common/mcp_integration.py +99 -79
  52. runbooks/common/memory_optimization.py +140 -118
  53. runbooks/common/module_cli_base.py +37 -58
  54. runbooks/common/organizations_client.py +175 -193
  55. runbooks/common/patterns.py +23 -25
  56. runbooks/common/performance_monitoring.py +67 -71
  57. runbooks/common/performance_optimization_engine.py +283 -274
  58. runbooks/common/profile_utils.py +111 -37
  59. runbooks/common/rich_utils.py +315 -141
  60. runbooks/common/sre_performance_suite.py +177 -186
  61. runbooks/enterprise/__init__.py +1 -1
  62. runbooks/enterprise/logging.py +144 -106
  63. runbooks/enterprise/security.py +187 -204
  64. runbooks/enterprise/validation.py +43 -56
  65. runbooks/finops/__init__.py +26 -30
  66. runbooks/finops/account_resolver.py +1 -1
  67. runbooks/finops/advanced_optimization_engine.py +980 -0
  68. runbooks/finops/automation_core.py +268 -231
  69. runbooks/finops/business_case_config.py +184 -179
  70. runbooks/finops/cli.py +660 -139
  71. runbooks/finops/commvault_ec2_analysis.py +157 -164
  72. runbooks/finops/compute_cost_optimizer.py +336 -320
  73. runbooks/finops/config.py +20 -20
  74. runbooks/finops/cost_optimizer.py +484 -618
  75. runbooks/finops/cost_processor.py +332 -214
  76. runbooks/finops/dashboard_runner.py +1006 -172
  77. runbooks/finops/ebs_cost_optimizer.py +991 -657
  78. runbooks/finops/elastic_ip_optimizer.py +317 -257
  79. runbooks/finops/enhanced_mcp_integration.py +340 -0
  80. runbooks/finops/enhanced_progress.py +32 -29
  81. runbooks/finops/enhanced_trend_visualization.py +3 -2
  82. runbooks/finops/enterprise_wrappers.py +223 -285
  83. runbooks/finops/executive_export.py +203 -160
  84. runbooks/finops/helpers.py +130 -288
  85. runbooks/finops/iam_guidance.py +1 -1
  86. runbooks/finops/infrastructure/__init__.py +80 -0
  87. runbooks/finops/infrastructure/commands.py +506 -0
  88. runbooks/finops/infrastructure/load_balancer_optimizer.py +866 -0
  89. runbooks/finops/infrastructure/vpc_endpoint_optimizer.py +832 -0
  90. runbooks/finops/markdown_exporter.py +337 -174
  91. runbooks/finops/mcp_validator.py +1952 -0
  92. runbooks/finops/nat_gateway_optimizer.py +1512 -481
  93. runbooks/finops/network_cost_optimizer.py +657 -587
  94. runbooks/finops/notebook_utils.py +226 -188
  95. runbooks/finops/optimization_engine.py +1136 -0
  96. runbooks/finops/optimizer.py +19 -23
  97. runbooks/finops/rds_snapshot_optimizer.py +367 -411
  98. runbooks/finops/reservation_optimizer.py +427 -363
  99. runbooks/finops/scenario_cli_integration.py +64 -65
  100. runbooks/finops/scenarios.py +1277 -438
  101. runbooks/finops/schemas.py +218 -182
  102. runbooks/finops/snapshot_manager.py +2289 -0
  103. runbooks/finops/types.py +3 -3
  104. runbooks/finops/validation_framework.py +259 -265
  105. runbooks/finops/vpc_cleanup_exporter.py +189 -144
  106. runbooks/finops/vpc_cleanup_optimizer.py +591 -573
  107. runbooks/finops/workspaces_analyzer.py +171 -182
  108. runbooks/integration/__init__.py +89 -0
  109. runbooks/integration/mcp_integration.py +1920 -0
  110. runbooks/inventory/CLAUDE.md +816 -0
  111. runbooks/inventory/__init__.py +2 -2
  112. runbooks/inventory/aws_decorators.py +2 -3
  113. runbooks/inventory/check_cloudtrail_compliance.py +2 -4
  114. runbooks/inventory/check_controltower_readiness.py +152 -151
  115. runbooks/inventory/check_landingzone_readiness.py +85 -84
  116. runbooks/inventory/cloud_foundations_integration.py +144 -149
  117. runbooks/inventory/collectors/aws_comprehensive.py +1 -1
  118. runbooks/inventory/collectors/aws_networking.py +109 -99
  119. runbooks/inventory/collectors/base.py +4 -0
  120. runbooks/inventory/core/collector.py +495 -313
  121. runbooks/inventory/core/formatter.py +11 -0
  122. runbooks/inventory/draw_org_structure.py +8 -9
  123. runbooks/inventory/drift_detection_cli.py +69 -96
  124. runbooks/inventory/ec2_vpc_utils.py +2 -2
  125. runbooks/inventory/find_cfn_drift_detection.py +5 -7
  126. runbooks/inventory/find_cfn_orphaned_stacks.py +7 -9
  127. runbooks/inventory/find_cfn_stackset_drift.py +5 -6
  128. runbooks/inventory/find_ec2_security_groups.py +48 -42
  129. runbooks/inventory/find_landingzone_versions.py +4 -6
  130. runbooks/inventory/find_vpc_flow_logs.py +7 -9
  131. runbooks/inventory/inventory_mcp_cli.py +48 -46
  132. runbooks/inventory/inventory_modules.py +103 -91
  133. runbooks/inventory/list_cfn_stacks.py +9 -10
  134. runbooks/inventory/list_cfn_stackset_operation_results.py +1 -3
  135. runbooks/inventory/list_cfn_stackset_operations.py +79 -57
  136. runbooks/inventory/list_cfn_stacksets.py +8 -10
  137. runbooks/inventory/list_config_recorders_delivery_channels.py +49 -39
  138. runbooks/inventory/list_ds_directories.py +65 -53
  139. runbooks/inventory/list_ec2_availability_zones.py +2 -4
  140. runbooks/inventory/list_ec2_ebs_volumes.py +32 -35
  141. runbooks/inventory/list_ec2_instances.py +23 -28
  142. runbooks/inventory/list_ecs_clusters_and_tasks.py +26 -34
  143. runbooks/inventory/list_elbs_load_balancers.py +22 -20
  144. runbooks/inventory/list_enis_network_interfaces.py +26 -33
  145. runbooks/inventory/list_guardduty_detectors.py +2 -4
  146. runbooks/inventory/list_iam_policies.py +2 -4
  147. runbooks/inventory/list_iam_roles.py +5 -7
  148. runbooks/inventory/list_iam_saml_providers.py +4 -6
  149. runbooks/inventory/list_lambda_functions.py +38 -38
  150. runbooks/inventory/list_org_accounts.py +6 -8
  151. runbooks/inventory/list_org_accounts_users.py +55 -44
  152. runbooks/inventory/list_rds_db_instances.py +31 -33
  153. runbooks/inventory/list_rds_snapshots_aggregator.py +192 -208
  154. runbooks/inventory/list_route53_hosted_zones.py +3 -5
  155. runbooks/inventory/list_servicecatalog_provisioned_products.py +37 -41
  156. runbooks/inventory/list_sns_topics.py +2 -4
  157. runbooks/inventory/list_ssm_parameters.py +4 -7
  158. runbooks/inventory/list_vpc_subnets.py +2 -4
  159. runbooks/inventory/list_vpcs.py +7 -10
  160. runbooks/inventory/mcp_inventory_validator.py +554 -468
  161. runbooks/inventory/mcp_vpc_validator.py +359 -442
  162. runbooks/inventory/organizations_discovery.py +63 -55
  163. runbooks/inventory/recover_cfn_stack_ids.py +7 -8
  164. runbooks/inventory/requirements.txt +0 -1
  165. runbooks/inventory/rich_inventory_display.py +35 -34
  166. runbooks/inventory/run_on_multi_accounts.py +3 -5
  167. runbooks/inventory/unified_validation_engine.py +281 -253
  168. runbooks/inventory/verify_ec2_security_groups.py +1 -1
  169. runbooks/inventory/vpc_analyzer.py +735 -697
  170. runbooks/inventory/vpc_architecture_validator.py +293 -348
  171. runbooks/inventory/vpc_dependency_analyzer.py +384 -380
  172. runbooks/inventory/vpc_flow_analyzer.py +1 -1
  173. runbooks/main.py +49 -34
  174. runbooks/main_final.py +91 -60
  175. runbooks/main_minimal.py +22 -10
  176. runbooks/main_optimized.py +131 -100
  177. runbooks/main_ultra_minimal.py +7 -2
  178. runbooks/mcp/__init__.py +36 -0
  179. runbooks/mcp/integration.py +679 -0
  180. runbooks/monitoring/performance_monitor.py +9 -4
  181. runbooks/operate/dynamodb_operations.py +3 -1
  182. runbooks/operate/ec2_operations.py +145 -137
  183. runbooks/operate/iam_operations.py +146 -152
  184. runbooks/operate/networking_cost_heatmap.py +29 -8
  185. runbooks/operate/rds_operations.py +223 -254
  186. runbooks/operate/s3_operations.py +107 -118
  187. runbooks/operate/vpc_operations.py +646 -616
  188. runbooks/remediation/base.py +1 -1
  189. runbooks/remediation/commons.py +10 -7
  190. runbooks/remediation/commvault_ec2_analysis.py +70 -66
  191. runbooks/remediation/ec2_unattached_ebs_volumes.py +1 -0
  192. runbooks/remediation/multi_account.py +24 -21
  193. runbooks/remediation/rds_snapshot_list.py +86 -60
  194. runbooks/remediation/remediation_cli.py +92 -146
  195. runbooks/remediation/universal_account_discovery.py +83 -79
  196. runbooks/remediation/workspaces_list.py +46 -41
  197. runbooks/security/__init__.py +19 -0
  198. runbooks/security/assessment_runner.py +1150 -0
  199. runbooks/security/baseline_checker.py +812 -0
  200. runbooks/security/cloudops_automation_security_validator.py +509 -535
  201. runbooks/security/compliance_automation_engine.py +17 -17
  202. runbooks/security/config/__init__.py +2 -2
  203. runbooks/security/config/compliance_config.py +50 -50
  204. runbooks/security/config_template_generator.py +63 -76
  205. runbooks/security/enterprise_security_framework.py +1 -1
  206. runbooks/security/executive_security_dashboard.py +519 -508
  207. runbooks/security/multi_account_security_controls.py +959 -1210
  208. runbooks/security/real_time_security_monitor.py +422 -444
  209. runbooks/security/security_baseline_tester.py +1 -1
  210. runbooks/security/security_cli.py +143 -112
  211. runbooks/security/test_2way_validation.py +439 -0
  212. runbooks/security/two_way_validation_framework.py +852 -0
  213. runbooks/sre/production_monitoring_framework.py +167 -177
  214. runbooks/tdd/__init__.py +15 -0
  215. runbooks/tdd/cli.py +1071 -0
  216. runbooks/utils/__init__.py +14 -17
  217. runbooks/utils/logger.py +7 -2
  218. runbooks/utils/version_validator.py +50 -47
  219. runbooks/validation/__init__.py +6 -6
  220. runbooks/validation/cli.py +9 -3
  221. runbooks/validation/comprehensive_2way_validator.py +745 -704
  222. runbooks/validation/mcp_validator.py +906 -228
  223. runbooks/validation/terraform_citations_validator.py +104 -115
  224. runbooks/validation/terraform_drift_detector.py +461 -454
  225. runbooks/vpc/README.md +617 -0
  226. runbooks/vpc/__init__.py +8 -1
  227. runbooks/vpc/analyzer.py +577 -0
  228. runbooks/vpc/cleanup_wrapper.py +476 -413
  229. runbooks/vpc/cli_cloudtrail_commands.py +339 -0
  230. runbooks/vpc/cli_mcp_validation_commands.py +480 -0
  231. runbooks/vpc/cloudtrail_audit_integration.py +717 -0
  232. runbooks/vpc/config.py +92 -97
  233. runbooks/vpc/cost_engine.py +411 -148
  234. runbooks/vpc/cost_explorer_integration.py +553 -0
  235. runbooks/vpc/cross_account_session.py +101 -106
  236. runbooks/vpc/enhanced_mcp_validation.py +917 -0
  237. runbooks/vpc/eni_gate_validator.py +961 -0
  238. runbooks/vpc/heatmap_engine.py +185 -160
  239. runbooks/vpc/mcp_no_eni_validator.py +680 -639
  240. runbooks/vpc/nat_gateway_optimizer.py +358 -0
  241. runbooks/vpc/networking_wrapper.py +15 -8
  242. runbooks/vpc/pdca_remediation_planner.py +528 -0
  243. runbooks/vpc/performance_optimized_analyzer.py +219 -231
  244. runbooks/vpc/runbooks_adapter.py +1167 -241
  245. runbooks/vpc/tdd_red_phase_stubs.py +601 -0
  246. runbooks/vpc/test_data_loader.py +358 -0
  247. runbooks/vpc/tests/conftest.py +314 -4
  248. runbooks/vpc/tests/test_cleanup_framework.py +1022 -0
  249. runbooks/vpc/tests/test_cost_engine.py +0 -2
  250. runbooks/vpc/topology_generator.py +326 -0
  251. runbooks/vpc/unified_scenarios.py +1297 -1124
  252. runbooks/vpc/vpc_cleanup_integration.py +1943 -1115
  253. runbooks-1.1.6.dist-info/METADATA +327 -0
  254. runbooks-1.1.6.dist-info/RECORD +489 -0
  255. runbooks/finops/README.md +0 -414
  256. runbooks/finops/accuracy_cross_validator.py +0 -647
  257. runbooks/finops/business_cases.py +0 -950
  258. runbooks/finops/dashboard_router.py +0 -922
  259. runbooks/finops/ebs_optimizer.py +0 -973
  260. runbooks/finops/embedded_mcp_validator.py +0 -1629
  261. runbooks/finops/enhanced_dashboard_runner.py +0 -527
  262. runbooks/finops/finops_dashboard.py +0 -584
  263. runbooks/finops/finops_scenarios.py +0 -1218
  264. runbooks/finops/legacy_migration.py +0 -730
  265. runbooks/finops/multi_dashboard.py +0 -1519
  266. runbooks/finops/single_dashboard.py +0 -1113
  267. runbooks/finops/unlimited_scenarios.py +0 -393
  268. runbooks-1.1.4.dist-info/METADATA +0 -800
  269. runbooks-1.1.4.dist-info/RECORD +0 -468
  270. {runbooks-1.1.4.dist-info → runbooks-1.1.6.dist-info}/WHEEL +0 -0
  271. {runbooks-1.1.4.dist-info → runbooks-1.1.6.dist-info}/entry_points.txt +0 -0
  272. {runbooks-1.1.4.dist-info → runbooks-1.1.6.dist-info}/licenses/LICENSE +0 -0
  273. {runbooks-1.1.4.dist-info → runbooks-1.1.6.dist-info}/top_level.txt +0 -0
@@ -93,11 +93,11 @@ from time import time
93
93
  import Inventory_Modules
94
94
  from ArgumentsClass import CommonArguments
95
95
  from botocore.exceptions import ClientError
96
- from colorama import Fore, init
96
+ from runbooks.common.rich_utils import console
97
97
  from Inventory_Modules import del_config_recorder_or_delivery_channel2, display_results, get_all_credentials
98
- from tqdm.auto import tqdm
98
+ # Migrated to Rich.Progress - see rich_utils.py for enterprise UX standards
99
+ # from tqdm.auto import tqdm
99
100
 
100
- init()
101
101
  __version__ = "2024.05.31"
102
102
 
103
103
 
@@ -409,7 +409,7 @@ def check_accounts_for_delivery_channels_and_config_recorders(CredentialList, fF
409
409
  logging.info(
410
410
  f"{ERASE_LINE}Finished finding items in account {c_account_credentials['AccountId']} in region {c_account_credentials['Region']} - {c_PlaceCount} / {c_PlacesToLook}"
411
411
  )
412
- pbar.update() # Update progress bar for operational visibility
412
+ pbar.update(pbar_task, advance=1) # Update Rich progress bar for operational visibility
413
413
  self.queue.task_done() # Mark queue item as completed
414
414
 
415
415
  # Initialize processing context and data structures for Config discovery
@@ -421,40 +421,50 @@ def check_accounts_for_delivery_channels_and_config_recorders(CredentialList, fF
421
421
 
422
422
  checkqueue = Queue() # Queue for work distribution across worker threads
423
423
 
424
+ # Import Rich display utilities for professional progress tracking
425
+ from runbooks.common.rich_utils import create_progress_bar
426
+
424
427
  # Initialize progress tracking for operational visibility during Config discovery
425
- pbar = tqdm(
426
- desc=f"Finding config recorders / delivery channels from {len(AllCredentials)} accounts and regions",
427
- total=len(AllCredentials),
428
- unit=" accounts & regions",
429
- )
428
+ with create_progress_bar() as progress:
429
+ task = progress.add_task(
430
+ f"[cyan]Finding config recorders / delivery channels from {len(AllCredentials)} accounts and regions...",
431
+ total=len(AllCredentials)
432
+ )
430
433
 
431
- # Create and start worker thread pool for concurrent Config component discovery
432
- for x in range(WorkerThreads):
433
- worker = Find_Config_Recorders_and_Delivery_Channels(checkqueue)
434
- # Daemon threads allow main thread exit even if workers are still processing
435
- worker.daemon = True
436
- worker.start()
437
-
438
- # Queue Config discovery work items for worker thread processing
439
- # Note: Credential list already includes regional context, eliminating need for nested region iteration
440
- for credential in CredentialList:
441
- logging.info(f"Connecting to account {credential['AccountId']} in region {credential['Region']}")
442
- try:
443
- # Queue account and region combination for worker thread processing
444
- # Note: Tuple structure is critical for proper parameter expansion in worker threads
445
- checkqueue.put((credential, fFixRun, fFragments, len(CredentialList), PlaceCount))
446
- except ClientError as my_Error:
447
- # Handle authorization failures with informative error messaging
448
- if "AuthFailure" in str(my_Error):
449
- logging.error(
450
- f"Authorization Failure accessing account {credential['AccountId']} in {credential['Region']} region"
451
- )
452
- logging.warning(f"It's possible that the region {credential['Region']} hasn't been opted-into")
453
- pass
434
+ # Make progress object available to worker threads via global (multi-threaded pattern)
435
+ global pbar
436
+ pbar = progress
437
+ global pbar_task
438
+ pbar_task = task
439
+
440
+ # Create and start worker thread pool for concurrent Config component discovery
441
+ for x in range(WorkerThreads):
442
+ worker = Find_Config_Recorders_and_Delivery_Channels(checkqueue)
443
+ # Daemon threads allow main thread exit even if workers are still processing
444
+ worker.daemon = True
445
+ worker.start()
446
+
447
+ # Queue Config discovery work items for worker thread processing
448
+ # Note: Credential list already includes regional context, eliminating need for nested region iteration
449
+ for credential in CredentialList:
450
+ logging.info(f"Connecting to account {credential['AccountId']} in region {credential['Region']}")
451
+ try:
452
+ # Queue account and region combination for worker thread processing
453
+ # Note: Tuple structure is critical for proper parameter expansion in worker threads
454
+ checkqueue.put((credential, fFixRun, fFragments, len(CredentialList), PlaceCount))
455
+ except ClientError as my_Error:
456
+ # Handle authorization failures with informative error messaging
457
+ if "AuthFailure" in str(my_Error):
458
+ logging.error(
459
+ f"Authorization Failure accessing account {credential['AccountId']} in {credential['Region']} region"
460
+ )
461
+ logging.warning(f"It's possible that the region {credential['Region']} hasn't been opted-into")
462
+ pass
463
+
464
+ # Wait for all worker threads to complete processing
465
+ checkqueue.join()
466
+ # Progress bar auto-closes when exiting context manager
454
467
 
455
- # Wait for all worker threads to complete processing
456
- checkqueue.join()
457
- pbar.close() # Close progress bar after completion
458
468
  return account_crs_and_dcs
459
469
 
460
470
 
@@ -619,7 +629,7 @@ if __name__ == "__main__":
619
629
  print()
620
630
  milestone_time1 = time()
621
631
  print(
622
- f"{Fore.GREEN}\t\tFiguring out what regions are available to your accounts, and capturing credentials for all accounts in those regions took: {(milestone_time1 - begin_time):.3f} seconds{Fore.RESET}"
632
+ f"[green]\t\tFiguring out what regions are available to your accounts, and capturing credentials for all accounts in those regions took: {(milestone_time1 - begin_time):.3f} seconds"
623
633
  )
624
634
  print()
625
635
  print(f"Now running through all accounts and regions identified to find resources...")
@@ -631,7 +641,7 @@ if __name__ == "__main__":
631
641
  print()
632
642
  milestone_time2 = time()
633
643
  print(
634
- f"{Fore.GREEN}\t\tChecking {len(AllCredentials)} places took: {(milestone_time2 - milestone_time1):.3f} seconds{Fore.RESET}"
644
+ f"[green]\t\tChecking {len(AllCredentials)} places took: {(milestone_time2 - milestone_time1):.3f} seconds"
635
645
  )
636
646
  print()
637
647
  cr = 0
@@ -650,7 +660,7 @@ if __name__ == "__main__":
650
660
  print()
651
661
  milestone_time3 = time()
652
662
  print(
653
- f"{Fore.GREEN}\t\tSorting the list of places took: {(milestone_time3 - milestone_time2):.3f} seconds{Fore.RESET}"
663
+ f"[green]\t\tSorting the list of places took: {(milestone_time3 - milestone_time2):.3f} seconds"
654
664
  )
655
665
  print()
656
666
  display_results(all_sorted_config_recorders_and_delivery_channels, display_dict, None, pFilename)
@@ -675,7 +685,7 @@ if __name__ == "__main__":
675
685
 
676
686
  if pTiming:
677
687
  print(ERASE_LINE)
678
- print(f"{Fore.GREEN}This whole script took {time() - begin_time:.3f} seconds{Fore.RESET}")
688
+ print(f"[green]This whole script took {time() - begin_time:.3f} seconds")
679
689
  print()
680
690
  print("Thank you for using this tool")
681
691
  print()
@@ -60,11 +60,11 @@ from time import time
60
60
 
61
61
  from ArgumentsClass import CommonArguments
62
62
  from botocore.exceptions import ClientError
63
- from colorama import Fore, init
63
+ from runbooks.common.rich_utils import console
64
64
  from Inventory_Modules import display_results, find_directories2, get_all_credentials
65
- from tqdm.auto import tqdm
65
+ # Migrated to Rich.Progress - see rich_utils.py for enterprise UX standards
66
+ # from tqdm.auto import tqdm
66
67
 
67
- init()
68
68
  __version__ = "2024.05.31"
69
69
 
70
70
 
@@ -193,54 +193,66 @@ def find_all_directories(f_credentials, f_fragments, f_exact):
193
193
  """
194
194
  AllDirectories = [] # Aggregated list for all discovered directories
195
195
 
196
+ # Import Rich display utilities for professional progress tracking
197
+ from runbooks.common.rich_utils import create_progress_bar
198
+
196
199
  # TODO: Need to use multi-threading here for improved performance
197
200
  # Sequential processing with progress tracking for operational visibility
198
- for credential in tqdm(
199
- f_credentials, desc=f"Looking through {len(f_credentials)} accounts and regions", unit="credentials"
200
- ):
201
- logging.info(f"{ERASE_LINE}Looking in account: {credential['AccountId']} in region {credential['Region']}")
202
-
203
- # Skip failed credentials to avoid API errors
204
- if not credential["Success"]:
205
- continue
206
-
207
- try:
208
- # Discover directories using Directory Service API with fragment filtering
209
- directories = find_directories2(credential, credential["Region"], f_fragments, f_exact)
210
- logging.info(f"directories: {directories}")
211
- logging.info(
212
- f"{ERASE_LINE}Account: {credential['AccountId']} Region: {credential['Region']} Found {len(directories)} directories"
213
- )
214
-
215
- # Process and aggregate discovered directories with organizational context
216
- if directories:
217
- for directory in directories:
218
- # Enhance directory metadata with organizational and regional context
219
- # Available directory metadata includes:
220
- # - DirectoryName: Human-readable directory identifier
221
- # - DirectoryId: Unique AWS Directory Service identifier
222
- # - HomeRegion: Primary directory service region
223
- # - Status: Operational status (Active, Creating, etc.)
224
- # - Type: Directory type (SimpleAD, MicrosoftAD, etc.)
225
- # - Owner: Directory ownership context
226
-
227
- directory.update(
228
- {
229
- "MgmtAccount": credential["MgmtAccount"], # Management account context
230
- "Region": credential["Region"], # Regional deployment information
231
- "AccountId": credential["AccountId"], # Account ownership details
232
- }
233
- )
234
- AllDirectories.append(directory)
235
-
236
- except TypeError as my_Error:
237
- # Handle type errors from malformed Directory Service API responses
238
- logging.info(f"Error: {my_Error}")
239
- continue
240
- except ClientError as my_Error:
241
- # Handle AWS API authorization failures with informative logging
242
- if "AuthFailure" in str(my_Error):
243
- logging.error(f"{ERASE_LINE} Account {credential['AccountId']} : Authorization Failure")
201
+ with create_progress_bar() as progress:
202
+ task = progress.add_task(
203
+ f"[cyan]Looking through {len(f_credentials)} accounts and regions...",
204
+ total=len(f_credentials)
205
+ )
206
+
207
+ for credential in f_credentials:
208
+ logging.info(f"Looking in account: {credential['AccountId']} in region {credential['Region']}")
209
+
210
+ # Skip failed credentials to avoid API errors
211
+ if not credential["Success"]:
212
+ progress.update(task, advance=1)
213
+ continue
214
+
215
+ try:
216
+ # Discover directories using Directory Service API with fragment filtering
217
+ directories = find_directories2(credential, credential["Region"], f_fragments, f_exact)
218
+ logging.info(f"directories: {directories}")
219
+ logging.info(
220
+ f"Account: {credential['AccountId']} Region: {credential['Region']} Found {len(directories)} directories"
221
+ )
222
+
223
+ # Process and aggregate discovered directories with organizational context
224
+ if directories:
225
+ for directory in directories:
226
+ # Enhance directory metadata with organizational and regional context
227
+ # Available directory metadata includes:
228
+ # - DirectoryName: Human-readable directory identifier
229
+ # - DirectoryId: Unique AWS Directory Service identifier
230
+ # - HomeRegion: Primary directory service region
231
+ # - Status: Operational status (Active, Creating, etc.)
232
+ # - Type: Directory type (SimpleAD, MicrosoftAD, etc.)
233
+ # - Owner: Directory ownership context
234
+
235
+ directory.update(
236
+ {
237
+ "MgmtAccount": credential["MgmtAccount"], # Management account context
238
+ "Region": credential["Region"], # Regional deployment information
239
+ "AccountId": credential["AccountId"], # Account ownership details
240
+ }
241
+ )
242
+ AllDirectories.append(directory)
243
+
244
+ except TypeError as my_Error:
245
+ # Handle type errors from malformed Directory Service API responses
246
+ logging.info(f"Error: {my_Error}")
247
+ progress.update(task, advance=1)
248
+ continue
249
+ except ClientError as my_Error:
250
+ # Handle AWS API authorization failures with informative logging
251
+ if "AuthFailure" in str(my_Error):
252
+ logging.error(f" Account {credential['AccountId']} : Authorization Failure")
253
+
254
+ # Update progress after processing each credential
255
+ progress.update(task, advance=1)
244
256
 
245
257
  return AllDirectories
246
258
 
@@ -293,7 +305,7 @@ if __name__ == "__main__":
293
305
  # Display credential retrieval timing for performance optimization
294
306
  if pTiming:
295
307
  print(
296
- f"{Fore.GREEN}\tAfter getting credentials, this script took {time() - begin_time:.3f} seconds{Fore.RESET}"
308
+ f"[green]\tAfter getting credentials, this script took {time() - begin_time:.3f} seconds"
297
309
  )
298
310
  print()
299
311
 
@@ -304,7 +316,7 @@ if __name__ == "__main__":
304
316
  # Display credential parsing timing for operational metrics
305
317
  if pTiming:
306
318
  print(
307
- f"{Fore.GREEN}\tAfter parsing out all Regions, Account and Profiles, this script took {time() - begin_time:.3f} seconds{Fore.RESET}"
319
+ f"[green]\tAfter parsing out all Regions, Account and Profiles, this script took {time() - begin_time:.3f} seconds"
308
320
  )
309
321
  print()
310
322
 
@@ -340,7 +352,7 @@ if __name__ == "__main__":
340
352
  display_results(sorted_Results, display_dict, "None")
341
353
 
342
354
  # Provide operational summary with discovery metrics and performance timing
343
- print(ERASE_LINE)
355
+ console.print()
344
356
  print(
345
357
  f"Found {len(all_directories)} directories across {len(AccountList)} accounts across {len(RegionList)} regions"
346
358
  )
@@ -348,7 +360,7 @@ if __name__ == "__main__":
348
360
 
349
361
  # Display total execution timing for performance analysis and optimization
350
362
  if pTiming:
351
- print(f"{Fore.GREEN}\tThis script took {time() - begin_time:.3f} seconds{Fore.RESET}")
363
+ print(f"[green]\tThis script took {time() - begin_time:.3f} seconds")
352
364
  print()
353
365
  print("Thank you for using this script")
354
366
  print()
@@ -67,12 +67,10 @@ import sys
67
67
  from time import time
68
68
 
69
69
  from ArgumentsClass import CommonArguments
70
- from colorama import Fore, init
70
+ from runbooks.common.rich_utils import console
71
71
  from Inventory_Modules import display_results, get_all_credentials, get_region_azs2
72
72
 
73
- init()
74
73
  __version__ = "2024.03.06"
75
- ERASE_LINE = "\x1b[2K"
76
74
  begin_time = time()
77
75
 
78
76
 
@@ -281,6 +279,6 @@ if __name__ == "__main__":
281
279
 
282
280
  print()
283
281
  if pTiming:
284
- print(f"{Fore.GREEN}This script took {time() - begin_time:.2f} seconds{Fore.RESET}")
282
+ print(f"[green]This script took {time() - begin_time:.2f} seconds")
285
283
  print("Thanks for using this script")
286
284
  print()
@@ -7,15 +7,13 @@ from time import time
7
7
 
8
8
  from ArgumentsClass import CommonArguments
9
9
  from botocore.exceptions import ClientError
10
- from colorama import Fore, init
10
+ from runbooks.common.rich_utils import console
11
11
  from Inventory_Modules import display_results, find_account_volumes2, get_all_credentials
12
- from tqdm.auto import tqdm
12
+ from runbooks.common.rich_utils import create_progress_bar
13
13
 
14
- init()
15
14
  __version__ = "2024.05.31"
16
15
 
17
16
  # ANSI escape code for clearing current line (progress bar cleanup)
18
- ERASE_LINE = "\x1b[2K"
19
17
 
20
18
 
21
19
  ##################
@@ -92,7 +90,7 @@ def present_results(fVolumesFound: list):
92
90
  # Calculate and display orphaned volumes
93
91
  orphaned_volumes = [vol for vol in de_dupe_VolumesFound if vol.get("State") == "available"]
94
92
  if orphaned_volumes:
95
- print(f"{Fore.YELLOW}Warning: {len(orphaned_volumes)} orphaned (unattached) volumes found{Fore.RESET}")
93
+ print(f"[yellow]Warning: {len(orphaned_volumes)} orphaned (unattached) volumes found")
96
94
  total_orphaned_size = sum(vol.get("Size", 0) for vol in orphaned_volumes)
97
95
  print(f"Total orphaned storage: {total_orphaned_size} GB")
98
96
 
@@ -136,9 +134,9 @@ def check_accounts_for_ebs_volumes(f_CredentialList, f_fragment_list=None):
136
134
  continue
137
135
  finally:
138
136
  logging.info(
139
- f"{ERASE_LINE}Finished finding EBS volumes in account {c_account_credentials['AccountId']} in region {c_account_credentials['Region']}"
137
+ f"Finished finding EBS volumes in account {c_account_credentials['AccountId']} in region {c_account_credentials['Region']}"
140
138
  )
141
- pbar.update()
139
+ progress.update(task, advance=1)
142
140
  self.queue.task_done()
143
141
 
144
142
  if f_fragment_list is None:
@@ -148,32 +146,31 @@ def check_accounts_for_ebs_volumes(f_CredentialList, f_fragment_list=None):
148
146
 
149
147
  checkqueue = Queue()
150
148
 
151
- pbar = tqdm(
152
- desc=f"Finding ebs volumes from {len(f_CredentialList)} accounts and regions",
153
- total=len(f_CredentialList),
154
- unit=" accounts & regions",
155
- )
156
-
157
- for x in range(WorkerThreads):
158
- worker = FindVolumes(checkqueue)
159
- # Setting daemon to True will let the main thread exit even though the workers are blocking
160
- worker.daemon = True
161
- worker.start()
162
-
163
- for credential in f_CredentialList:
164
- logging.info(f"Connecting to account {credential['AccountId']}")
165
- try:
166
- # print(f"{ERASE_LINE}Queuing account {credential['AccountId']} in region {region}", end='\r')
167
- checkqueue.put((credential, credential["Region"], f_fragment_list))
168
- except ClientError as my_Error:
169
- if "AuthFailure" in str(my_Error):
170
- logging.error(
171
- f"Authorization Failure accessing account {credential['AccountId']} in '{credential['Region']}' region"
172
- )
173
- logging.warning(f"It's possible that the region '{credential['Region']}' hasn't been opted-into")
174
- pass
175
- checkqueue.join()
176
- pbar.close()
149
+ with create_progress_bar() as progress:
150
+ task = progress.add_task(
151
+ f"Finding ebs volumes from {len(f_CredentialList)} accounts and regions",
152
+ total=len(f_CredentialList)
153
+ )
154
+
155
+ for x in range(WorkerThreads):
156
+ worker = FindVolumes(checkqueue)
157
+ # Setting daemon to True will let the main thread exit even though the workers are blocking
158
+ worker.daemon = True
159
+ worker.start()
160
+
161
+ for credential in f_CredentialList:
162
+ logging.info(f"Connecting to account {credential['AccountId']}")
163
+ try:
164
+ # print(f"Queuing account {credential['AccountId']} in region {region}", end='\r')
165
+ checkqueue.put((credential, credential["Region"], f_fragment_list))
166
+ except ClientError as my_Error:
167
+ if "AuthFailure" in str(my_Error):
168
+ logging.error(
169
+ f"Authorization Failure accessing account {credential['AccountId']} in '{credential['Region']}' region"
170
+ )
171
+ logging.warning(f"It's possible that the region '{credential['Region']}' hasn't been opted-into")
172
+ pass
173
+ checkqueue.join()
177
174
  return AllVolumes
178
175
 
179
176
 
@@ -231,8 +228,8 @@ def main():
231
228
 
232
229
  # Display execution timing if requested
233
230
  if pTiming:
234
- print(ERASE_LINE)
235
- print(f"{Fore.GREEN}This script completed in {time() - begin_time:.2f} seconds{Fore.RESET}")
231
+ console.print()
232
+ print(f"[green]This script completed in {time() - begin_time:.2f} seconds")
236
233
 
237
234
 
238
235
  if __name__ == "__main__":
@@ -62,13 +62,10 @@ from time import time
62
62
  from . import inventory_modules as Inventory_Modules
63
63
  from .ArgumentsClass import CommonArguments
64
64
  from botocore.exceptions import ClientError
65
- from colorama import Fore, init
66
65
  from .inventory_modules import display_results, get_all_credentials
67
- from tqdm.auto import tqdm
66
+ from runbooks.common.rich_utils import create_progress_bar
68
67
 
69
- init()
70
68
  __version__ = "2025.04.09"
71
- ERASE_LINE = "\x1b[2K"
72
69
  begin_time = time()
73
70
 
74
71
 
@@ -322,30 +319,28 @@ def find_all_instances(fAllCredentials: list, fStatus: str = None) -> list:
322
319
  AllInstances = []
323
320
  WorkerThreads = min(len(fAllCredentials), 25)
324
321
 
325
- pbar = tqdm(
326
- desc=f"Finding instances from {len(fAllCredentials)} locations", total=len(fAllCredentials), unit=" locations"
327
- )
328
-
329
- for x in range(WorkerThreads):
330
- worker = FindInstances(checkqueue)
331
- # Setting daemon to True will let the main thread exit even though the workers are blocking
332
- worker.daemon = True
333
- worker.start()
334
-
335
- for credential in fAllCredentials:
336
- logging.info(f"Beginning to queue data - starting with {credential['AccountId']}")
337
- try:
338
- # I don't know why - but double parens are necessary below. If you remove them, only the first parameter is queued.
339
- checkqueue.put((credential))
340
- except ClientError as my_Error:
341
- if "AuthFailure" in str(my_Error):
342
- logging.error(
343
- f"Authorization Failure accessing account {credential['AccountId']} in {credential['Region']} region"
344
- )
345
- logging.warning(f"It's possible that the region {credential['Region']} hasn't been opted-into")
346
- pass
347
- checkqueue.join()
348
- pbar.close()
322
+ with create_progress_bar() as progress:
323
+ task = progress.add_task(f"Finding instances from {len(fAllCredentials)} locations", total=len(fAllCredentials))
324
+
325
+ for x in range(WorkerThreads):
326
+ worker = FindInstances(checkqueue)
327
+ # Setting daemon to True will let the main thread exit even though the workers are blocking
328
+ worker.daemon = True
329
+ worker.start()
330
+
331
+ for credential in fAllCredentials:
332
+ logging.info(f"Beginning to queue data - starting with {credential['AccountId']}")
333
+ try:
334
+ # I don't know why - but double parens are necessary below. If you remove them, only the first parameter is queued.
335
+ checkqueue.put((credential))
336
+ except ClientError as my_Error:
337
+ if "AuthFailure" in str(my_Error):
338
+ logging.error(
339
+ f"Authorization Failure accessing account {credential['AccountId']} in {credential['Region']} region"
340
+ )
341
+ logging.warning(f"It's possible that the region {credential['Region']} hasn't been opted-into")
342
+ pass
343
+ checkqueue.join()
349
344
  return AllInstances
350
345
 
351
346
 
@@ -90,13 +90,11 @@ from time import time
90
90
  import Inventory_Modules
91
91
  from ArgumentsClass import CommonArguments
92
92
  from botocore.exceptions import ClientError
93
- from colorama import Fore, init
93
+ from runbooks.common.rich_utils import console
94
94
  from Inventory_Modules import display_results, find_account_ecs_clusters_services_and_tasks2, get_all_credentials
95
- from tqdm.auto import tqdm
95
+ from runbooks.common.rich_utils import create_progress_bar
96
96
 
97
- init()
98
97
  __version__ = "2024.09.06"
99
- ERASE_LINE = "\x1b[2K"
100
98
  begin_time = time()
101
99
 
102
100
  # TODO: Need a table at the bottom that summarizes the results, by instance-type, by running/ stopped, maybe by account and region
@@ -246,7 +244,7 @@ def find_all_clusters_and_tasks(fAllCredentials: list, fStatus: str = None) -> l
246
244
  """
247
245
  while True:
248
246
  # Retrieve ECS discovery work item from thread-safe queue
249
- c_account_credentials = self.queue.get()
247
+ c_account_credentials, c_progress, c_task = self.queue.get()
250
248
  logging.info(f"De-queued info for account number {c_account_credentials['AccountId']}")
251
249
 
252
250
  try:
@@ -406,7 +404,7 @@ def find_all_clusters_and_tasks(fAllCredentials: list, fStatus: str = None) -> l
406
404
  continue
407
405
  finally:
408
406
  # Ensure progress tracking and queue management regardless of success/failure
409
- pbar.update()
407
+ c_progress.update(c_task, advance=1)
410
408
  self.queue.task_done()
411
409
 
412
410
  ###########
@@ -423,13 +421,6 @@ def find_all_clusters_and_tasks(fAllCredentials: list, fStatus: str = None) -> l
423
421
  # Maximum 25 threads to prevent overwhelming AWS APIs while maintaining efficiency
424
422
  WorkerThreads = min(len(fAllCredentials), 25)
425
423
 
426
- # Initialize progress tracking for operational visibility during large-scale operations
427
- pbar = tqdm(
428
- desc=f"Finding ECS clusters, services and tasks from {len(fAllCredentials)} accounts / regions",
429
- total=len(fAllCredentials),
430
- unit=" locations",
431
- )
432
-
433
424
  # Start worker threads for concurrent ECS resource discovery
434
425
  for x in range(WorkerThreads):
435
426
  worker = FindInstances(checkqueue)
@@ -437,25 +428,26 @@ def find_all_clusters_and_tasks(fAllCredentials: list, fStatus: str = None) -> l
437
428
  worker.daemon = True
438
429
  worker.start()
439
430
 
440
- # Queue credential sets for processing by worker threads
441
- for credential in fAllCredentials:
442
- logging.info(f"Beginning to queue data - starting with {credential['AccountId']}")
443
- try:
444
- # Queue individual credential set for ECS resource discovery
445
- # Note: Single parameter queuing - credential dictionary contains all needed info
446
- checkqueue.put(credential)
447
- except ClientError as my_Error:
448
- # Handle authorization failures during credential queuing
449
- if "AuthFailure" in str(my_Error):
450
- logging.error(
451
- f"Authorization Failure accessing account {credential['AccountId']} in {credential['Region']} region"
452
- )
453
- logging.warning(f"It's possible that the region {credential['Region']} hasn't been opted-into")
454
- pass
455
-
456
- # Wait for all queued work to complete before proceeding
457
- checkqueue.join()
458
- pbar.close()
431
+ # Queue credential sets for processing by worker threads with progress tracking
432
+ with create_progress_bar() as progress:
433
+ task = progress.add_task("Discovering ECS clusters, services and tasks", total=len(fAllCredentials))
434
+ for credential in fAllCredentials:
435
+ logging.info(f"Beginning to queue data - starting with {credential['AccountId']}")
436
+ try:
437
+ # Queue individual credential set for ECS resource discovery with progress tracking
438
+ # Tuple format: (credential, progress, task)
439
+ checkqueue.put((credential, progress, task))
440
+ except ClientError as my_Error:
441
+ # Handle authorization failures during credential queuing
442
+ if "AuthFailure" in str(my_Error):
443
+ logging.error(
444
+ f"Authorization Failure accessing account {credential['AccountId']} in {credential['Region']} region"
445
+ )
446
+ logging.warning(f"It's possible that the region {credential['Region']} hasn't been opted-into")
447
+ pass
448
+
449
+ # Wait for all queued work to complete before proceeding
450
+ checkqueue.join()
459
451
  return AllInstances
460
452
 
461
453
 
@@ -514,7 +506,7 @@ if __name__ == "__main__":
514
506
  print()
515
507
  milestone_time1 = time()
516
508
  print(
517
- f"{Fore.GREEN}\t\tCredential discovery and region enumeration took: {(milestone_time1 - begin_time):.3f} seconds{Fore.RESET}"
509
+ f"[green]\t\tCredential discovery and region enumeration took: {(milestone_time1 - begin_time):.3f} seconds"
518
510
  )
519
511
  print()
520
512
 
@@ -549,7 +541,7 @@ if __name__ == "__main__":
549
541
  # Display performance timing metrics for operational optimization and SLA compliance
550
542
  if pTiming:
551
543
  print(ERASE_LINE)
552
- print(f"{Fore.GREEN}This script took {time() - begin_time:.2f} seconds{Fore.RESET}")
544
+ print(f"[green]This script took {time() - begin_time:.2f} seconds")
553
545
 
554
546
  print(ERASE_LINE)
555
547