runbooks 1.1.4__py3-none-any.whl → 1.1.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runbooks/__init__.py +31 -2
- runbooks/__init___optimized.py +18 -4
- runbooks/_platform/__init__.py +1 -5
- runbooks/_platform/core/runbooks_wrapper.py +141 -138
- runbooks/aws2/accuracy_validator.py +812 -0
- runbooks/base.py +7 -0
- runbooks/cfat/assessment/compliance.py +1 -1
- runbooks/cfat/assessment/runner.py +1 -0
- runbooks/cfat/cloud_foundations_assessment.py +227 -239
- runbooks/cli/__init__.py +1 -1
- runbooks/cli/commands/cfat.py +64 -23
- runbooks/cli/commands/finops.py +1005 -54
- runbooks/cli/commands/inventory.py +135 -91
- runbooks/cli/commands/operate.py +9 -36
- runbooks/cli/commands/security.py +42 -18
- runbooks/cli/commands/validation.py +432 -18
- runbooks/cli/commands/vpc.py +81 -17
- runbooks/cli/registry.py +22 -10
- runbooks/cloudops/__init__.py +20 -27
- runbooks/cloudops/base.py +96 -107
- runbooks/cloudops/cost_optimizer.py +544 -542
- runbooks/cloudops/infrastructure_optimizer.py +5 -4
- runbooks/cloudops/interfaces.py +224 -225
- runbooks/cloudops/lifecycle_manager.py +5 -4
- runbooks/cloudops/mcp_cost_validation.py +252 -235
- runbooks/cloudops/models.py +78 -53
- runbooks/cloudops/monitoring_automation.py +5 -4
- runbooks/cloudops/notebook_framework.py +177 -213
- runbooks/cloudops/security_enforcer.py +125 -159
- runbooks/common/accuracy_validator.py +17 -12
- runbooks/common/aws_pricing.py +349 -326
- runbooks/common/aws_pricing_api.py +211 -212
- runbooks/common/aws_profile_manager.py +40 -36
- runbooks/common/aws_utils.py +74 -79
- runbooks/common/business_logic.py +126 -104
- runbooks/common/cli_decorators.py +36 -60
- runbooks/common/comprehensive_cost_explorer_integration.py +455 -463
- runbooks/common/cross_account_manager.py +197 -204
- runbooks/common/date_utils.py +27 -39
- runbooks/common/decorators.py +29 -19
- runbooks/common/dry_run_examples.py +173 -208
- runbooks/common/dry_run_framework.py +157 -155
- runbooks/common/enhanced_exception_handler.py +15 -4
- runbooks/common/enhanced_logging_example.py +50 -64
- runbooks/common/enhanced_logging_integration_example.py +65 -37
- runbooks/common/env_utils.py +16 -16
- runbooks/common/error_handling.py +40 -38
- runbooks/common/lazy_loader.py +41 -23
- runbooks/common/logging_integration_helper.py +79 -86
- runbooks/common/mcp_cost_explorer_integration.py +476 -493
- runbooks/common/mcp_integration.py +99 -79
- runbooks/common/memory_optimization.py +140 -118
- runbooks/common/module_cli_base.py +37 -58
- runbooks/common/organizations_client.py +175 -193
- runbooks/common/patterns.py +23 -25
- runbooks/common/performance_monitoring.py +67 -71
- runbooks/common/performance_optimization_engine.py +283 -274
- runbooks/common/profile_utils.py +111 -37
- runbooks/common/rich_utils.py +315 -141
- runbooks/common/sre_performance_suite.py +177 -186
- runbooks/enterprise/__init__.py +1 -1
- runbooks/enterprise/logging.py +144 -106
- runbooks/enterprise/security.py +187 -204
- runbooks/enterprise/validation.py +43 -56
- runbooks/finops/__init__.py +26 -30
- runbooks/finops/account_resolver.py +1 -1
- runbooks/finops/advanced_optimization_engine.py +980 -0
- runbooks/finops/automation_core.py +268 -231
- runbooks/finops/business_case_config.py +184 -179
- runbooks/finops/cli.py +660 -139
- runbooks/finops/commvault_ec2_analysis.py +157 -164
- runbooks/finops/compute_cost_optimizer.py +336 -320
- runbooks/finops/config.py +20 -20
- runbooks/finops/cost_optimizer.py +484 -618
- runbooks/finops/cost_processor.py +332 -214
- runbooks/finops/dashboard_runner.py +1006 -172
- runbooks/finops/ebs_cost_optimizer.py +991 -657
- runbooks/finops/elastic_ip_optimizer.py +317 -257
- runbooks/finops/enhanced_mcp_integration.py +340 -0
- runbooks/finops/enhanced_progress.py +32 -29
- runbooks/finops/enhanced_trend_visualization.py +3 -2
- runbooks/finops/enterprise_wrappers.py +223 -285
- runbooks/finops/executive_export.py +203 -160
- runbooks/finops/helpers.py +130 -288
- runbooks/finops/iam_guidance.py +1 -1
- runbooks/finops/infrastructure/__init__.py +80 -0
- runbooks/finops/infrastructure/commands.py +506 -0
- runbooks/finops/infrastructure/load_balancer_optimizer.py +866 -0
- runbooks/finops/infrastructure/vpc_endpoint_optimizer.py +832 -0
- runbooks/finops/markdown_exporter.py +337 -174
- runbooks/finops/mcp_validator.py +1952 -0
- runbooks/finops/nat_gateway_optimizer.py +1512 -481
- runbooks/finops/network_cost_optimizer.py +657 -587
- runbooks/finops/notebook_utils.py +226 -188
- runbooks/finops/optimization_engine.py +1136 -0
- runbooks/finops/optimizer.py +19 -23
- runbooks/finops/rds_snapshot_optimizer.py +367 -411
- runbooks/finops/reservation_optimizer.py +427 -363
- runbooks/finops/scenario_cli_integration.py +64 -65
- runbooks/finops/scenarios.py +1277 -438
- runbooks/finops/schemas.py +218 -182
- runbooks/finops/snapshot_manager.py +2289 -0
- runbooks/finops/types.py +3 -3
- runbooks/finops/validation_framework.py +259 -265
- runbooks/finops/vpc_cleanup_exporter.py +189 -144
- runbooks/finops/vpc_cleanup_optimizer.py +591 -573
- runbooks/finops/workspaces_analyzer.py +171 -182
- runbooks/integration/__init__.py +89 -0
- runbooks/integration/mcp_integration.py +1920 -0
- runbooks/inventory/CLAUDE.md +816 -0
- runbooks/inventory/__init__.py +2 -2
- runbooks/inventory/aws_decorators.py +2 -3
- runbooks/inventory/check_cloudtrail_compliance.py +2 -4
- runbooks/inventory/check_controltower_readiness.py +152 -151
- runbooks/inventory/check_landingzone_readiness.py +85 -84
- runbooks/inventory/cloud_foundations_integration.py +144 -149
- runbooks/inventory/collectors/aws_comprehensive.py +1 -1
- runbooks/inventory/collectors/aws_networking.py +109 -99
- runbooks/inventory/collectors/base.py +4 -0
- runbooks/inventory/core/collector.py +495 -313
- runbooks/inventory/core/formatter.py +11 -0
- runbooks/inventory/draw_org_structure.py +8 -9
- runbooks/inventory/drift_detection_cli.py +69 -96
- runbooks/inventory/ec2_vpc_utils.py +2 -2
- runbooks/inventory/find_cfn_drift_detection.py +5 -7
- runbooks/inventory/find_cfn_orphaned_stacks.py +7 -9
- runbooks/inventory/find_cfn_stackset_drift.py +5 -6
- runbooks/inventory/find_ec2_security_groups.py +48 -42
- runbooks/inventory/find_landingzone_versions.py +4 -6
- runbooks/inventory/find_vpc_flow_logs.py +7 -9
- runbooks/inventory/inventory_mcp_cli.py +48 -46
- runbooks/inventory/inventory_modules.py +103 -91
- runbooks/inventory/list_cfn_stacks.py +9 -10
- runbooks/inventory/list_cfn_stackset_operation_results.py +1 -3
- runbooks/inventory/list_cfn_stackset_operations.py +79 -57
- runbooks/inventory/list_cfn_stacksets.py +8 -10
- runbooks/inventory/list_config_recorders_delivery_channels.py +49 -39
- runbooks/inventory/list_ds_directories.py +65 -53
- runbooks/inventory/list_ec2_availability_zones.py +2 -4
- runbooks/inventory/list_ec2_ebs_volumes.py +32 -35
- runbooks/inventory/list_ec2_instances.py +23 -28
- runbooks/inventory/list_ecs_clusters_and_tasks.py +26 -34
- runbooks/inventory/list_elbs_load_balancers.py +22 -20
- runbooks/inventory/list_enis_network_interfaces.py +26 -33
- runbooks/inventory/list_guardduty_detectors.py +2 -4
- runbooks/inventory/list_iam_policies.py +2 -4
- runbooks/inventory/list_iam_roles.py +5 -7
- runbooks/inventory/list_iam_saml_providers.py +4 -6
- runbooks/inventory/list_lambda_functions.py +38 -38
- runbooks/inventory/list_org_accounts.py +6 -8
- runbooks/inventory/list_org_accounts_users.py +55 -44
- runbooks/inventory/list_rds_db_instances.py +31 -33
- runbooks/inventory/list_rds_snapshots_aggregator.py +192 -208
- runbooks/inventory/list_route53_hosted_zones.py +3 -5
- runbooks/inventory/list_servicecatalog_provisioned_products.py +37 -41
- runbooks/inventory/list_sns_topics.py +2 -4
- runbooks/inventory/list_ssm_parameters.py +4 -7
- runbooks/inventory/list_vpc_subnets.py +2 -4
- runbooks/inventory/list_vpcs.py +7 -10
- runbooks/inventory/mcp_inventory_validator.py +554 -468
- runbooks/inventory/mcp_vpc_validator.py +359 -442
- runbooks/inventory/organizations_discovery.py +63 -55
- runbooks/inventory/recover_cfn_stack_ids.py +7 -8
- runbooks/inventory/requirements.txt +0 -1
- runbooks/inventory/rich_inventory_display.py +35 -34
- runbooks/inventory/run_on_multi_accounts.py +3 -5
- runbooks/inventory/unified_validation_engine.py +281 -253
- runbooks/inventory/verify_ec2_security_groups.py +1 -1
- runbooks/inventory/vpc_analyzer.py +735 -697
- runbooks/inventory/vpc_architecture_validator.py +293 -348
- runbooks/inventory/vpc_dependency_analyzer.py +384 -380
- runbooks/inventory/vpc_flow_analyzer.py +1 -1
- runbooks/main.py +49 -34
- runbooks/main_final.py +91 -60
- runbooks/main_minimal.py +22 -10
- runbooks/main_optimized.py +131 -100
- runbooks/main_ultra_minimal.py +7 -2
- runbooks/mcp/__init__.py +36 -0
- runbooks/mcp/integration.py +679 -0
- runbooks/monitoring/performance_monitor.py +9 -4
- runbooks/operate/dynamodb_operations.py +3 -1
- runbooks/operate/ec2_operations.py +145 -137
- runbooks/operate/iam_operations.py +146 -152
- runbooks/operate/networking_cost_heatmap.py +29 -8
- runbooks/operate/rds_operations.py +223 -254
- runbooks/operate/s3_operations.py +107 -118
- runbooks/operate/vpc_operations.py +646 -616
- runbooks/remediation/base.py +1 -1
- runbooks/remediation/commons.py +10 -7
- runbooks/remediation/commvault_ec2_analysis.py +70 -66
- runbooks/remediation/ec2_unattached_ebs_volumes.py +1 -0
- runbooks/remediation/multi_account.py +24 -21
- runbooks/remediation/rds_snapshot_list.py +86 -60
- runbooks/remediation/remediation_cli.py +92 -146
- runbooks/remediation/universal_account_discovery.py +83 -79
- runbooks/remediation/workspaces_list.py +46 -41
- runbooks/security/__init__.py +19 -0
- runbooks/security/assessment_runner.py +1150 -0
- runbooks/security/baseline_checker.py +812 -0
- runbooks/security/cloudops_automation_security_validator.py +509 -535
- runbooks/security/compliance_automation_engine.py +17 -17
- runbooks/security/config/__init__.py +2 -2
- runbooks/security/config/compliance_config.py +50 -50
- runbooks/security/config_template_generator.py +63 -76
- runbooks/security/enterprise_security_framework.py +1 -1
- runbooks/security/executive_security_dashboard.py +519 -508
- runbooks/security/multi_account_security_controls.py +959 -1210
- runbooks/security/real_time_security_monitor.py +422 -444
- runbooks/security/security_baseline_tester.py +1 -1
- runbooks/security/security_cli.py +143 -112
- runbooks/security/test_2way_validation.py +439 -0
- runbooks/security/two_way_validation_framework.py +852 -0
- runbooks/sre/production_monitoring_framework.py +167 -177
- runbooks/tdd/__init__.py +15 -0
- runbooks/tdd/cli.py +1071 -0
- runbooks/utils/__init__.py +14 -17
- runbooks/utils/logger.py +7 -2
- runbooks/utils/version_validator.py +50 -47
- runbooks/validation/__init__.py +6 -6
- runbooks/validation/cli.py +9 -3
- runbooks/validation/comprehensive_2way_validator.py +745 -704
- runbooks/validation/mcp_validator.py +906 -228
- runbooks/validation/terraform_citations_validator.py +104 -115
- runbooks/validation/terraform_drift_detector.py +461 -454
- runbooks/vpc/README.md +617 -0
- runbooks/vpc/__init__.py +8 -1
- runbooks/vpc/analyzer.py +577 -0
- runbooks/vpc/cleanup_wrapper.py +476 -413
- runbooks/vpc/cli_cloudtrail_commands.py +339 -0
- runbooks/vpc/cli_mcp_validation_commands.py +480 -0
- runbooks/vpc/cloudtrail_audit_integration.py +717 -0
- runbooks/vpc/config.py +92 -97
- runbooks/vpc/cost_engine.py +411 -148
- runbooks/vpc/cost_explorer_integration.py +553 -0
- runbooks/vpc/cross_account_session.py +101 -106
- runbooks/vpc/enhanced_mcp_validation.py +917 -0
- runbooks/vpc/eni_gate_validator.py +961 -0
- runbooks/vpc/heatmap_engine.py +185 -160
- runbooks/vpc/mcp_no_eni_validator.py +680 -639
- runbooks/vpc/nat_gateway_optimizer.py +358 -0
- runbooks/vpc/networking_wrapper.py +15 -8
- runbooks/vpc/pdca_remediation_planner.py +528 -0
- runbooks/vpc/performance_optimized_analyzer.py +219 -231
- runbooks/vpc/runbooks_adapter.py +1167 -241
- runbooks/vpc/tdd_red_phase_stubs.py +601 -0
- runbooks/vpc/test_data_loader.py +358 -0
- runbooks/vpc/tests/conftest.py +314 -4
- runbooks/vpc/tests/test_cleanup_framework.py +1022 -0
- runbooks/vpc/tests/test_cost_engine.py +0 -2
- runbooks/vpc/topology_generator.py +326 -0
- runbooks/vpc/unified_scenarios.py +1297 -1124
- runbooks/vpc/vpc_cleanup_integration.py +1943 -1115
- runbooks-1.1.6.dist-info/METADATA +327 -0
- runbooks-1.1.6.dist-info/RECORD +489 -0
- runbooks/finops/README.md +0 -414
- runbooks/finops/accuracy_cross_validator.py +0 -647
- runbooks/finops/business_cases.py +0 -950
- runbooks/finops/dashboard_router.py +0 -922
- runbooks/finops/ebs_optimizer.py +0 -973
- runbooks/finops/embedded_mcp_validator.py +0 -1629
- runbooks/finops/enhanced_dashboard_runner.py +0 -527
- runbooks/finops/finops_dashboard.py +0 -584
- runbooks/finops/finops_scenarios.py +0 -1218
- runbooks/finops/legacy_migration.py +0 -730
- runbooks/finops/multi_dashboard.py +0 -1519
- runbooks/finops/single_dashboard.py +0 -1113
- runbooks/finops/unlimited_scenarios.py +0 -393
- runbooks-1.1.4.dist-info/METADATA +0 -800
- runbooks-1.1.4.dist-info/RECORD +0 -468
- {runbooks-1.1.4.dist-info → runbooks-1.1.6.dist-info}/WHEEL +0 -0
- {runbooks-1.1.4.dist-info → runbooks-1.1.6.dist-info}/entry_points.txt +0 -0
- {runbooks-1.1.4.dist-info → runbooks-1.1.6.dist-info}/licenses/LICENSE +0 -0
- {runbooks-1.1.4.dist-info → runbooks-1.1.6.dist-info}/top_level.txt +0 -0
@@ -93,11 +93,11 @@ from time import time
|
|
93
93
|
import Inventory_Modules
|
94
94
|
from ArgumentsClass import CommonArguments
|
95
95
|
from botocore.exceptions import ClientError
|
96
|
-
from
|
96
|
+
from runbooks.common.rich_utils import console
|
97
97
|
from Inventory_Modules import del_config_recorder_or_delivery_channel2, display_results, get_all_credentials
|
98
|
-
|
98
|
+
# Migrated to Rich.Progress - see rich_utils.py for enterprise UX standards
|
99
|
+
# from tqdm.auto import tqdm
|
99
100
|
|
100
|
-
init()
|
101
101
|
__version__ = "2024.05.31"
|
102
102
|
|
103
103
|
|
@@ -409,7 +409,7 @@ def check_accounts_for_delivery_channels_and_config_recorders(CredentialList, fF
|
|
409
409
|
logging.info(
|
410
410
|
f"{ERASE_LINE}Finished finding items in account {c_account_credentials['AccountId']} in region {c_account_credentials['Region']} - {c_PlaceCount} / {c_PlacesToLook}"
|
411
411
|
)
|
412
|
-
pbar.update() # Update progress bar for operational visibility
|
412
|
+
pbar.update(pbar_task, advance=1) # Update Rich progress bar for operational visibility
|
413
413
|
self.queue.task_done() # Mark queue item as completed
|
414
414
|
|
415
415
|
# Initialize processing context and data structures for Config discovery
|
@@ -421,40 +421,50 @@ def check_accounts_for_delivery_channels_and_config_recorders(CredentialList, fF
|
|
421
421
|
|
422
422
|
checkqueue = Queue() # Queue for work distribution across worker threads
|
423
423
|
|
424
|
+
# Import Rich display utilities for professional progress tracking
|
425
|
+
from runbooks.common.rich_utils import create_progress_bar
|
426
|
+
|
424
427
|
# Initialize progress tracking for operational visibility during Config discovery
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
428
|
+
with create_progress_bar() as progress:
|
429
|
+
task = progress.add_task(
|
430
|
+
f"[cyan]Finding config recorders / delivery channels from {len(AllCredentials)} accounts and regions...",
|
431
|
+
total=len(AllCredentials)
|
432
|
+
)
|
430
433
|
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
|
450
|
-
|
451
|
-
)
|
452
|
-
|
453
|
-
|
434
|
+
# Make progress object available to worker threads via global (multi-threaded pattern)
|
435
|
+
global pbar
|
436
|
+
pbar = progress
|
437
|
+
global pbar_task
|
438
|
+
pbar_task = task
|
439
|
+
|
440
|
+
# Create and start worker thread pool for concurrent Config component discovery
|
441
|
+
for x in range(WorkerThreads):
|
442
|
+
worker = Find_Config_Recorders_and_Delivery_Channels(checkqueue)
|
443
|
+
# Daemon threads allow main thread exit even if workers are still processing
|
444
|
+
worker.daemon = True
|
445
|
+
worker.start()
|
446
|
+
|
447
|
+
# Queue Config discovery work items for worker thread processing
|
448
|
+
# Note: Credential list already includes regional context, eliminating need for nested region iteration
|
449
|
+
for credential in CredentialList:
|
450
|
+
logging.info(f"Connecting to account {credential['AccountId']} in region {credential['Region']}")
|
451
|
+
try:
|
452
|
+
# Queue account and region combination for worker thread processing
|
453
|
+
# Note: Tuple structure is critical for proper parameter expansion in worker threads
|
454
|
+
checkqueue.put((credential, fFixRun, fFragments, len(CredentialList), PlaceCount))
|
455
|
+
except ClientError as my_Error:
|
456
|
+
# Handle authorization failures with informative error messaging
|
457
|
+
if "AuthFailure" in str(my_Error):
|
458
|
+
logging.error(
|
459
|
+
f"Authorization Failure accessing account {credential['AccountId']} in {credential['Region']} region"
|
460
|
+
)
|
461
|
+
logging.warning(f"It's possible that the region {credential['Region']} hasn't been opted-into")
|
462
|
+
pass
|
463
|
+
|
464
|
+
# Wait for all worker threads to complete processing
|
465
|
+
checkqueue.join()
|
466
|
+
# Progress bar auto-closes when exiting context manager
|
454
467
|
|
455
|
-
# Wait for all worker threads to complete processing
|
456
|
-
checkqueue.join()
|
457
|
-
pbar.close() # Close progress bar after completion
|
458
468
|
return account_crs_and_dcs
|
459
469
|
|
460
470
|
|
@@ -619,7 +629,7 @@ if __name__ == "__main__":
|
|
619
629
|
print()
|
620
630
|
milestone_time1 = time()
|
621
631
|
print(
|
622
|
-
f"
|
632
|
+
f"[green]\t\tFiguring out what regions are available to your accounts, and capturing credentials for all accounts in those regions took: {(milestone_time1 - begin_time):.3f} seconds"
|
623
633
|
)
|
624
634
|
print()
|
625
635
|
print(f"Now running through all accounts and regions identified to find resources...")
|
@@ -631,7 +641,7 @@ if __name__ == "__main__":
|
|
631
641
|
print()
|
632
642
|
milestone_time2 = time()
|
633
643
|
print(
|
634
|
-
f"
|
644
|
+
f"[green]\t\tChecking {len(AllCredentials)} places took: {(milestone_time2 - milestone_time1):.3f} seconds"
|
635
645
|
)
|
636
646
|
print()
|
637
647
|
cr = 0
|
@@ -650,7 +660,7 @@ if __name__ == "__main__":
|
|
650
660
|
print()
|
651
661
|
milestone_time3 = time()
|
652
662
|
print(
|
653
|
-
f"
|
663
|
+
f"[green]\t\tSorting the list of places took: {(milestone_time3 - milestone_time2):.3f} seconds"
|
654
664
|
)
|
655
665
|
print()
|
656
666
|
display_results(all_sorted_config_recorders_and_delivery_channels, display_dict, None, pFilename)
|
@@ -675,7 +685,7 @@ if __name__ == "__main__":
|
|
675
685
|
|
676
686
|
if pTiming:
|
677
687
|
print(ERASE_LINE)
|
678
|
-
print(f"
|
688
|
+
print(f"[green]This whole script took {time() - begin_time:.3f} seconds")
|
679
689
|
print()
|
680
690
|
print("Thank you for using this tool")
|
681
691
|
print()
|
@@ -60,11 +60,11 @@ from time import time
|
|
60
60
|
|
61
61
|
from ArgumentsClass import CommonArguments
|
62
62
|
from botocore.exceptions import ClientError
|
63
|
-
from
|
63
|
+
from runbooks.common.rich_utils import console
|
64
64
|
from Inventory_Modules import display_results, find_directories2, get_all_credentials
|
65
|
-
|
65
|
+
# Migrated to Rich.Progress - see rich_utils.py for enterprise UX standards
|
66
|
+
# from tqdm.auto import tqdm
|
66
67
|
|
67
|
-
init()
|
68
68
|
__version__ = "2024.05.31"
|
69
69
|
|
70
70
|
|
@@ -193,54 +193,66 @@ def find_all_directories(f_credentials, f_fragments, f_exact):
|
|
193
193
|
"""
|
194
194
|
AllDirectories = [] # Aggregated list for all discovered directories
|
195
195
|
|
196
|
+
# Import Rich display utilities for professional progress tracking
|
197
|
+
from runbooks.common.rich_utils import create_progress_bar
|
198
|
+
|
196
199
|
# TODO: Need to use multi-threading here for improved performance
|
197
200
|
# Sequential processing with progress tracking for operational visibility
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
logging.
|
201
|
+
with create_progress_bar() as progress:
|
202
|
+
task = progress.add_task(
|
203
|
+
f"[cyan]Looking through {len(f_credentials)} accounts and regions...",
|
204
|
+
total=len(f_credentials)
|
205
|
+
)
|
206
|
+
|
207
|
+
for credential in f_credentials:
|
208
|
+
logging.info(f"Looking in account: {credential['AccountId']} in region {credential['Region']}")
|
209
|
+
|
210
|
+
# Skip failed credentials to avoid API errors
|
211
|
+
if not credential["Success"]:
|
212
|
+
progress.update(task, advance=1)
|
213
|
+
continue
|
214
|
+
|
215
|
+
try:
|
216
|
+
# Discover directories using Directory Service API with fragment filtering
|
217
|
+
directories = find_directories2(credential, credential["Region"], f_fragments, f_exact)
|
218
|
+
logging.info(f"directories: {directories}")
|
219
|
+
logging.info(
|
220
|
+
f"Account: {credential['AccountId']} Region: {credential['Region']} Found {len(directories)} directories"
|
221
|
+
)
|
222
|
+
|
223
|
+
# Process and aggregate discovered directories with organizational context
|
224
|
+
if directories:
|
225
|
+
for directory in directories:
|
226
|
+
# Enhance directory metadata with organizational and regional context
|
227
|
+
# Available directory metadata includes:
|
228
|
+
# - DirectoryName: Human-readable directory identifier
|
229
|
+
# - DirectoryId: Unique AWS Directory Service identifier
|
230
|
+
# - HomeRegion: Primary directory service region
|
231
|
+
# - Status: Operational status (Active, Creating, etc.)
|
232
|
+
# - Type: Directory type (SimpleAD, MicrosoftAD, etc.)
|
233
|
+
# - Owner: Directory ownership context
|
234
|
+
|
235
|
+
directory.update(
|
236
|
+
{
|
237
|
+
"MgmtAccount": credential["MgmtAccount"], # Management account context
|
238
|
+
"Region": credential["Region"], # Regional deployment information
|
239
|
+
"AccountId": credential["AccountId"], # Account ownership details
|
240
|
+
}
|
241
|
+
)
|
242
|
+
AllDirectories.append(directory)
|
243
|
+
|
244
|
+
except TypeError as my_Error:
|
245
|
+
# Handle type errors from malformed Directory Service API responses
|
246
|
+
logging.info(f"Error: {my_Error}")
|
247
|
+
progress.update(task, advance=1)
|
248
|
+
continue
|
249
|
+
except ClientError as my_Error:
|
250
|
+
# Handle AWS API authorization failures with informative logging
|
251
|
+
if "AuthFailure" in str(my_Error):
|
252
|
+
logging.error(f" Account {credential['AccountId']} : Authorization Failure")
|
253
|
+
|
254
|
+
# Update progress after processing each credential
|
255
|
+
progress.update(task, advance=1)
|
244
256
|
|
245
257
|
return AllDirectories
|
246
258
|
|
@@ -293,7 +305,7 @@ if __name__ == "__main__":
|
|
293
305
|
# Display credential retrieval timing for performance optimization
|
294
306
|
if pTiming:
|
295
307
|
print(
|
296
|
-
f"
|
308
|
+
f"[green]\tAfter getting credentials, this script took {time() - begin_time:.3f} seconds"
|
297
309
|
)
|
298
310
|
print()
|
299
311
|
|
@@ -304,7 +316,7 @@ if __name__ == "__main__":
|
|
304
316
|
# Display credential parsing timing for operational metrics
|
305
317
|
if pTiming:
|
306
318
|
print(
|
307
|
-
f"
|
319
|
+
f"[green]\tAfter parsing out all Regions, Account and Profiles, this script took {time() - begin_time:.3f} seconds"
|
308
320
|
)
|
309
321
|
print()
|
310
322
|
|
@@ -340,7 +352,7 @@ if __name__ == "__main__":
|
|
340
352
|
display_results(sorted_Results, display_dict, "None")
|
341
353
|
|
342
354
|
# Provide operational summary with discovery metrics and performance timing
|
343
|
-
print(
|
355
|
+
console.print()
|
344
356
|
print(
|
345
357
|
f"Found {len(all_directories)} directories across {len(AccountList)} accounts across {len(RegionList)} regions"
|
346
358
|
)
|
@@ -348,7 +360,7 @@ if __name__ == "__main__":
|
|
348
360
|
|
349
361
|
# Display total execution timing for performance analysis and optimization
|
350
362
|
if pTiming:
|
351
|
-
print(f"
|
363
|
+
print(f"[green]\tThis script took {time() - begin_time:.3f} seconds")
|
352
364
|
print()
|
353
365
|
print("Thank you for using this script")
|
354
366
|
print()
|
@@ -67,12 +67,10 @@ import sys
|
|
67
67
|
from time import time
|
68
68
|
|
69
69
|
from ArgumentsClass import CommonArguments
|
70
|
-
from
|
70
|
+
from runbooks.common.rich_utils import console
|
71
71
|
from Inventory_Modules import display_results, get_all_credentials, get_region_azs2
|
72
72
|
|
73
|
-
init()
|
74
73
|
__version__ = "2024.03.06"
|
75
|
-
ERASE_LINE = "\x1b[2K"
|
76
74
|
begin_time = time()
|
77
75
|
|
78
76
|
|
@@ -281,6 +279,6 @@ if __name__ == "__main__":
|
|
281
279
|
|
282
280
|
print()
|
283
281
|
if pTiming:
|
284
|
-
print(f"
|
282
|
+
print(f"[green]This script took {time() - begin_time:.2f} seconds")
|
285
283
|
print("Thanks for using this script")
|
286
284
|
print()
|
@@ -7,15 +7,13 @@ from time import time
|
|
7
7
|
|
8
8
|
from ArgumentsClass import CommonArguments
|
9
9
|
from botocore.exceptions import ClientError
|
10
|
-
from
|
10
|
+
from runbooks.common.rich_utils import console
|
11
11
|
from Inventory_Modules import display_results, find_account_volumes2, get_all_credentials
|
12
|
-
from
|
12
|
+
from runbooks.common.rich_utils import create_progress_bar
|
13
13
|
|
14
|
-
init()
|
15
14
|
__version__ = "2024.05.31"
|
16
15
|
|
17
16
|
# ANSI escape code for clearing current line (progress bar cleanup)
|
18
|
-
ERASE_LINE = "\x1b[2K"
|
19
17
|
|
20
18
|
|
21
19
|
##################
|
@@ -92,7 +90,7 @@ def present_results(fVolumesFound: list):
|
|
92
90
|
# Calculate and display orphaned volumes
|
93
91
|
orphaned_volumes = [vol for vol in de_dupe_VolumesFound if vol.get("State") == "available"]
|
94
92
|
if orphaned_volumes:
|
95
|
-
print(f"
|
93
|
+
print(f"[yellow]Warning: {len(orphaned_volumes)} orphaned (unattached) volumes found")
|
96
94
|
total_orphaned_size = sum(vol.get("Size", 0) for vol in orphaned_volumes)
|
97
95
|
print(f"Total orphaned storage: {total_orphaned_size} GB")
|
98
96
|
|
@@ -136,9 +134,9 @@ def check_accounts_for_ebs_volumes(f_CredentialList, f_fragment_list=None):
|
|
136
134
|
continue
|
137
135
|
finally:
|
138
136
|
logging.info(
|
139
|
-
f"
|
137
|
+
f"Finished finding EBS volumes in account {c_account_credentials['AccountId']} in region {c_account_credentials['Region']}"
|
140
138
|
)
|
141
|
-
|
139
|
+
progress.update(task, advance=1)
|
142
140
|
self.queue.task_done()
|
143
141
|
|
144
142
|
if f_fragment_list is None:
|
@@ -148,32 +146,31 @@ def check_accounts_for_ebs_volumes(f_CredentialList, f_fragment_list=None):
|
|
148
146
|
|
149
147
|
checkqueue = Queue()
|
150
148
|
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
pbar.close()
|
149
|
+
with create_progress_bar() as progress:
|
150
|
+
task = progress.add_task(
|
151
|
+
f"Finding ebs volumes from {len(f_CredentialList)} accounts and regions",
|
152
|
+
total=len(f_CredentialList)
|
153
|
+
)
|
154
|
+
|
155
|
+
for x in range(WorkerThreads):
|
156
|
+
worker = FindVolumes(checkqueue)
|
157
|
+
# Setting daemon to True will let the main thread exit even though the workers are blocking
|
158
|
+
worker.daemon = True
|
159
|
+
worker.start()
|
160
|
+
|
161
|
+
for credential in f_CredentialList:
|
162
|
+
logging.info(f"Connecting to account {credential['AccountId']}")
|
163
|
+
try:
|
164
|
+
# print(f"Queuing account {credential['AccountId']} in region {region}", end='\r')
|
165
|
+
checkqueue.put((credential, credential["Region"], f_fragment_list))
|
166
|
+
except ClientError as my_Error:
|
167
|
+
if "AuthFailure" in str(my_Error):
|
168
|
+
logging.error(
|
169
|
+
f"Authorization Failure accessing account {credential['AccountId']} in '{credential['Region']}' region"
|
170
|
+
)
|
171
|
+
logging.warning(f"It's possible that the region '{credential['Region']}' hasn't been opted-into")
|
172
|
+
pass
|
173
|
+
checkqueue.join()
|
177
174
|
return AllVolumes
|
178
175
|
|
179
176
|
|
@@ -231,8 +228,8 @@ def main():
|
|
231
228
|
|
232
229
|
# Display execution timing if requested
|
233
230
|
if pTiming:
|
234
|
-
print(
|
235
|
-
print(f"
|
231
|
+
console.print()
|
232
|
+
print(f"[green]This script completed in {time() - begin_time:.2f} seconds")
|
236
233
|
|
237
234
|
|
238
235
|
if __name__ == "__main__":
|
@@ -62,13 +62,10 @@ from time import time
|
|
62
62
|
from . import inventory_modules as Inventory_Modules
|
63
63
|
from .ArgumentsClass import CommonArguments
|
64
64
|
from botocore.exceptions import ClientError
|
65
|
-
from colorama import Fore, init
|
66
65
|
from .inventory_modules import display_results, get_all_credentials
|
67
|
-
from
|
66
|
+
from runbooks.common.rich_utils import create_progress_bar
|
68
67
|
|
69
|
-
init()
|
70
68
|
__version__ = "2025.04.09"
|
71
|
-
ERASE_LINE = "\x1b[2K"
|
72
69
|
begin_time = time()
|
73
70
|
|
74
71
|
|
@@ -322,30 +319,28 @@ def find_all_instances(fAllCredentials: list, fStatus: str = None) -> list:
|
|
322
319
|
AllInstances = []
|
323
320
|
WorkerThreads = min(len(fAllCredentials), 25)
|
324
321
|
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
checkqueue.join()
|
348
|
-
pbar.close()
|
322
|
+
with create_progress_bar() as progress:
|
323
|
+
task = progress.add_task(f"Finding instances from {len(fAllCredentials)} locations", total=len(fAllCredentials))
|
324
|
+
|
325
|
+
for x in range(WorkerThreads):
|
326
|
+
worker = FindInstances(checkqueue)
|
327
|
+
# Setting daemon to True will let the main thread exit even though the workers are blocking
|
328
|
+
worker.daemon = True
|
329
|
+
worker.start()
|
330
|
+
|
331
|
+
for credential in fAllCredentials:
|
332
|
+
logging.info(f"Beginning to queue data - starting with {credential['AccountId']}")
|
333
|
+
try:
|
334
|
+
# I don't know why - but double parens are necessary below. If you remove them, only the first parameter is queued.
|
335
|
+
checkqueue.put((credential))
|
336
|
+
except ClientError as my_Error:
|
337
|
+
if "AuthFailure" in str(my_Error):
|
338
|
+
logging.error(
|
339
|
+
f"Authorization Failure accessing account {credential['AccountId']} in {credential['Region']} region"
|
340
|
+
)
|
341
|
+
logging.warning(f"It's possible that the region {credential['Region']} hasn't been opted-into")
|
342
|
+
pass
|
343
|
+
checkqueue.join()
|
349
344
|
return AllInstances
|
350
345
|
|
351
346
|
|
@@ -90,13 +90,11 @@ from time import time
|
|
90
90
|
import Inventory_Modules
|
91
91
|
from ArgumentsClass import CommonArguments
|
92
92
|
from botocore.exceptions import ClientError
|
93
|
-
from
|
93
|
+
from runbooks.common.rich_utils import console
|
94
94
|
from Inventory_Modules import display_results, find_account_ecs_clusters_services_and_tasks2, get_all_credentials
|
95
|
-
from
|
95
|
+
from runbooks.common.rich_utils import create_progress_bar
|
96
96
|
|
97
|
-
init()
|
98
97
|
__version__ = "2024.09.06"
|
99
|
-
ERASE_LINE = "\x1b[2K"
|
100
98
|
begin_time = time()
|
101
99
|
|
102
100
|
# TODO: Need a table at the bottom that summarizes the results, by instance-type, by running/ stopped, maybe by account and region
|
@@ -246,7 +244,7 @@ def find_all_clusters_and_tasks(fAllCredentials: list, fStatus: str = None) -> l
|
|
246
244
|
"""
|
247
245
|
while True:
|
248
246
|
# Retrieve ECS discovery work item from thread-safe queue
|
249
|
-
c_account_credentials = self.queue.get()
|
247
|
+
c_account_credentials, c_progress, c_task = self.queue.get()
|
250
248
|
logging.info(f"De-queued info for account number {c_account_credentials['AccountId']}")
|
251
249
|
|
252
250
|
try:
|
@@ -406,7 +404,7 @@ def find_all_clusters_and_tasks(fAllCredentials: list, fStatus: str = None) -> l
|
|
406
404
|
continue
|
407
405
|
finally:
|
408
406
|
# Ensure progress tracking and queue management regardless of success/failure
|
409
|
-
|
407
|
+
c_progress.update(c_task, advance=1)
|
410
408
|
self.queue.task_done()
|
411
409
|
|
412
410
|
###########
|
@@ -423,13 +421,6 @@ def find_all_clusters_and_tasks(fAllCredentials: list, fStatus: str = None) -> l
|
|
423
421
|
# Maximum 25 threads to prevent overwhelming AWS APIs while maintaining efficiency
|
424
422
|
WorkerThreads = min(len(fAllCredentials), 25)
|
425
423
|
|
426
|
-
# Initialize progress tracking for operational visibility during large-scale operations
|
427
|
-
pbar = tqdm(
|
428
|
-
desc=f"Finding ECS clusters, services and tasks from {len(fAllCredentials)} accounts / regions",
|
429
|
-
total=len(fAllCredentials),
|
430
|
-
unit=" locations",
|
431
|
-
)
|
432
|
-
|
433
424
|
# Start worker threads for concurrent ECS resource discovery
|
434
425
|
for x in range(WorkerThreads):
|
435
426
|
worker = FindInstances(checkqueue)
|
@@ -437,25 +428,26 @@ def find_all_clusters_and_tasks(fAllCredentials: list, fStatus: str = None) -> l
|
|
437
428
|
worker.daemon = True
|
438
429
|
worker.start()
|
439
430
|
|
440
|
-
# Queue credential sets for processing by worker threads
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
|
450
|
-
|
451
|
-
|
452
|
-
|
453
|
-
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
458
|
-
|
431
|
+
# Queue credential sets for processing by worker threads with progress tracking
|
432
|
+
with create_progress_bar() as progress:
|
433
|
+
task = progress.add_task("Discovering ECS clusters, services and tasks", total=len(fAllCredentials))
|
434
|
+
for credential in fAllCredentials:
|
435
|
+
logging.info(f"Beginning to queue data - starting with {credential['AccountId']}")
|
436
|
+
try:
|
437
|
+
# Queue individual credential set for ECS resource discovery with progress tracking
|
438
|
+
# Tuple format: (credential, progress, task)
|
439
|
+
checkqueue.put((credential, progress, task))
|
440
|
+
except ClientError as my_Error:
|
441
|
+
# Handle authorization failures during credential queuing
|
442
|
+
if "AuthFailure" in str(my_Error):
|
443
|
+
logging.error(
|
444
|
+
f"Authorization Failure accessing account {credential['AccountId']} in {credential['Region']} region"
|
445
|
+
)
|
446
|
+
logging.warning(f"It's possible that the region {credential['Region']} hasn't been opted-into")
|
447
|
+
pass
|
448
|
+
|
449
|
+
# Wait for all queued work to complete before proceeding
|
450
|
+
checkqueue.join()
|
459
451
|
return AllInstances
|
460
452
|
|
461
453
|
|
@@ -514,7 +506,7 @@ if __name__ == "__main__":
|
|
514
506
|
print()
|
515
507
|
milestone_time1 = time()
|
516
508
|
print(
|
517
|
-
f"
|
509
|
+
f"[green]\t\tCredential discovery and region enumeration took: {(milestone_time1 - begin_time):.3f} seconds"
|
518
510
|
)
|
519
511
|
print()
|
520
512
|
|
@@ -549,7 +541,7 @@ if __name__ == "__main__":
|
|
549
541
|
# Display performance timing metrics for operational optimization and SLA compliance
|
550
542
|
if pTiming:
|
551
543
|
print(ERASE_LINE)
|
552
|
-
print(f"
|
544
|
+
print(f"[green]This script took {time() - begin_time:.2f} seconds")
|
553
545
|
|
554
546
|
print(ERASE_LINE)
|
555
547
|
|