aws-inventory-manager 0.17.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aws_inventory_manager-0.17.12.dist-info/LICENSE +21 -0
- aws_inventory_manager-0.17.12.dist-info/METADATA +1292 -0
- aws_inventory_manager-0.17.12.dist-info/RECORD +152 -0
- aws_inventory_manager-0.17.12.dist-info/WHEEL +5 -0
- aws_inventory_manager-0.17.12.dist-info/entry_points.txt +2 -0
- aws_inventory_manager-0.17.12.dist-info/top_level.txt +1 -0
- src/__init__.py +3 -0
- src/aws/__init__.py +11 -0
- src/aws/client.py +128 -0
- src/aws/credentials.py +191 -0
- src/aws/rate_limiter.py +177 -0
- src/cli/__init__.py +12 -0
- src/cli/config.py +130 -0
- src/cli/main.py +4046 -0
- src/cloudtrail/__init__.py +5 -0
- src/cloudtrail/query.py +642 -0
- src/config_service/__init__.py +21 -0
- src/config_service/collector.py +346 -0
- src/config_service/detector.py +256 -0
- src/config_service/resource_type_mapping.py +328 -0
- src/cost/__init__.py +5 -0
- src/cost/analyzer.py +226 -0
- src/cost/explorer.py +209 -0
- src/cost/reporter.py +237 -0
- src/delta/__init__.py +5 -0
- src/delta/calculator.py +206 -0
- src/delta/differ.py +185 -0
- src/delta/formatters.py +272 -0
- src/delta/models.py +154 -0
- src/delta/reporter.py +234 -0
- src/matching/__init__.py +6 -0
- src/matching/config.py +52 -0
- src/matching/normalizer.py +450 -0
- src/matching/prompts.py +33 -0
- src/models/__init__.py +21 -0
- src/models/config_diff.py +135 -0
- src/models/cost_report.py +87 -0
- src/models/deletion_operation.py +104 -0
- src/models/deletion_record.py +97 -0
- src/models/delta_report.py +122 -0
- src/models/efs_resource.py +80 -0
- src/models/elasticache_resource.py +90 -0
- src/models/group.py +318 -0
- src/models/inventory.py +133 -0
- src/models/protection_rule.py +123 -0
- src/models/report.py +288 -0
- src/models/resource.py +111 -0
- src/models/security_finding.py +102 -0
- src/models/snapshot.py +122 -0
- src/restore/__init__.py +20 -0
- src/restore/audit.py +175 -0
- src/restore/cleaner.py +461 -0
- src/restore/config.py +209 -0
- src/restore/deleter.py +976 -0
- src/restore/dependency.py +254 -0
- src/restore/safety.py +115 -0
- src/security/__init__.py +0 -0
- src/security/checks/__init__.py +0 -0
- src/security/checks/base.py +56 -0
- src/security/checks/ec2_checks.py +88 -0
- src/security/checks/elasticache_checks.py +149 -0
- src/security/checks/iam_checks.py +102 -0
- src/security/checks/rds_checks.py +140 -0
- src/security/checks/s3_checks.py +95 -0
- src/security/checks/secrets_checks.py +96 -0
- src/security/checks/sg_checks.py +142 -0
- src/security/cis_mapper.py +97 -0
- src/security/models.py +53 -0
- src/security/reporter.py +174 -0
- src/security/scanner.py +87 -0
- src/snapshot/__init__.py +6 -0
- src/snapshot/capturer.py +453 -0
- src/snapshot/filter.py +259 -0
- src/snapshot/inventory_storage.py +236 -0
- src/snapshot/report_formatter.py +250 -0
- src/snapshot/reporter.py +189 -0
- src/snapshot/resource_collectors/__init__.py +5 -0
- src/snapshot/resource_collectors/apigateway.py +140 -0
- src/snapshot/resource_collectors/backup.py +136 -0
- src/snapshot/resource_collectors/base.py +81 -0
- src/snapshot/resource_collectors/cloudformation.py +55 -0
- src/snapshot/resource_collectors/cloudwatch.py +109 -0
- src/snapshot/resource_collectors/codebuild.py +69 -0
- src/snapshot/resource_collectors/codepipeline.py +82 -0
- src/snapshot/resource_collectors/dynamodb.py +65 -0
- src/snapshot/resource_collectors/ec2.py +240 -0
- src/snapshot/resource_collectors/ecs.py +215 -0
- src/snapshot/resource_collectors/efs_collector.py +102 -0
- src/snapshot/resource_collectors/eks.py +200 -0
- src/snapshot/resource_collectors/elasticache_collector.py +79 -0
- src/snapshot/resource_collectors/elb.py +126 -0
- src/snapshot/resource_collectors/eventbridge.py +156 -0
- src/snapshot/resource_collectors/glue.py +199 -0
- src/snapshot/resource_collectors/iam.py +188 -0
- src/snapshot/resource_collectors/kms.py +111 -0
- src/snapshot/resource_collectors/lambda_func.py +139 -0
- src/snapshot/resource_collectors/rds.py +109 -0
- src/snapshot/resource_collectors/route53.py +86 -0
- src/snapshot/resource_collectors/s3.py +105 -0
- src/snapshot/resource_collectors/secretsmanager.py +70 -0
- src/snapshot/resource_collectors/sns.py +68 -0
- src/snapshot/resource_collectors/sqs.py +82 -0
- src/snapshot/resource_collectors/ssm.py +160 -0
- src/snapshot/resource_collectors/stepfunctions.py +74 -0
- src/snapshot/resource_collectors/vpcendpoints.py +79 -0
- src/snapshot/resource_collectors/waf.py +159 -0
- src/snapshot/storage.py +351 -0
- src/storage/__init__.py +21 -0
- src/storage/audit_store.py +419 -0
- src/storage/database.py +294 -0
- src/storage/group_store.py +763 -0
- src/storage/inventory_store.py +320 -0
- src/storage/resource_store.py +416 -0
- src/storage/schema.py +339 -0
- src/storage/snapshot_store.py +363 -0
- src/utils/__init__.py +12 -0
- src/utils/export.py +305 -0
- src/utils/hash.py +60 -0
- src/utils/logging.py +63 -0
- src/utils/pagination.py +41 -0
- src/utils/paths.py +51 -0
- src/utils/progress.py +41 -0
- src/utils/unsupported_resources.py +306 -0
- src/web/__init__.py +5 -0
- src/web/app.py +97 -0
- src/web/dependencies.py +69 -0
- src/web/routes/__init__.py +1 -0
- src/web/routes/api/__init__.py +18 -0
- src/web/routes/api/charts.py +156 -0
- src/web/routes/api/cleanup.py +186 -0
- src/web/routes/api/filters.py +253 -0
- src/web/routes/api/groups.py +305 -0
- src/web/routes/api/inventories.py +80 -0
- src/web/routes/api/queries.py +202 -0
- src/web/routes/api/resources.py +393 -0
- src/web/routes/api/snapshots.py +314 -0
- src/web/routes/api/views.py +260 -0
- src/web/routes/pages.py +198 -0
- src/web/services/__init__.py +1 -0
- src/web/templates/base.html +955 -0
- src/web/templates/components/navbar.html +31 -0
- src/web/templates/components/sidebar.html +104 -0
- src/web/templates/pages/audit_logs.html +86 -0
- src/web/templates/pages/cleanup.html +279 -0
- src/web/templates/pages/dashboard.html +227 -0
- src/web/templates/pages/diff.html +175 -0
- src/web/templates/pages/error.html +30 -0
- src/web/templates/pages/groups.html +721 -0
- src/web/templates/pages/queries.html +246 -0
- src/web/templates/pages/resources.html +2429 -0
- src/web/templates/pages/snapshot_detail.html +271 -0
- src/web/templates/pages/snapshots.html +429 -0
src/storage/schema.py
ADDED
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
"""SQLite schema definitions for AWS Inventory Manager."""
|
|
2
|
+
|
|
3
|
+
SCHEMA_VERSION = "1.2.0"
|
|
4
|
+
|
|
5
|
+
# Schema creation SQL
|
|
6
|
+
SCHEMA_SQL = """
|
|
7
|
+
-- Schema version tracking
|
|
8
|
+
CREATE TABLE IF NOT EXISTS schema_info (
|
|
9
|
+
key TEXT PRIMARY KEY,
|
|
10
|
+
value TEXT NOT NULL
|
|
11
|
+
);
|
|
12
|
+
|
|
13
|
+
-- Core snapshots table
|
|
14
|
+
CREATE TABLE IF NOT EXISTS snapshots (
|
|
15
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
16
|
+
name TEXT UNIQUE NOT NULL,
|
|
17
|
+
created_at TIMESTAMP NOT NULL,
|
|
18
|
+
account_id TEXT NOT NULL,
|
|
19
|
+
regions TEXT NOT NULL,
|
|
20
|
+
resource_count INTEGER DEFAULT 0,
|
|
21
|
+
total_resources_before_filter INTEGER,
|
|
22
|
+
service_counts TEXT,
|
|
23
|
+
metadata TEXT,
|
|
24
|
+
filters_applied TEXT,
|
|
25
|
+
schema_version TEXT DEFAULT '1.1',
|
|
26
|
+
inventory_name TEXT DEFAULT 'default',
|
|
27
|
+
is_active BOOLEAN DEFAULT 0
|
|
28
|
+
);
|
|
29
|
+
|
|
30
|
+
-- Resources table
|
|
31
|
+
CREATE TABLE IF NOT EXISTS resources (
|
|
32
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
33
|
+
snapshot_id INTEGER NOT NULL,
|
|
34
|
+
arn TEXT NOT NULL,
|
|
35
|
+
resource_type TEXT NOT NULL,
|
|
36
|
+
name TEXT NOT NULL,
|
|
37
|
+
region TEXT NOT NULL,
|
|
38
|
+
config_hash TEXT NOT NULL,
|
|
39
|
+
raw_config TEXT,
|
|
40
|
+
created_at TIMESTAMP,
|
|
41
|
+
source TEXT DEFAULT 'direct_api',
|
|
42
|
+
canonical_name TEXT,
|
|
43
|
+
normalized_name TEXT,
|
|
44
|
+
extracted_patterns TEXT,
|
|
45
|
+
normalization_method TEXT,
|
|
46
|
+
FOREIGN KEY (snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE,
|
|
47
|
+
UNIQUE(snapshot_id, arn)
|
|
48
|
+
);
|
|
49
|
+
|
|
50
|
+
-- Normalized tags for efficient querying
|
|
51
|
+
CREATE TABLE IF NOT EXISTS resource_tags (
|
|
52
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
53
|
+
resource_id INTEGER NOT NULL,
|
|
54
|
+
key TEXT NOT NULL,
|
|
55
|
+
value TEXT NOT NULL,
|
|
56
|
+
FOREIGN KEY (resource_id) REFERENCES resources(id) ON DELETE CASCADE
|
|
57
|
+
);
|
|
58
|
+
|
|
59
|
+
-- Inventories table
|
|
60
|
+
CREATE TABLE IF NOT EXISTS inventories (
|
|
61
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
62
|
+
name TEXT NOT NULL,
|
|
63
|
+
account_id TEXT NOT NULL,
|
|
64
|
+
description TEXT DEFAULT '',
|
|
65
|
+
include_tags TEXT,
|
|
66
|
+
exclude_tags TEXT,
|
|
67
|
+
active_snapshot_id INTEGER,
|
|
68
|
+
created_at TIMESTAMP NOT NULL,
|
|
69
|
+
last_updated TIMESTAMP NOT NULL,
|
|
70
|
+
FOREIGN KEY (active_snapshot_id) REFERENCES snapshots(id) ON DELETE SET NULL,
|
|
71
|
+
UNIQUE(name, account_id)
|
|
72
|
+
);
|
|
73
|
+
|
|
74
|
+
-- Link table for inventory snapshots (many-to-many)
|
|
75
|
+
CREATE TABLE IF NOT EXISTS inventory_snapshots (
|
|
76
|
+
inventory_id INTEGER NOT NULL,
|
|
77
|
+
snapshot_id INTEGER NOT NULL,
|
|
78
|
+
PRIMARY KEY (inventory_id, snapshot_id),
|
|
79
|
+
FOREIGN KEY (inventory_id) REFERENCES inventories(id) ON DELETE CASCADE,
|
|
80
|
+
FOREIGN KEY (snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE
|
|
81
|
+
);
|
|
82
|
+
|
|
83
|
+
-- Audit operations table
|
|
84
|
+
CREATE TABLE IF NOT EXISTS audit_operations (
|
|
85
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
86
|
+
operation_id TEXT UNIQUE NOT NULL,
|
|
87
|
+
baseline_snapshot TEXT NOT NULL,
|
|
88
|
+
timestamp TIMESTAMP NOT NULL,
|
|
89
|
+
aws_profile TEXT,
|
|
90
|
+
account_id TEXT NOT NULL,
|
|
91
|
+
mode TEXT NOT NULL,
|
|
92
|
+
status TEXT NOT NULL,
|
|
93
|
+
total_resources INTEGER,
|
|
94
|
+
succeeded_count INTEGER,
|
|
95
|
+
failed_count INTEGER,
|
|
96
|
+
skipped_count INTEGER,
|
|
97
|
+
duration_seconds REAL,
|
|
98
|
+
filters TEXT
|
|
99
|
+
);
|
|
100
|
+
|
|
101
|
+
-- Audit records table
|
|
102
|
+
CREATE TABLE IF NOT EXISTS audit_records (
|
|
103
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
104
|
+
operation_id TEXT NOT NULL,
|
|
105
|
+
resource_arn TEXT NOT NULL,
|
|
106
|
+
resource_id TEXT,
|
|
107
|
+
resource_type TEXT NOT NULL,
|
|
108
|
+
region TEXT NOT NULL,
|
|
109
|
+
status TEXT NOT NULL,
|
|
110
|
+
error_code TEXT,
|
|
111
|
+
error_message TEXT,
|
|
112
|
+
protection_reason TEXT,
|
|
113
|
+
deletion_tier TEXT,
|
|
114
|
+
tags TEXT,
|
|
115
|
+
estimated_monthly_cost REAL,
|
|
116
|
+
FOREIGN KEY (operation_id) REFERENCES audit_operations(operation_id) ON DELETE CASCADE
|
|
117
|
+
);
|
|
118
|
+
|
|
119
|
+
-- Saved queries table (for web UI)
|
|
120
|
+
CREATE TABLE IF NOT EXISTS saved_queries (
|
|
121
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
122
|
+
name TEXT UNIQUE NOT NULL,
|
|
123
|
+
description TEXT,
|
|
124
|
+
sql_text TEXT NOT NULL,
|
|
125
|
+
category TEXT DEFAULT 'custom',
|
|
126
|
+
is_favorite BOOLEAN DEFAULT 0,
|
|
127
|
+
created_at TIMESTAMP NOT NULL,
|
|
128
|
+
last_run_at TIMESTAMP,
|
|
129
|
+
run_count INTEGER DEFAULT 0
|
|
130
|
+
);
|
|
131
|
+
|
|
132
|
+
-- Saved filters table (for resource explorer)
|
|
133
|
+
CREATE TABLE IF NOT EXISTS saved_filters (
|
|
134
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
135
|
+
name TEXT UNIQUE NOT NULL,
|
|
136
|
+
description TEXT,
|
|
137
|
+
filter_config TEXT NOT NULL,
|
|
138
|
+
is_favorite BOOLEAN DEFAULT 0,
|
|
139
|
+
created_at TIMESTAMP NOT NULL,
|
|
140
|
+
last_used_at TIMESTAMP,
|
|
141
|
+
use_count INTEGER DEFAULT 0
|
|
142
|
+
);
|
|
143
|
+
|
|
144
|
+
-- Saved views table (for customizable resource views)
|
|
145
|
+
CREATE TABLE IF NOT EXISTS saved_views (
|
|
146
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
147
|
+
name TEXT UNIQUE NOT NULL,
|
|
148
|
+
description TEXT,
|
|
149
|
+
view_config TEXT NOT NULL,
|
|
150
|
+
is_default BOOLEAN DEFAULT 0,
|
|
151
|
+
is_favorite BOOLEAN DEFAULT 0,
|
|
152
|
+
created_at TIMESTAMP NOT NULL,
|
|
153
|
+
last_used_at TIMESTAMP,
|
|
154
|
+
use_count INTEGER DEFAULT 0
|
|
155
|
+
);
|
|
156
|
+
|
|
157
|
+
-- Resource groups table (for baseline comparison)
|
|
158
|
+
CREATE TABLE IF NOT EXISTS resource_groups (
|
|
159
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
160
|
+
name TEXT UNIQUE NOT NULL,
|
|
161
|
+
description TEXT,
|
|
162
|
+
source_snapshot TEXT,
|
|
163
|
+
resource_count INTEGER DEFAULT 0,
|
|
164
|
+
is_favorite BOOLEAN DEFAULT 0,
|
|
165
|
+
created_at TIMESTAMP NOT NULL,
|
|
166
|
+
last_updated TIMESTAMP NOT NULL
|
|
167
|
+
);
|
|
168
|
+
|
|
169
|
+
-- Resource group members table (normalized for efficient querying)
|
|
170
|
+
CREATE TABLE IF NOT EXISTS resource_group_members (
|
|
171
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
172
|
+
group_id INTEGER NOT NULL,
|
|
173
|
+
resource_name TEXT NOT NULL,
|
|
174
|
+
resource_type TEXT NOT NULL,
|
|
175
|
+
original_arn TEXT,
|
|
176
|
+
match_strategy TEXT DEFAULT 'physical_name',
|
|
177
|
+
FOREIGN KEY (group_id) REFERENCES resource_groups(id) ON DELETE CASCADE,
|
|
178
|
+
UNIQUE (group_id, resource_name, resource_type)
|
|
179
|
+
);
|
|
180
|
+
"""
|
|
181
|
+
|
|
182
|
+
# Indexes for common queries (created separately for better error handling)
|
|
183
|
+
# SQLite performance tips applied:
|
|
184
|
+
# - Indexes on foreign keys for faster JOINs
|
|
185
|
+
# - Composite indexes for common query patterns
|
|
186
|
+
# - Covering indexes where possible
|
|
187
|
+
INDEXES_SQL = """
|
|
188
|
+
-- Resources indexes
|
|
189
|
+
CREATE INDEX IF NOT EXISTS idx_resources_arn ON resources(arn);
|
|
190
|
+
CREATE INDEX IF NOT EXISTS idx_resources_type ON resources(resource_type);
|
|
191
|
+
CREATE INDEX IF NOT EXISTS idx_resources_region ON resources(region);
|
|
192
|
+
CREATE INDEX IF NOT EXISTS idx_resources_created ON resources(created_at);
|
|
193
|
+
CREATE INDEX IF NOT EXISTS idx_resources_snapshot ON resources(snapshot_id);
|
|
194
|
+
CREATE INDEX IF NOT EXISTS idx_resources_type_region ON resources(resource_type, region);
|
|
195
|
+
CREATE INDEX IF NOT EXISTS idx_resources_canonical_name_type ON resources(canonical_name, resource_type);
|
|
196
|
+
CREATE INDEX IF NOT EXISTS idx_resources_normalized_name_type ON resources(normalized_name, resource_type);
|
|
197
|
+
|
|
198
|
+
-- Tags indexes (for efficient tag queries)
|
|
199
|
+
CREATE INDEX IF NOT EXISTS idx_tags_resource ON resource_tags(resource_id);
|
|
200
|
+
CREATE INDEX IF NOT EXISTS idx_tags_key ON resource_tags(key);
|
|
201
|
+
CREATE INDEX IF NOT EXISTS idx_tags_value ON resource_tags(value);
|
|
202
|
+
CREATE INDEX IF NOT EXISTS idx_tags_kv ON resource_tags(key, value);
|
|
203
|
+
|
|
204
|
+
-- Snapshots indexes
|
|
205
|
+
CREATE INDEX IF NOT EXISTS idx_snapshots_account ON snapshots(account_id);
|
|
206
|
+
CREATE INDEX IF NOT EXISTS idx_snapshots_created ON snapshots(created_at);
|
|
207
|
+
CREATE INDEX IF NOT EXISTS idx_snapshots_name ON snapshots(name);
|
|
208
|
+
CREATE INDEX IF NOT EXISTS idx_snapshots_account_created ON snapshots(account_id, created_at DESC);
|
|
209
|
+
|
|
210
|
+
-- Inventories indexes
|
|
211
|
+
CREATE INDEX IF NOT EXISTS idx_inventories_account ON inventories(account_id);
|
|
212
|
+
CREATE INDEX IF NOT EXISTS idx_inventories_name_account ON inventories(name, account_id);
|
|
213
|
+
|
|
214
|
+
-- Audit indexes (for history queries and filtering)
|
|
215
|
+
CREATE INDEX IF NOT EXISTS idx_audit_ops_timestamp ON audit_operations(timestamp DESC);
|
|
216
|
+
CREATE INDEX IF NOT EXISTS idx_audit_ops_account ON audit_operations(account_id);
|
|
217
|
+
CREATE INDEX IF NOT EXISTS idx_audit_ops_account_timestamp ON audit_operations(account_id, timestamp DESC);
|
|
218
|
+
CREATE INDEX IF NOT EXISTS idx_audit_records_operation ON audit_records(operation_id);
|
|
219
|
+
CREATE INDEX IF NOT EXISTS idx_audit_records_arn ON audit_records(resource_arn);
|
|
220
|
+
CREATE INDEX IF NOT EXISTS idx_audit_records_type ON audit_records(resource_type);
|
|
221
|
+
CREATE INDEX IF NOT EXISTS idx_audit_records_region ON audit_records(region);
|
|
222
|
+
CREATE INDEX IF NOT EXISTS idx_audit_records_status ON audit_records(status);
|
|
223
|
+
|
|
224
|
+
-- Saved queries indexes
|
|
225
|
+
CREATE INDEX IF NOT EXISTS idx_queries_category ON saved_queries(category);
|
|
226
|
+
CREATE INDEX IF NOT EXISTS idx_queries_favorite ON saved_queries(is_favorite);
|
|
227
|
+
CREATE INDEX IF NOT EXISTS idx_queries_last_run ON saved_queries(last_run_at DESC);
|
|
228
|
+
|
|
229
|
+
-- Saved filters indexes
|
|
230
|
+
CREATE INDEX IF NOT EXISTS idx_filters_favorite ON saved_filters(is_favorite);
|
|
231
|
+
CREATE INDEX IF NOT EXISTS idx_filters_last_used ON saved_filters(last_used_at DESC);
|
|
232
|
+
|
|
233
|
+
-- Saved views indexes
|
|
234
|
+
CREATE INDEX IF NOT EXISTS idx_views_default ON saved_views(is_default);
|
|
235
|
+
CREATE INDEX IF NOT EXISTS idx_views_favorite ON saved_views(is_favorite);
|
|
236
|
+
CREATE INDEX IF NOT EXISTS idx_views_last_used ON saved_views(last_used_at DESC);
|
|
237
|
+
|
|
238
|
+
-- Resource groups indexes
|
|
239
|
+
CREATE INDEX IF NOT EXISTS idx_groups_name ON resource_groups(name);
|
|
240
|
+
CREATE INDEX IF NOT EXISTS idx_groups_favorite ON resource_groups(is_favorite);
|
|
241
|
+
CREATE INDEX IF NOT EXISTS idx_groups_created ON resource_groups(created_at DESC);
|
|
242
|
+
|
|
243
|
+
-- Resource group members indexes
|
|
244
|
+
CREATE INDEX IF NOT EXISTS idx_group_members_group ON resource_group_members(group_id);
|
|
245
|
+
CREATE INDEX IF NOT EXISTS idx_group_members_name_type ON resource_group_members(resource_name, resource_type);
|
|
246
|
+
CREATE INDEX IF NOT EXISTS idx_group_members_strategy ON resource_group_members(match_strategy);
|
|
247
|
+
"""
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
MIGRATIONS = {
|
|
251
|
+
"1.1.0": [
|
|
252
|
+
# Add canonical_name column to resources table
|
|
253
|
+
"ALTER TABLE resources ADD COLUMN canonical_name TEXT",
|
|
254
|
+
# Add match_strategy column to resource_group_members table
|
|
255
|
+
"ALTER TABLE resource_group_members ADD COLUMN match_strategy TEXT DEFAULT 'physical_name'",
|
|
256
|
+
# Backfill canonical_name from CloudFormation logical-id tag
|
|
257
|
+
"""
|
|
258
|
+
UPDATE resources
|
|
259
|
+
SET canonical_name = (
|
|
260
|
+
SELECT value FROM resource_tags
|
|
261
|
+
WHERE resource_tags.resource_id = resources.id
|
|
262
|
+
AND key = 'aws:cloudformation:logical-id'
|
|
263
|
+
)
|
|
264
|
+
WHERE canonical_name IS NULL
|
|
265
|
+
""",
|
|
266
|
+
# Fallback to physical name for resources without CloudFormation tag
|
|
267
|
+
"""
|
|
268
|
+
UPDATE resources
|
|
269
|
+
SET canonical_name = COALESCE(name, arn)
|
|
270
|
+
WHERE canonical_name IS NULL
|
|
271
|
+
""",
|
|
272
|
+
],
|
|
273
|
+
"1.2.0": [
|
|
274
|
+
# Add normalized_name column for pattern-stripped names
|
|
275
|
+
"ALTER TABLE resources ADD COLUMN normalized_name TEXT",
|
|
276
|
+
# Add extracted_patterns column for storing what was stripped (JSON)
|
|
277
|
+
"ALTER TABLE resources ADD COLUMN extracted_patterns TEXT",
|
|
278
|
+
# Add normalization_method column for tracking how normalization was done
|
|
279
|
+
"ALTER TABLE resources ADD COLUMN normalization_method TEXT",
|
|
280
|
+
# Backfill normalized_name from CloudFormation logical-id tag
|
|
281
|
+
"""
|
|
282
|
+
UPDATE resources
|
|
283
|
+
SET normalized_name = (
|
|
284
|
+
SELECT value FROM resource_tags
|
|
285
|
+
WHERE resource_tags.resource_id = resources.id
|
|
286
|
+
AND key = 'aws:cloudformation:logical-id'
|
|
287
|
+
),
|
|
288
|
+
normalization_method = 'tag:logical-id'
|
|
289
|
+
WHERE normalized_name IS NULL
|
|
290
|
+
AND EXISTS (
|
|
291
|
+
SELECT 1 FROM resource_tags
|
|
292
|
+
WHERE resource_tags.resource_id = resources.id
|
|
293
|
+
AND key = 'aws:cloudformation:logical-id'
|
|
294
|
+
)
|
|
295
|
+
""",
|
|
296
|
+
# Backfill from Name tag
|
|
297
|
+
"""
|
|
298
|
+
UPDATE resources
|
|
299
|
+
SET normalized_name = (
|
|
300
|
+
SELECT value FROM resource_tags
|
|
301
|
+
WHERE resource_tags.resource_id = resources.id
|
|
302
|
+
AND key = 'Name'
|
|
303
|
+
),
|
|
304
|
+
normalization_method = 'tag:Name'
|
|
305
|
+
WHERE normalized_name IS NULL
|
|
306
|
+
AND EXISTS (
|
|
307
|
+
SELECT 1 FROM resource_tags
|
|
308
|
+
WHERE resource_tags.resource_id = resources.id
|
|
309
|
+
AND key = 'Name'
|
|
310
|
+
)
|
|
311
|
+
""",
|
|
312
|
+
# Fallback to physical name (pattern extraction needs Python, done on re-snapshot)
|
|
313
|
+
"""
|
|
314
|
+
UPDATE resources
|
|
315
|
+
SET normalized_name = COALESCE(name, arn),
|
|
316
|
+
normalization_method = 'none'
|
|
317
|
+
WHERE normalized_name IS NULL
|
|
318
|
+
""",
|
|
319
|
+
],
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def get_schema_sql() -> str:
|
|
324
|
+
"""Get the full schema SQL."""
|
|
325
|
+
return SCHEMA_SQL
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def get_indexes_sql() -> str:
|
|
329
|
+
"""Get the indexes SQL."""
|
|
330
|
+
return INDEXES_SQL
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def get_migrations() -> dict:
|
|
334
|
+
"""Get the migrations dictionary.
|
|
335
|
+
|
|
336
|
+
Returns:
|
|
337
|
+
Dict mapping version strings to lists of SQL statements
|
|
338
|
+
"""
|
|
339
|
+
return MIGRATIONS
|
|
@@ -0,0 +1,363 @@
|
|
|
1
|
+
"""Snapshot storage operations for SQLite backend."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import logging
|
|
5
|
+
from datetime import datetime, timezone
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Dict, List, Optional
|
|
8
|
+
|
|
9
|
+
from ..matching import ResourceNormalizer
|
|
10
|
+
from ..models.resource import Resource
|
|
11
|
+
from ..models.snapshot import Snapshot
|
|
12
|
+
from .database import Database, json_deserialize, json_serialize
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def compute_canonical_name(name: str, tags: Optional[Dict[str, str]], arn: str) -> str:
|
|
18
|
+
"""Compute canonical name for a resource.
|
|
19
|
+
|
|
20
|
+
Priority order:
|
|
21
|
+
1. aws:cloudformation:logical-id tag (stable across recreations)
|
|
22
|
+
2. Resource name
|
|
23
|
+
3. ARN as fallback
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
name: Resource physical name
|
|
27
|
+
tags: Resource tags
|
|
28
|
+
arn: Resource ARN
|
|
29
|
+
|
|
30
|
+
Returns:
|
|
31
|
+
Canonical name for matching
|
|
32
|
+
"""
|
|
33
|
+
if tags and "aws:cloudformation:logical-id" in tags:
|
|
34
|
+
return tags["aws:cloudformation:logical-id"]
|
|
35
|
+
return name or arn
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class SnapshotStore:
|
|
39
|
+
"""CRUD operations for snapshots in SQLite database."""
|
|
40
|
+
|
|
41
|
+
def __init__(self, db: Database):
|
|
42
|
+
"""Initialize snapshot store.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
db: Database connection manager
|
|
46
|
+
"""
|
|
47
|
+
self.db = db
|
|
48
|
+
|
|
49
|
+
def save(self, snapshot: Snapshot) -> int:
|
|
50
|
+
"""Save snapshot and all its resources to database.
|
|
51
|
+
|
|
52
|
+
Args:
|
|
53
|
+
snapshot: Snapshot to save
|
|
54
|
+
|
|
55
|
+
Returns:
|
|
56
|
+
Database ID of saved snapshot
|
|
57
|
+
"""
|
|
58
|
+
# Create normalizer for computing normalized names
|
|
59
|
+
normalizer = ResourceNormalizer()
|
|
60
|
+
|
|
61
|
+
with self.db.transaction() as cursor:
|
|
62
|
+
# Insert snapshot
|
|
63
|
+
cursor.execute(
|
|
64
|
+
"""
|
|
65
|
+
INSERT INTO snapshots (
|
|
66
|
+
name, created_at, account_id, regions, resource_count,
|
|
67
|
+
total_resources_before_filter, service_counts, metadata,
|
|
68
|
+
filters_applied, schema_version, inventory_name, is_active
|
|
69
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
70
|
+
""",
|
|
71
|
+
(
|
|
72
|
+
snapshot.name,
|
|
73
|
+
snapshot.created_at.isoformat(),
|
|
74
|
+
snapshot.account_id,
|
|
75
|
+
json_serialize(snapshot.regions),
|
|
76
|
+
snapshot.resource_count,
|
|
77
|
+
snapshot.total_resources_before_filter,
|
|
78
|
+
json_serialize(snapshot.service_counts),
|
|
79
|
+
json_serialize(snapshot.metadata),
|
|
80
|
+
json_serialize(snapshot.filters_applied),
|
|
81
|
+
snapshot.schema_version,
|
|
82
|
+
snapshot.inventory_name,
|
|
83
|
+
snapshot.is_active,
|
|
84
|
+
),
|
|
85
|
+
)
|
|
86
|
+
snapshot_id = cursor.lastrowid
|
|
87
|
+
|
|
88
|
+
# Insert resources
|
|
89
|
+
for resource in snapshot.resources:
|
|
90
|
+
# Compute canonical name (for backward compatibility)
|
|
91
|
+
canonical = compute_canonical_name(resource.name, resource.tags, resource.arn)
|
|
92
|
+
|
|
93
|
+
# Compute normalized name with pattern extraction
|
|
94
|
+
norm_result = normalizer.normalize_single(
|
|
95
|
+
resource.name,
|
|
96
|
+
resource.resource_type,
|
|
97
|
+
resource.tags,
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
cursor.execute(
|
|
101
|
+
"""
|
|
102
|
+
INSERT INTO resources (
|
|
103
|
+
snapshot_id, arn, resource_type, name, region,
|
|
104
|
+
config_hash, raw_config, created_at, source, canonical_name,
|
|
105
|
+
normalized_name, extracted_patterns, normalization_method
|
|
106
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
107
|
+
""",
|
|
108
|
+
(
|
|
109
|
+
snapshot_id,
|
|
110
|
+
resource.arn,
|
|
111
|
+
resource.resource_type,
|
|
112
|
+
resource.name,
|
|
113
|
+
resource.region,
|
|
114
|
+
resource.config_hash,
|
|
115
|
+
json_serialize(resource.raw_config),
|
|
116
|
+
resource.created_at.isoformat() if resource.created_at else None,
|
|
117
|
+
resource.source,
|
|
118
|
+
canonical,
|
|
119
|
+
norm_result.normalized_name,
|
|
120
|
+
json_serialize(norm_result.extracted_patterns),
|
|
121
|
+
norm_result.method,
|
|
122
|
+
),
|
|
123
|
+
)
|
|
124
|
+
resource_id = cursor.lastrowid
|
|
125
|
+
|
|
126
|
+
# Insert tags
|
|
127
|
+
if resource.tags:
|
|
128
|
+
tag_data = [(resource_id, k, v) for k, v in resource.tags.items()]
|
|
129
|
+
cursor.executemany(
|
|
130
|
+
"INSERT INTO resource_tags (resource_id, key, value) VALUES (?, ?, ?)",
|
|
131
|
+
tag_data,
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
logger.debug(f"Saved snapshot '{snapshot.name}' with {len(snapshot.resources)} resources (id={snapshot_id})")
|
|
135
|
+
return snapshot_id
|
|
136
|
+
|
|
137
|
+
def load(self, name: str) -> Optional[Snapshot]:
|
|
138
|
+
"""Load snapshot by name with all resources.
|
|
139
|
+
|
|
140
|
+
Args:
|
|
141
|
+
name: Snapshot name
|
|
142
|
+
|
|
143
|
+
Returns:
|
|
144
|
+
Snapshot object or None if not found
|
|
145
|
+
"""
|
|
146
|
+
# Get snapshot
|
|
147
|
+
snapshot_row = self.db.fetchone("SELECT * FROM snapshots WHERE name = ?", (name,))
|
|
148
|
+
if not snapshot_row:
|
|
149
|
+
return None
|
|
150
|
+
|
|
151
|
+
snapshot_id = snapshot_row["id"]
|
|
152
|
+
|
|
153
|
+
# Get resources
|
|
154
|
+
resource_rows = self.db.fetchall(
|
|
155
|
+
"SELECT * FROM resources WHERE snapshot_id = ?",
|
|
156
|
+
(snapshot_id,),
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
# Get tags for all resources in one query
|
|
160
|
+
resource_ids = [r["id"] for r in resource_rows]
|
|
161
|
+
tags_by_resource: Dict[int, Dict[str, str]] = {}
|
|
162
|
+
|
|
163
|
+
if resource_ids:
|
|
164
|
+
placeholders = ",".join("?" * len(resource_ids))
|
|
165
|
+
tag_rows = self.db.fetchall(
|
|
166
|
+
f"SELECT resource_id, key, value FROM resource_tags WHERE resource_id IN ({placeholders})",
|
|
167
|
+
tuple(resource_ids),
|
|
168
|
+
)
|
|
169
|
+
for tag_row in tag_rows:
|
|
170
|
+
rid = tag_row["resource_id"]
|
|
171
|
+
if rid not in tags_by_resource:
|
|
172
|
+
tags_by_resource[rid] = {}
|
|
173
|
+
tags_by_resource[rid][tag_row["key"]] = tag_row["value"]
|
|
174
|
+
|
|
175
|
+
# Build Resource objects
|
|
176
|
+
resources = []
|
|
177
|
+
for row in resource_rows:
|
|
178
|
+
created_at = None
|
|
179
|
+
if row["created_at"]:
|
|
180
|
+
try:
|
|
181
|
+
created_at = datetime.fromisoformat(row["created_at"])
|
|
182
|
+
except ValueError:
|
|
183
|
+
pass
|
|
184
|
+
|
|
185
|
+
resource = Resource(
|
|
186
|
+
arn=row["arn"],
|
|
187
|
+
resource_type=row["resource_type"],
|
|
188
|
+
name=row["name"],
|
|
189
|
+
region=row["region"],
|
|
190
|
+
config_hash=row["config_hash"],
|
|
191
|
+
raw_config=json_deserialize(row["raw_config"]),
|
|
192
|
+
tags=tags_by_resource.get(row["id"], {}),
|
|
193
|
+
created_at=created_at,
|
|
194
|
+
source=row["source"] or "direct_api",
|
|
195
|
+
)
|
|
196
|
+
resources.append(resource)
|
|
197
|
+
|
|
198
|
+
# Build Snapshot
|
|
199
|
+
created_at = datetime.fromisoformat(snapshot_row["created_at"])
|
|
200
|
+
if created_at.tzinfo is None:
|
|
201
|
+
created_at = created_at.replace(tzinfo=timezone.utc)
|
|
202
|
+
|
|
203
|
+
snapshot = Snapshot(
|
|
204
|
+
name=snapshot_row["name"],
|
|
205
|
+
created_at=created_at,
|
|
206
|
+
account_id=snapshot_row["account_id"],
|
|
207
|
+
regions=json_deserialize(snapshot_row["regions"]) or [],
|
|
208
|
+
resources=resources,
|
|
209
|
+
is_active=bool(snapshot_row["is_active"]),
|
|
210
|
+
resource_count=snapshot_row["resource_count"],
|
|
211
|
+
total_resources_before_filter=snapshot_row.get("total_resources_before_filter"),
|
|
212
|
+
service_counts=json_deserialize(snapshot_row["service_counts"]) or {},
|
|
213
|
+
metadata=json_deserialize(snapshot_row["metadata"]) or {},
|
|
214
|
+
filters_applied=json_deserialize(snapshot_row["filters_applied"]),
|
|
215
|
+
inventory_name=snapshot_row["inventory_name"] or "default",
|
|
216
|
+
schema_version=snapshot_row["schema_version"] or "1.1",
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
logger.debug(f"Loaded snapshot '{name}' with {len(resources)} resources")
|
|
220
|
+
return snapshot
|
|
221
|
+
|
|
222
|
+
def list_all(self) -> List[Dict[str, Any]]:
|
|
223
|
+
"""List all snapshots with metadata (no resources).
|
|
224
|
+
|
|
225
|
+
Returns:
|
|
226
|
+
List of snapshot metadata dictionaries
|
|
227
|
+
"""
|
|
228
|
+
rows = self.db.fetchall(
|
|
229
|
+
"""
|
|
230
|
+
SELECT name, created_at, account_id, regions, resource_count,
|
|
231
|
+
service_counts, is_active, inventory_name
|
|
232
|
+
FROM snapshots
|
|
233
|
+
ORDER BY created_at DESC
|
|
234
|
+
"""
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
results = []
|
|
238
|
+
for row in rows:
|
|
239
|
+
created_at = datetime.fromisoformat(row["created_at"])
|
|
240
|
+
results.append(
|
|
241
|
+
{
|
|
242
|
+
"name": row["name"],
|
|
243
|
+
"created_at": created_at,
|
|
244
|
+
"account_id": row["account_id"],
|
|
245
|
+
"regions": json_deserialize(row["regions"]) or [],
|
|
246
|
+
"resource_count": row["resource_count"],
|
|
247
|
+
"service_counts": json_deserialize(row["service_counts"]) or {},
|
|
248
|
+
"is_active": bool(row["is_active"]),
|
|
249
|
+
"inventory_name": row["inventory_name"],
|
|
250
|
+
}
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
return results
|
|
254
|
+
|
|
255
|
+
def delete(self, name: str) -> bool:
|
|
256
|
+
"""Delete snapshot and cascade to resources.
|
|
257
|
+
|
|
258
|
+
Args:
|
|
259
|
+
name: Snapshot name to delete
|
|
260
|
+
|
|
261
|
+
Returns:
|
|
262
|
+
True if deleted, False if not found
|
|
263
|
+
"""
|
|
264
|
+
with self.db.transaction() as cursor:
|
|
265
|
+
cursor.execute("DELETE FROM snapshots WHERE name = ?", (name,))
|
|
266
|
+
deleted = cursor.rowcount > 0
|
|
267
|
+
|
|
268
|
+
if deleted:
|
|
269
|
+
logger.debug(f"Deleted snapshot '{name}'")
|
|
270
|
+
return deleted
|
|
271
|
+
|
|
272
|
+
def exists(self, name: str) -> bool:
|
|
273
|
+
"""Check if snapshot exists.
|
|
274
|
+
|
|
275
|
+
Args:
|
|
276
|
+
name: Snapshot name
|
|
277
|
+
|
|
278
|
+
Returns:
|
|
279
|
+
True if exists
|
|
280
|
+
"""
|
|
281
|
+
row = self.db.fetchone("SELECT 1 FROM snapshots WHERE name = ?", (name,))
|
|
282
|
+
return row is not None
|
|
283
|
+
|
|
284
|
+
def rename(self, old_name: str, new_name: str) -> bool:
|
|
285
|
+
"""Rename a snapshot.
|
|
286
|
+
|
|
287
|
+
Args:
|
|
288
|
+
old_name: Current snapshot name
|
|
289
|
+
new_name: New snapshot name
|
|
290
|
+
|
|
291
|
+
Returns:
|
|
292
|
+
True if renamed, False if old_name not found
|
|
293
|
+
|
|
294
|
+
Raises:
|
|
295
|
+
ValueError: If new_name already exists
|
|
296
|
+
"""
|
|
297
|
+
if not self.exists(old_name):
|
|
298
|
+
return False
|
|
299
|
+
|
|
300
|
+
if self.exists(new_name):
|
|
301
|
+
raise ValueError(f"Snapshot '{new_name}' already exists")
|
|
302
|
+
|
|
303
|
+
with self.db.transaction() as cursor:
|
|
304
|
+
cursor.execute(
|
|
305
|
+
"UPDATE snapshots SET name = ? WHERE name = ?",
|
|
306
|
+
(new_name, old_name),
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
logger.debug(f"Renamed snapshot '{old_name}' to '{new_name}'")
|
|
310
|
+
return True
|
|
311
|
+
|
|
312
|
+
def get_active(self) -> Optional[str]:
|
|
313
|
+
"""Get name of active snapshot.
|
|
314
|
+
|
|
315
|
+
Returns:
|
|
316
|
+
Active snapshot name or None
|
|
317
|
+
"""
|
|
318
|
+
row = self.db.fetchone("SELECT name FROM snapshots WHERE is_active = 1")
|
|
319
|
+
return row["name"] if row else None
|
|
320
|
+
|
|
321
|
+
def set_active(self, name: str) -> None:
|
|
322
|
+
"""Set snapshot as active baseline.
|
|
323
|
+
|
|
324
|
+
Args:
|
|
325
|
+
name: Snapshot name to set as active
|
|
326
|
+
"""
|
|
327
|
+
with self.db.transaction() as cursor:
|
|
328
|
+
# Clear previous active
|
|
329
|
+
cursor.execute("UPDATE snapshots SET is_active = 0 WHERE is_active = 1")
|
|
330
|
+
# Set new active
|
|
331
|
+
cursor.execute("UPDATE snapshots SET is_active = 1 WHERE name = ?", (name,))
|
|
332
|
+
|
|
333
|
+
logger.debug(f"Set active snapshot: '{name}'")
|
|
334
|
+
|
|
335
|
+
def get_id(self, name: str) -> Optional[int]:
|
|
336
|
+
"""Get database ID for snapshot.
|
|
337
|
+
|
|
338
|
+
Args:
|
|
339
|
+
name: Snapshot name
|
|
340
|
+
|
|
341
|
+
Returns:
|
|
342
|
+
Database ID or None
|
|
343
|
+
"""
|
|
344
|
+
row = self.db.fetchone("SELECT id FROM snapshots WHERE name = ?", (name,))
|
|
345
|
+
return row["id"] if row else None
|
|
346
|
+
|
|
347
|
+
def get_resource_count(self) -> int:
|
|
348
|
+
"""Get total resource count across all snapshots.
|
|
349
|
+
|
|
350
|
+
Returns:
|
|
351
|
+
Total resource count
|
|
352
|
+
"""
|
|
353
|
+
row = self.db.fetchone("SELECT COUNT(*) as count FROM resources")
|
|
354
|
+
return row["count"] if row else 0
|
|
355
|
+
|
|
356
|
+
def get_snapshot_count(self) -> int:
|
|
357
|
+
"""Get total snapshot count.
|
|
358
|
+
|
|
359
|
+
Returns:
|
|
360
|
+
Snapshot count
|
|
361
|
+
"""
|
|
362
|
+
row = self.db.fetchone("SELECT COUNT(*) as count FROM snapshots")
|
|
363
|
+
return row["count"] if row else 0
|