aws-inventory-manager 0.13.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of aws-inventory-manager might be problematic. Click here for more details.
- aws_inventory_manager-0.13.2.dist-info/LICENSE +21 -0
- aws_inventory_manager-0.13.2.dist-info/METADATA +1226 -0
- aws_inventory_manager-0.13.2.dist-info/RECORD +145 -0
- aws_inventory_manager-0.13.2.dist-info/WHEEL +5 -0
- aws_inventory_manager-0.13.2.dist-info/entry_points.txt +2 -0
- aws_inventory_manager-0.13.2.dist-info/top_level.txt +1 -0
- src/__init__.py +3 -0
- src/aws/__init__.py +11 -0
- src/aws/client.py +128 -0
- src/aws/credentials.py +191 -0
- src/aws/rate_limiter.py +177 -0
- src/cli/__init__.py +12 -0
- src/cli/config.py +130 -0
- src/cli/main.py +3626 -0
- src/config_service/__init__.py +21 -0
- src/config_service/collector.py +346 -0
- src/config_service/detector.py +256 -0
- src/config_service/resource_type_mapping.py +328 -0
- src/cost/__init__.py +5 -0
- src/cost/analyzer.py +226 -0
- src/cost/explorer.py +209 -0
- src/cost/reporter.py +237 -0
- src/delta/__init__.py +5 -0
- src/delta/calculator.py +206 -0
- src/delta/differ.py +185 -0
- src/delta/formatters.py +272 -0
- src/delta/models.py +154 -0
- src/delta/reporter.py +234 -0
- src/models/__init__.py +21 -0
- src/models/config_diff.py +135 -0
- src/models/cost_report.py +87 -0
- src/models/deletion_operation.py +104 -0
- src/models/deletion_record.py +97 -0
- src/models/delta_report.py +122 -0
- src/models/efs_resource.py +80 -0
- src/models/elasticache_resource.py +90 -0
- src/models/group.py +318 -0
- src/models/inventory.py +133 -0
- src/models/protection_rule.py +123 -0
- src/models/report.py +288 -0
- src/models/resource.py +111 -0
- src/models/security_finding.py +102 -0
- src/models/snapshot.py +122 -0
- src/restore/__init__.py +20 -0
- src/restore/audit.py +175 -0
- src/restore/cleaner.py +461 -0
- src/restore/config.py +209 -0
- src/restore/deleter.py +976 -0
- src/restore/dependency.py +254 -0
- src/restore/safety.py +115 -0
- src/security/__init__.py +0 -0
- src/security/checks/__init__.py +0 -0
- src/security/checks/base.py +56 -0
- src/security/checks/ec2_checks.py +88 -0
- src/security/checks/elasticache_checks.py +149 -0
- src/security/checks/iam_checks.py +102 -0
- src/security/checks/rds_checks.py +140 -0
- src/security/checks/s3_checks.py +95 -0
- src/security/checks/secrets_checks.py +96 -0
- src/security/checks/sg_checks.py +142 -0
- src/security/cis_mapper.py +97 -0
- src/security/models.py +53 -0
- src/security/reporter.py +174 -0
- src/security/scanner.py +87 -0
- src/snapshot/__init__.py +6 -0
- src/snapshot/capturer.py +451 -0
- src/snapshot/filter.py +259 -0
- src/snapshot/inventory_storage.py +236 -0
- src/snapshot/report_formatter.py +250 -0
- src/snapshot/reporter.py +189 -0
- src/snapshot/resource_collectors/__init__.py +5 -0
- src/snapshot/resource_collectors/apigateway.py +140 -0
- src/snapshot/resource_collectors/backup.py +136 -0
- src/snapshot/resource_collectors/base.py +81 -0
- src/snapshot/resource_collectors/cloudformation.py +55 -0
- src/snapshot/resource_collectors/cloudwatch.py +109 -0
- src/snapshot/resource_collectors/codebuild.py +69 -0
- src/snapshot/resource_collectors/codepipeline.py +82 -0
- src/snapshot/resource_collectors/dynamodb.py +65 -0
- src/snapshot/resource_collectors/ec2.py +240 -0
- src/snapshot/resource_collectors/ecs.py +215 -0
- src/snapshot/resource_collectors/efs_collector.py +102 -0
- src/snapshot/resource_collectors/eks.py +200 -0
- src/snapshot/resource_collectors/elasticache_collector.py +79 -0
- src/snapshot/resource_collectors/elb.py +126 -0
- src/snapshot/resource_collectors/eventbridge.py +156 -0
- src/snapshot/resource_collectors/iam.py +188 -0
- src/snapshot/resource_collectors/kms.py +111 -0
- src/snapshot/resource_collectors/lambda_func.py +139 -0
- src/snapshot/resource_collectors/rds.py +109 -0
- src/snapshot/resource_collectors/route53.py +86 -0
- src/snapshot/resource_collectors/s3.py +105 -0
- src/snapshot/resource_collectors/secretsmanager.py +70 -0
- src/snapshot/resource_collectors/sns.py +68 -0
- src/snapshot/resource_collectors/sqs.py +82 -0
- src/snapshot/resource_collectors/ssm.py +160 -0
- src/snapshot/resource_collectors/stepfunctions.py +74 -0
- src/snapshot/resource_collectors/vpcendpoints.py +79 -0
- src/snapshot/resource_collectors/waf.py +159 -0
- src/snapshot/storage.py +351 -0
- src/storage/__init__.py +21 -0
- src/storage/audit_store.py +419 -0
- src/storage/database.py +294 -0
- src/storage/group_store.py +749 -0
- src/storage/inventory_store.py +320 -0
- src/storage/resource_store.py +413 -0
- src/storage/schema.py +288 -0
- src/storage/snapshot_store.py +346 -0
- src/utils/__init__.py +12 -0
- src/utils/export.py +305 -0
- src/utils/hash.py +60 -0
- src/utils/logging.py +63 -0
- src/utils/pagination.py +41 -0
- src/utils/paths.py +51 -0
- src/utils/progress.py +41 -0
- src/utils/unsupported_resources.py +306 -0
- src/web/__init__.py +5 -0
- src/web/app.py +97 -0
- src/web/dependencies.py +69 -0
- src/web/routes/__init__.py +1 -0
- src/web/routes/api/__init__.py +18 -0
- src/web/routes/api/charts.py +156 -0
- src/web/routes/api/cleanup.py +186 -0
- src/web/routes/api/filters.py +253 -0
- src/web/routes/api/groups.py +305 -0
- src/web/routes/api/inventories.py +80 -0
- src/web/routes/api/queries.py +202 -0
- src/web/routes/api/resources.py +379 -0
- src/web/routes/api/snapshots.py +314 -0
- src/web/routes/api/views.py +260 -0
- src/web/routes/pages.py +198 -0
- src/web/services/__init__.py +1 -0
- src/web/templates/base.html +949 -0
- src/web/templates/components/navbar.html +31 -0
- src/web/templates/components/sidebar.html +104 -0
- src/web/templates/pages/audit_logs.html +86 -0
- src/web/templates/pages/cleanup.html +279 -0
- src/web/templates/pages/dashboard.html +227 -0
- src/web/templates/pages/diff.html +175 -0
- src/web/templates/pages/error.html +30 -0
- src/web/templates/pages/groups.html +721 -0
- src/web/templates/pages/queries.html +246 -0
- src/web/templates/pages/resources.html +2251 -0
- src/web/templates/pages/snapshot_detail.html +271 -0
- src/web/templates/pages/snapshots.html +429 -0
|
@@ -0,0 +1,413 @@
|
|
|
1
|
+
"""Resource query operations for SQLite backend."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import re
|
|
5
|
+
from datetime import datetime
|
|
6
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
7
|
+
|
|
8
|
+
from .database import Database, json_deserialize
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ResourceStore:
|
|
14
|
+
"""Query operations for resources across snapshots."""
|
|
15
|
+
|
|
16
|
+
def __init__(self, db: Database):
|
|
17
|
+
"""Initialize resource store.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
db: Database connection manager
|
|
21
|
+
"""
|
|
22
|
+
self.db = db
|
|
23
|
+
|
|
24
|
+
def query_raw(self, sql: str, params: Tuple = ()) -> List[Dict[str, Any]]:
|
|
25
|
+
"""Execute raw SQL query on the database.
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
sql: SQL query (should be SELECT only)
|
|
29
|
+
params: Query parameters
|
|
30
|
+
|
|
31
|
+
Returns:
|
|
32
|
+
List of result dictionaries
|
|
33
|
+
|
|
34
|
+
Raises:
|
|
35
|
+
ValueError: If query is not a SELECT statement
|
|
36
|
+
"""
|
|
37
|
+
# Basic SQL injection prevention - only allow SELECT
|
|
38
|
+
sql_upper = sql.strip().upper()
|
|
39
|
+
if not sql_upper.startswith("SELECT"):
|
|
40
|
+
raise ValueError("Only SELECT queries are allowed")
|
|
41
|
+
|
|
42
|
+
# Block dangerous keywords
|
|
43
|
+
dangerous = ["DROP", "DELETE", "UPDATE", "INSERT", "ALTER", "CREATE", "TRUNCATE"]
|
|
44
|
+
for keyword in dangerous:
|
|
45
|
+
if re.search(rf"\b{keyword}\b", sql_upper):
|
|
46
|
+
raise ValueError(f"Query contains forbidden keyword: {keyword}")
|
|
47
|
+
|
|
48
|
+
return self.db.fetchall(sql, params)
|
|
49
|
+
|
|
50
|
+
def search(
|
|
51
|
+
self,
|
|
52
|
+
arn_pattern: Optional[str] = None,
|
|
53
|
+
resource_type: Optional[str] = None,
|
|
54
|
+
region: Optional[str] = None,
|
|
55
|
+
tag_key: Optional[str] = None,
|
|
56
|
+
tag_value: Optional[str] = None,
|
|
57
|
+
snapshot_name: Optional[str] = None,
|
|
58
|
+
created_before: Optional[datetime] = None,
|
|
59
|
+
created_after: Optional[datetime] = None,
|
|
60
|
+
limit: int = 100,
|
|
61
|
+
offset: int = 0,
|
|
62
|
+
) -> List[Dict[str, Any]]:
|
|
63
|
+
"""Search resources with filters.
|
|
64
|
+
|
|
65
|
+
Args:
|
|
66
|
+
arn_pattern: ARN pattern to match (supports % wildcard)
|
|
67
|
+
resource_type: Filter by resource type (exact or partial)
|
|
68
|
+
region: Filter by region
|
|
69
|
+
tag_key: Filter by tag key
|
|
70
|
+
tag_value: Filter by tag value (requires tag_key)
|
|
71
|
+
snapshot_name: Limit to specific snapshot
|
|
72
|
+
created_before: Resources created before this date
|
|
73
|
+
created_after: Resources created after this date
|
|
74
|
+
limit: Maximum results to return
|
|
75
|
+
offset: Offset for pagination
|
|
76
|
+
|
|
77
|
+
Returns:
|
|
78
|
+
List of matching resources with snapshot info
|
|
79
|
+
"""
|
|
80
|
+
conditions = []
|
|
81
|
+
params: List[Any] = []
|
|
82
|
+
|
|
83
|
+
# Build query with joins
|
|
84
|
+
base_query = """
|
|
85
|
+
SELECT DISTINCT
|
|
86
|
+
r.arn,
|
|
87
|
+
r.resource_type,
|
|
88
|
+
r.name,
|
|
89
|
+
r.region,
|
|
90
|
+
r.config_hash,
|
|
91
|
+
r.created_at,
|
|
92
|
+
r.source,
|
|
93
|
+
s.name as snapshot_name,
|
|
94
|
+
s.created_at as snapshot_created_at,
|
|
95
|
+
s.account_id
|
|
96
|
+
FROM resources r
|
|
97
|
+
JOIN snapshots s ON r.snapshot_id = s.id
|
|
98
|
+
"""
|
|
99
|
+
|
|
100
|
+
# Tag join if filtering by tags
|
|
101
|
+
if tag_key:
|
|
102
|
+
base_query += " JOIN resource_tags t ON r.id = t.resource_id"
|
|
103
|
+
conditions.append("t.key = ?")
|
|
104
|
+
params.append(tag_key)
|
|
105
|
+
if tag_value:
|
|
106
|
+
conditions.append("t.value = ?")
|
|
107
|
+
params.append(tag_value)
|
|
108
|
+
|
|
109
|
+
# ARN filter
|
|
110
|
+
if arn_pattern:
|
|
111
|
+
if "%" in arn_pattern:
|
|
112
|
+
conditions.append("r.arn LIKE ?")
|
|
113
|
+
else:
|
|
114
|
+
conditions.append("r.arn LIKE ?")
|
|
115
|
+
arn_pattern = f"%{arn_pattern}%"
|
|
116
|
+
params.append(arn_pattern)
|
|
117
|
+
|
|
118
|
+
# Resource type filter
|
|
119
|
+
if resource_type:
|
|
120
|
+
if ":" in resource_type:
|
|
121
|
+
conditions.append("r.resource_type = ?")
|
|
122
|
+
else:
|
|
123
|
+
conditions.append("r.resource_type LIKE ?")
|
|
124
|
+
resource_type = f"%{resource_type}%"
|
|
125
|
+
params.append(resource_type)
|
|
126
|
+
|
|
127
|
+
# Region filter
|
|
128
|
+
if region:
|
|
129
|
+
conditions.append("r.region = ?")
|
|
130
|
+
params.append(region)
|
|
131
|
+
|
|
132
|
+
# Snapshot filter
|
|
133
|
+
if snapshot_name:
|
|
134
|
+
conditions.append("s.name = ?")
|
|
135
|
+
params.append(snapshot_name)
|
|
136
|
+
|
|
137
|
+
# Date filters
|
|
138
|
+
if created_before:
|
|
139
|
+
conditions.append("r.created_at < ?")
|
|
140
|
+
params.append(created_before.isoformat())
|
|
141
|
+
|
|
142
|
+
if created_after:
|
|
143
|
+
conditions.append("r.created_at >= ?")
|
|
144
|
+
params.append(created_after.isoformat())
|
|
145
|
+
|
|
146
|
+
# Build final query
|
|
147
|
+
if conditions:
|
|
148
|
+
base_query += " WHERE " + " AND ".join(conditions)
|
|
149
|
+
|
|
150
|
+
base_query += " ORDER BY s.created_at DESC, r.arn"
|
|
151
|
+
base_query += f" LIMIT {limit} OFFSET {offset}"
|
|
152
|
+
|
|
153
|
+
return self.db.fetchall(base_query, tuple(params))
|
|
154
|
+
|
|
155
|
+
def get_history(self, arn: str) -> List[Dict[str, Any]]:
|
|
156
|
+
"""Get all snapshots containing a specific resource.
|
|
157
|
+
|
|
158
|
+
Args:
|
|
159
|
+
arn: Resource ARN
|
|
160
|
+
|
|
161
|
+
Returns:
|
|
162
|
+
List of snapshots with resource details, ordered by date
|
|
163
|
+
"""
|
|
164
|
+
return self.db.fetchall(
|
|
165
|
+
"""
|
|
166
|
+
SELECT
|
|
167
|
+
s.name as snapshot_name,
|
|
168
|
+
s.created_at as snapshot_created_at,
|
|
169
|
+
s.account_id,
|
|
170
|
+
r.config_hash,
|
|
171
|
+
r.created_at as resource_created_at,
|
|
172
|
+
r.source
|
|
173
|
+
FROM resources r
|
|
174
|
+
JOIN snapshots s ON r.snapshot_id = s.id
|
|
175
|
+
WHERE r.arn = ?
|
|
176
|
+
ORDER BY s.created_at DESC
|
|
177
|
+
""",
|
|
178
|
+
(arn,),
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
def get_stats(
|
|
182
|
+
self,
|
|
183
|
+
snapshot_name: Optional[str] = None,
|
|
184
|
+
group_by: str = "type",
|
|
185
|
+
) -> List[Dict[str, Any]]:
|
|
186
|
+
"""Get resource statistics.
|
|
187
|
+
|
|
188
|
+
Args:
|
|
189
|
+
snapshot_name: Limit to specific snapshot (None for all)
|
|
190
|
+
group_by: Grouping field - 'type', 'region', 'service', 'snapshot'
|
|
191
|
+
|
|
192
|
+
Returns:
|
|
193
|
+
List of statistics grouped by specified field
|
|
194
|
+
"""
|
|
195
|
+
group_field = {
|
|
196
|
+
"type": "r.resource_type",
|
|
197
|
+
"region": "r.region",
|
|
198
|
+
"service": "SUBSTR(r.resource_type, 1, INSTR(r.resource_type, ':') - 1)",
|
|
199
|
+
"snapshot": "s.name",
|
|
200
|
+
}.get(group_by, "r.resource_type")
|
|
201
|
+
|
|
202
|
+
base_query = f"""
|
|
203
|
+
SELECT
|
|
204
|
+
{group_field} as group_key,
|
|
205
|
+
COUNT(*) as count
|
|
206
|
+
FROM resources r
|
|
207
|
+
JOIN snapshots s ON r.snapshot_id = s.id
|
|
208
|
+
"""
|
|
209
|
+
|
|
210
|
+
params: List[str] = []
|
|
211
|
+
if snapshot_name:
|
|
212
|
+
base_query += " WHERE s.name = ?"
|
|
213
|
+
params.append(snapshot_name)
|
|
214
|
+
|
|
215
|
+
base_query += f" GROUP BY {group_field} ORDER BY count DESC"
|
|
216
|
+
|
|
217
|
+
return self.db.fetchall(base_query, tuple(params))
|
|
218
|
+
|
|
219
|
+
def compare_snapshots(
|
|
220
|
+
self,
|
|
221
|
+
snapshot1_name: str,
|
|
222
|
+
snapshot2_name: str,
|
|
223
|
+
) -> Dict[str, List[Dict[str, Any]]]:
|
|
224
|
+
"""Compare resources between two snapshots.
|
|
225
|
+
|
|
226
|
+
Args:
|
|
227
|
+
snapshot1_name: First (older) snapshot name
|
|
228
|
+
snapshot2_name: Second (newer) snapshot name
|
|
229
|
+
|
|
230
|
+
Returns:
|
|
231
|
+
Dict with 'added', 'removed', 'modified' resource lists
|
|
232
|
+
"""
|
|
233
|
+
# Get resources from both snapshots indexed by ARN
|
|
234
|
+
snap1_resources = self.db.fetchall(
|
|
235
|
+
"""
|
|
236
|
+
SELECT r.arn, r.resource_type, r.name, r.region, r.config_hash
|
|
237
|
+
FROM resources r
|
|
238
|
+
JOIN snapshots s ON r.snapshot_id = s.id
|
|
239
|
+
WHERE s.name = ?
|
|
240
|
+
""",
|
|
241
|
+
(snapshot1_name,),
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
snap2_resources = self.db.fetchall(
|
|
245
|
+
"""
|
|
246
|
+
SELECT r.arn, r.resource_type, r.name, r.region, r.config_hash
|
|
247
|
+
FROM resources r
|
|
248
|
+
JOIN snapshots s ON r.snapshot_id = s.id
|
|
249
|
+
WHERE s.name = ?
|
|
250
|
+
""",
|
|
251
|
+
(snapshot2_name,),
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
snap1_by_arn = {r["arn"]: r for r in snap1_resources}
|
|
255
|
+
snap2_by_arn = {r["arn"]: r for r in snap2_resources}
|
|
256
|
+
|
|
257
|
+
snap1_arns = set(snap1_by_arn.keys())
|
|
258
|
+
snap2_arns = set(snap2_by_arn.keys())
|
|
259
|
+
|
|
260
|
+
added = [dict(snap2_by_arn[arn]) for arn in (snap2_arns - snap1_arns)]
|
|
261
|
+
removed = [dict(snap1_by_arn[arn]) for arn in (snap1_arns - snap2_arns)]
|
|
262
|
+
|
|
263
|
+
# Find modified (same ARN, different hash)
|
|
264
|
+
modified = []
|
|
265
|
+
for arn in snap1_arns & snap2_arns:
|
|
266
|
+
if snap1_by_arn[arn]["config_hash"] != snap2_by_arn[arn]["config_hash"]:
|
|
267
|
+
modified.append(
|
|
268
|
+
{
|
|
269
|
+
"arn": arn,
|
|
270
|
+
"resource_type": snap2_by_arn[arn]["resource_type"],
|
|
271
|
+
"name": snap2_by_arn[arn]["name"],
|
|
272
|
+
"region": snap2_by_arn[arn]["region"],
|
|
273
|
+
"old_hash": snap1_by_arn[arn]["config_hash"],
|
|
274
|
+
"new_hash": snap2_by_arn[arn]["config_hash"],
|
|
275
|
+
}
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
return {
|
|
279
|
+
"added": added,
|
|
280
|
+
"removed": removed,
|
|
281
|
+
"modified": modified,
|
|
282
|
+
"summary": {
|
|
283
|
+
"snapshot1": snapshot1_name,
|
|
284
|
+
"snapshot2": snapshot2_name,
|
|
285
|
+
"snapshot1_count": len(snap1_resources),
|
|
286
|
+
"snapshot2_count": len(snap2_resources),
|
|
287
|
+
"added_count": len(added),
|
|
288
|
+
"removed_count": len(removed),
|
|
289
|
+
"modified_count": len(modified),
|
|
290
|
+
},
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
def get_tags_for_resource(self, arn: str, snapshot_name: Optional[str] = None) -> Dict[str, str]:
|
|
294
|
+
"""Get tags for a specific resource.
|
|
295
|
+
|
|
296
|
+
Args:
|
|
297
|
+
arn: Resource ARN
|
|
298
|
+
snapshot_name: Specific snapshot (uses most recent if None)
|
|
299
|
+
|
|
300
|
+
Returns:
|
|
301
|
+
Dict of tag key-value pairs
|
|
302
|
+
"""
|
|
303
|
+
query = """
|
|
304
|
+
SELECT t.key, t.value
|
|
305
|
+
FROM resource_tags t
|
|
306
|
+
JOIN resources r ON t.resource_id = r.id
|
|
307
|
+
JOIN snapshots s ON r.snapshot_id = s.id
|
|
308
|
+
WHERE r.arn = ?
|
|
309
|
+
"""
|
|
310
|
+
params: List[str] = [arn]
|
|
311
|
+
|
|
312
|
+
if snapshot_name:
|
|
313
|
+
query += " AND s.name = ?"
|
|
314
|
+
params.append(snapshot_name)
|
|
315
|
+
else:
|
|
316
|
+
query += " ORDER BY s.created_at DESC LIMIT 100"
|
|
317
|
+
|
|
318
|
+
rows = self.db.fetchall(query, tuple(params))
|
|
319
|
+
return {row["key"]: row["value"] for row in rows}
|
|
320
|
+
|
|
321
|
+
def find_by_tag(
|
|
322
|
+
self,
|
|
323
|
+
tag_key: str,
|
|
324
|
+
tag_value: Optional[str] = None,
|
|
325
|
+
snapshot_name: Optional[str] = None,
|
|
326
|
+
limit: int = 100,
|
|
327
|
+
) -> List[Dict[str, Any]]:
|
|
328
|
+
"""Find resources by tag.
|
|
329
|
+
|
|
330
|
+
Args:
|
|
331
|
+
tag_key: Tag key to search for
|
|
332
|
+
tag_value: Optional tag value to match
|
|
333
|
+
snapshot_name: Limit to specific snapshot
|
|
334
|
+
limit: Maximum results
|
|
335
|
+
|
|
336
|
+
Returns:
|
|
337
|
+
List of matching resources
|
|
338
|
+
"""
|
|
339
|
+
query = """
|
|
340
|
+
SELECT DISTINCT
|
|
341
|
+
r.arn,
|
|
342
|
+
r.resource_type,
|
|
343
|
+
r.name,
|
|
344
|
+
r.region,
|
|
345
|
+
s.name as snapshot_name,
|
|
346
|
+
t.key as tag_key,
|
|
347
|
+
t.value as tag_value
|
|
348
|
+
FROM resources r
|
|
349
|
+
JOIN snapshots s ON r.snapshot_id = s.id
|
|
350
|
+
JOIN resource_tags t ON r.id = t.resource_id
|
|
351
|
+
WHERE t.key = ?
|
|
352
|
+
"""
|
|
353
|
+
params: List[Any] = [tag_key]
|
|
354
|
+
|
|
355
|
+
if tag_value:
|
|
356
|
+
query += " AND t.value = ?"
|
|
357
|
+
params.append(tag_value)
|
|
358
|
+
|
|
359
|
+
if snapshot_name:
|
|
360
|
+
query += " AND s.name = ?"
|
|
361
|
+
params.append(snapshot_name)
|
|
362
|
+
|
|
363
|
+
query += f" ORDER BY s.created_at DESC LIMIT {limit}"
|
|
364
|
+
|
|
365
|
+
return self.db.fetchall(query, tuple(params))
|
|
366
|
+
|
|
367
|
+
def get_unique_resource_types(self, snapshot_name: Optional[str] = None) -> List[str]:
|
|
368
|
+
"""Get list of unique resource types.
|
|
369
|
+
|
|
370
|
+
Args:
|
|
371
|
+
snapshot_name: Limit to specific snapshot
|
|
372
|
+
|
|
373
|
+
Returns:
|
|
374
|
+
List of resource type strings
|
|
375
|
+
"""
|
|
376
|
+
query = """
|
|
377
|
+
SELECT DISTINCT r.resource_type
|
|
378
|
+
FROM resources r
|
|
379
|
+
"""
|
|
380
|
+
params: List[str] = []
|
|
381
|
+
|
|
382
|
+
if snapshot_name:
|
|
383
|
+
query += " JOIN snapshots s ON r.snapshot_id = s.id WHERE s.name = ?"
|
|
384
|
+
params.append(snapshot_name)
|
|
385
|
+
|
|
386
|
+
query += " ORDER BY r.resource_type"
|
|
387
|
+
|
|
388
|
+
rows = self.db.fetchall(query, tuple(params))
|
|
389
|
+
return [row["resource_type"] for row in rows]
|
|
390
|
+
|
|
391
|
+
def get_unique_regions(self, snapshot_name: Optional[str] = None) -> List[str]:
|
|
392
|
+
"""Get list of unique regions.
|
|
393
|
+
|
|
394
|
+
Args:
|
|
395
|
+
snapshot_name: Limit to specific snapshot
|
|
396
|
+
|
|
397
|
+
Returns:
|
|
398
|
+
List of region strings
|
|
399
|
+
"""
|
|
400
|
+
query = """
|
|
401
|
+
SELECT DISTINCT r.region
|
|
402
|
+
FROM resources r
|
|
403
|
+
"""
|
|
404
|
+
params: List[str] = []
|
|
405
|
+
|
|
406
|
+
if snapshot_name:
|
|
407
|
+
query += " JOIN snapshots s ON r.snapshot_id = s.id WHERE s.name = ?"
|
|
408
|
+
params.append(snapshot_name)
|
|
409
|
+
|
|
410
|
+
query += " ORDER BY r.region"
|
|
411
|
+
|
|
412
|
+
rows = self.db.fetchall(query, tuple(params))
|
|
413
|
+
return [row["region"] for row in rows]
|
src/storage/schema.py
ADDED
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
"""SQLite schema definitions for AWS Inventory Manager."""
|
|
2
|
+
|
|
3
|
+
SCHEMA_VERSION = "1.1.0"
|
|
4
|
+
|
|
5
|
+
# Schema creation SQL
|
|
6
|
+
SCHEMA_SQL = """
|
|
7
|
+
-- Schema version tracking
|
|
8
|
+
CREATE TABLE IF NOT EXISTS schema_info (
|
|
9
|
+
key TEXT PRIMARY KEY,
|
|
10
|
+
value TEXT NOT NULL
|
|
11
|
+
);
|
|
12
|
+
|
|
13
|
+
-- Core snapshots table
|
|
14
|
+
CREATE TABLE IF NOT EXISTS snapshots (
|
|
15
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
16
|
+
name TEXT UNIQUE NOT NULL,
|
|
17
|
+
created_at TIMESTAMP NOT NULL,
|
|
18
|
+
account_id TEXT NOT NULL,
|
|
19
|
+
regions TEXT NOT NULL,
|
|
20
|
+
resource_count INTEGER DEFAULT 0,
|
|
21
|
+
total_resources_before_filter INTEGER,
|
|
22
|
+
service_counts TEXT,
|
|
23
|
+
metadata TEXT,
|
|
24
|
+
filters_applied TEXT,
|
|
25
|
+
schema_version TEXT DEFAULT '1.1',
|
|
26
|
+
inventory_name TEXT DEFAULT 'default',
|
|
27
|
+
is_active BOOLEAN DEFAULT 0
|
|
28
|
+
);
|
|
29
|
+
|
|
30
|
+
-- Resources table
|
|
31
|
+
CREATE TABLE IF NOT EXISTS resources (
|
|
32
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
33
|
+
snapshot_id INTEGER NOT NULL,
|
|
34
|
+
arn TEXT NOT NULL,
|
|
35
|
+
resource_type TEXT NOT NULL,
|
|
36
|
+
name TEXT NOT NULL,
|
|
37
|
+
region TEXT NOT NULL,
|
|
38
|
+
config_hash TEXT NOT NULL,
|
|
39
|
+
raw_config TEXT,
|
|
40
|
+
created_at TIMESTAMP,
|
|
41
|
+
source TEXT DEFAULT 'direct_api',
|
|
42
|
+
canonical_name TEXT,
|
|
43
|
+
FOREIGN KEY (snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE,
|
|
44
|
+
UNIQUE(snapshot_id, arn)
|
|
45
|
+
);
|
|
46
|
+
|
|
47
|
+
-- Normalized tags for efficient querying
|
|
48
|
+
CREATE TABLE IF NOT EXISTS resource_tags (
|
|
49
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
50
|
+
resource_id INTEGER NOT NULL,
|
|
51
|
+
key TEXT NOT NULL,
|
|
52
|
+
value TEXT NOT NULL,
|
|
53
|
+
FOREIGN KEY (resource_id) REFERENCES resources(id) ON DELETE CASCADE
|
|
54
|
+
);
|
|
55
|
+
|
|
56
|
+
-- Inventories table
|
|
57
|
+
CREATE TABLE IF NOT EXISTS inventories (
|
|
58
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
59
|
+
name TEXT NOT NULL,
|
|
60
|
+
account_id TEXT NOT NULL,
|
|
61
|
+
description TEXT DEFAULT '',
|
|
62
|
+
include_tags TEXT,
|
|
63
|
+
exclude_tags TEXT,
|
|
64
|
+
active_snapshot_id INTEGER,
|
|
65
|
+
created_at TIMESTAMP NOT NULL,
|
|
66
|
+
last_updated TIMESTAMP NOT NULL,
|
|
67
|
+
FOREIGN KEY (active_snapshot_id) REFERENCES snapshots(id) ON DELETE SET NULL,
|
|
68
|
+
UNIQUE(name, account_id)
|
|
69
|
+
);
|
|
70
|
+
|
|
71
|
+
-- Link table for inventory snapshots (many-to-many)
|
|
72
|
+
CREATE TABLE IF NOT EXISTS inventory_snapshots (
|
|
73
|
+
inventory_id INTEGER NOT NULL,
|
|
74
|
+
snapshot_id INTEGER NOT NULL,
|
|
75
|
+
PRIMARY KEY (inventory_id, snapshot_id),
|
|
76
|
+
FOREIGN KEY (inventory_id) REFERENCES inventories(id) ON DELETE CASCADE,
|
|
77
|
+
FOREIGN KEY (snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE
|
|
78
|
+
);
|
|
79
|
+
|
|
80
|
+
-- Audit operations table
|
|
81
|
+
CREATE TABLE IF NOT EXISTS audit_operations (
|
|
82
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
83
|
+
operation_id TEXT UNIQUE NOT NULL,
|
|
84
|
+
baseline_snapshot TEXT NOT NULL,
|
|
85
|
+
timestamp TIMESTAMP NOT NULL,
|
|
86
|
+
aws_profile TEXT,
|
|
87
|
+
account_id TEXT NOT NULL,
|
|
88
|
+
mode TEXT NOT NULL,
|
|
89
|
+
status TEXT NOT NULL,
|
|
90
|
+
total_resources INTEGER,
|
|
91
|
+
succeeded_count INTEGER,
|
|
92
|
+
failed_count INTEGER,
|
|
93
|
+
skipped_count INTEGER,
|
|
94
|
+
duration_seconds REAL,
|
|
95
|
+
filters TEXT
|
|
96
|
+
);
|
|
97
|
+
|
|
98
|
+
-- Audit records table
|
|
99
|
+
CREATE TABLE IF NOT EXISTS audit_records (
|
|
100
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
101
|
+
operation_id TEXT NOT NULL,
|
|
102
|
+
resource_arn TEXT NOT NULL,
|
|
103
|
+
resource_id TEXT,
|
|
104
|
+
resource_type TEXT NOT NULL,
|
|
105
|
+
region TEXT NOT NULL,
|
|
106
|
+
status TEXT NOT NULL,
|
|
107
|
+
error_code TEXT,
|
|
108
|
+
error_message TEXT,
|
|
109
|
+
protection_reason TEXT,
|
|
110
|
+
deletion_tier TEXT,
|
|
111
|
+
tags TEXT,
|
|
112
|
+
estimated_monthly_cost REAL,
|
|
113
|
+
FOREIGN KEY (operation_id) REFERENCES audit_operations(operation_id) ON DELETE CASCADE
|
|
114
|
+
);
|
|
115
|
+
|
|
116
|
+
-- Saved queries table (for web UI)
|
|
117
|
+
CREATE TABLE IF NOT EXISTS saved_queries (
|
|
118
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
119
|
+
name TEXT UNIQUE NOT NULL,
|
|
120
|
+
description TEXT,
|
|
121
|
+
sql_text TEXT NOT NULL,
|
|
122
|
+
category TEXT DEFAULT 'custom',
|
|
123
|
+
is_favorite BOOLEAN DEFAULT 0,
|
|
124
|
+
created_at TIMESTAMP NOT NULL,
|
|
125
|
+
last_run_at TIMESTAMP,
|
|
126
|
+
run_count INTEGER DEFAULT 0
|
|
127
|
+
);
|
|
128
|
+
|
|
129
|
+
-- Saved filters table (for resource explorer)
|
|
130
|
+
CREATE TABLE IF NOT EXISTS saved_filters (
|
|
131
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
132
|
+
name TEXT UNIQUE NOT NULL,
|
|
133
|
+
description TEXT,
|
|
134
|
+
filter_config TEXT NOT NULL,
|
|
135
|
+
is_favorite BOOLEAN DEFAULT 0,
|
|
136
|
+
created_at TIMESTAMP NOT NULL,
|
|
137
|
+
last_used_at TIMESTAMP,
|
|
138
|
+
use_count INTEGER DEFAULT 0
|
|
139
|
+
);
|
|
140
|
+
|
|
141
|
+
-- Saved views table (for customizable resource views)
|
|
142
|
+
CREATE TABLE IF NOT EXISTS saved_views (
|
|
143
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
144
|
+
name TEXT UNIQUE NOT NULL,
|
|
145
|
+
description TEXT,
|
|
146
|
+
view_config TEXT NOT NULL,
|
|
147
|
+
is_default BOOLEAN DEFAULT 0,
|
|
148
|
+
is_favorite BOOLEAN DEFAULT 0,
|
|
149
|
+
created_at TIMESTAMP NOT NULL,
|
|
150
|
+
last_used_at TIMESTAMP,
|
|
151
|
+
use_count INTEGER DEFAULT 0
|
|
152
|
+
);
|
|
153
|
+
|
|
154
|
+
-- Resource groups table (for baseline comparison)
|
|
155
|
+
CREATE TABLE IF NOT EXISTS resource_groups (
|
|
156
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
157
|
+
name TEXT UNIQUE NOT NULL,
|
|
158
|
+
description TEXT,
|
|
159
|
+
source_snapshot TEXT,
|
|
160
|
+
resource_count INTEGER DEFAULT 0,
|
|
161
|
+
is_favorite BOOLEAN DEFAULT 0,
|
|
162
|
+
created_at TIMESTAMP NOT NULL,
|
|
163
|
+
last_updated TIMESTAMP NOT NULL
|
|
164
|
+
);
|
|
165
|
+
|
|
166
|
+
-- Resource group members table (normalized for efficient querying)
|
|
167
|
+
CREATE TABLE IF NOT EXISTS resource_group_members (
|
|
168
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
169
|
+
group_id INTEGER NOT NULL,
|
|
170
|
+
resource_name TEXT NOT NULL,
|
|
171
|
+
resource_type TEXT NOT NULL,
|
|
172
|
+
original_arn TEXT,
|
|
173
|
+
match_strategy TEXT DEFAULT 'physical_name',
|
|
174
|
+
FOREIGN KEY (group_id) REFERENCES resource_groups(id) ON DELETE CASCADE,
|
|
175
|
+
UNIQUE (group_id, resource_name, resource_type)
|
|
176
|
+
);
|
|
177
|
+
"""
|
|
178
|
+
|
|
179
|
+
# Indexes for common queries (created separately for better error handling)
|
|
180
|
+
# SQLite performance tips applied:
|
|
181
|
+
# - Indexes on foreign keys for faster JOINs
|
|
182
|
+
# - Composite indexes for common query patterns
|
|
183
|
+
# - Covering indexes where possible
|
|
184
|
+
INDEXES_SQL = """
|
|
185
|
+
-- Resources indexes
|
|
186
|
+
CREATE INDEX IF NOT EXISTS idx_resources_arn ON resources(arn);
|
|
187
|
+
CREATE INDEX IF NOT EXISTS idx_resources_type ON resources(resource_type);
|
|
188
|
+
CREATE INDEX IF NOT EXISTS idx_resources_region ON resources(region);
|
|
189
|
+
CREATE INDEX IF NOT EXISTS idx_resources_created ON resources(created_at);
|
|
190
|
+
CREATE INDEX IF NOT EXISTS idx_resources_snapshot ON resources(snapshot_id);
|
|
191
|
+
CREATE INDEX IF NOT EXISTS idx_resources_type_region ON resources(resource_type, region);
|
|
192
|
+
CREATE INDEX IF NOT EXISTS idx_resources_canonical_name_type ON resources(canonical_name, resource_type);
|
|
193
|
+
|
|
194
|
+
-- Tags indexes (for efficient tag queries)
|
|
195
|
+
CREATE INDEX IF NOT EXISTS idx_tags_resource ON resource_tags(resource_id);
|
|
196
|
+
CREATE INDEX IF NOT EXISTS idx_tags_key ON resource_tags(key);
|
|
197
|
+
CREATE INDEX IF NOT EXISTS idx_tags_value ON resource_tags(value);
|
|
198
|
+
CREATE INDEX IF NOT EXISTS idx_tags_kv ON resource_tags(key, value);
|
|
199
|
+
|
|
200
|
+
-- Snapshots indexes
|
|
201
|
+
CREATE INDEX IF NOT EXISTS idx_snapshots_account ON snapshots(account_id);
|
|
202
|
+
CREATE INDEX IF NOT EXISTS idx_snapshots_created ON snapshots(created_at);
|
|
203
|
+
CREATE INDEX IF NOT EXISTS idx_snapshots_name ON snapshots(name);
|
|
204
|
+
CREATE INDEX IF NOT EXISTS idx_snapshots_account_created ON snapshots(account_id, created_at DESC);
|
|
205
|
+
|
|
206
|
+
-- Inventories indexes
|
|
207
|
+
CREATE INDEX IF NOT EXISTS idx_inventories_account ON inventories(account_id);
|
|
208
|
+
CREATE INDEX IF NOT EXISTS idx_inventories_name_account ON inventories(name, account_id);
|
|
209
|
+
|
|
210
|
+
-- Audit indexes (for history queries and filtering)
|
|
211
|
+
CREATE INDEX IF NOT EXISTS idx_audit_ops_timestamp ON audit_operations(timestamp DESC);
|
|
212
|
+
CREATE INDEX IF NOT EXISTS idx_audit_ops_account ON audit_operations(account_id);
|
|
213
|
+
CREATE INDEX IF NOT EXISTS idx_audit_ops_account_timestamp ON audit_operations(account_id, timestamp DESC);
|
|
214
|
+
CREATE INDEX IF NOT EXISTS idx_audit_records_operation ON audit_records(operation_id);
|
|
215
|
+
CREATE INDEX IF NOT EXISTS idx_audit_records_arn ON audit_records(resource_arn);
|
|
216
|
+
CREATE INDEX IF NOT EXISTS idx_audit_records_type ON audit_records(resource_type);
|
|
217
|
+
CREATE INDEX IF NOT EXISTS idx_audit_records_region ON audit_records(region);
|
|
218
|
+
CREATE INDEX IF NOT EXISTS idx_audit_records_status ON audit_records(status);
|
|
219
|
+
|
|
220
|
+
-- Saved queries indexes
|
|
221
|
+
CREATE INDEX IF NOT EXISTS idx_queries_category ON saved_queries(category);
|
|
222
|
+
CREATE INDEX IF NOT EXISTS idx_queries_favorite ON saved_queries(is_favorite);
|
|
223
|
+
CREATE INDEX IF NOT EXISTS idx_queries_last_run ON saved_queries(last_run_at DESC);
|
|
224
|
+
|
|
225
|
+
-- Saved filters indexes
|
|
226
|
+
CREATE INDEX IF NOT EXISTS idx_filters_favorite ON saved_filters(is_favorite);
|
|
227
|
+
CREATE INDEX IF NOT EXISTS idx_filters_last_used ON saved_filters(last_used_at DESC);
|
|
228
|
+
|
|
229
|
+
-- Saved views indexes
|
|
230
|
+
CREATE INDEX IF NOT EXISTS idx_views_default ON saved_views(is_default);
|
|
231
|
+
CREATE INDEX IF NOT EXISTS idx_views_favorite ON saved_views(is_favorite);
|
|
232
|
+
CREATE INDEX IF NOT EXISTS idx_views_last_used ON saved_views(last_used_at DESC);
|
|
233
|
+
|
|
234
|
+
-- Resource groups indexes
|
|
235
|
+
CREATE INDEX IF NOT EXISTS idx_groups_name ON resource_groups(name);
|
|
236
|
+
CREATE INDEX IF NOT EXISTS idx_groups_favorite ON resource_groups(is_favorite);
|
|
237
|
+
CREATE INDEX IF NOT EXISTS idx_groups_created ON resource_groups(created_at DESC);
|
|
238
|
+
|
|
239
|
+
-- Resource group members indexes
|
|
240
|
+
CREATE INDEX IF NOT EXISTS idx_group_members_group ON resource_group_members(group_id);
|
|
241
|
+
CREATE INDEX IF NOT EXISTS idx_group_members_name_type ON resource_group_members(resource_name, resource_type);
|
|
242
|
+
CREATE INDEX IF NOT EXISTS idx_group_members_strategy ON resource_group_members(match_strategy);
|
|
243
|
+
"""
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
MIGRATIONS = {
|
|
247
|
+
"1.1.0": [
|
|
248
|
+
# Add canonical_name column to resources table
|
|
249
|
+
"ALTER TABLE resources ADD COLUMN canonical_name TEXT",
|
|
250
|
+
# Add match_strategy column to resource_group_members table
|
|
251
|
+
"ALTER TABLE resource_group_members ADD COLUMN match_strategy TEXT DEFAULT 'physical_name'",
|
|
252
|
+
# Backfill canonical_name from CloudFormation logical-id tag
|
|
253
|
+
"""
|
|
254
|
+
UPDATE resources
|
|
255
|
+
SET canonical_name = (
|
|
256
|
+
SELECT value FROM resource_tags
|
|
257
|
+
WHERE resource_tags.resource_id = resources.id
|
|
258
|
+
AND key = 'aws:cloudformation:logical-id'
|
|
259
|
+
)
|
|
260
|
+
WHERE canonical_name IS NULL
|
|
261
|
+
""",
|
|
262
|
+
# Fallback to physical name for resources without CloudFormation tag
|
|
263
|
+
"""
|
|
264
|
+
UPDATE resources
|
|
265
|
+
SET canonical_name = COALESCE(name, arn)
|
|
266
|
+
WHERE canonical_name IS NULL
|
|
267
|
+
""",
|
|
268
|
+
],
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def get_schema_sql() -> str:
|
|
273
|
+
"""Get the full schema SQL."""
|
|
274
|
+
return SCHEMA_SQL
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def get_indexes_sql() -> str:
|
|
278
|
+
"""Get the indexes SQL."""
|
|
279
|
+
return INDEXES_SQL
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def get_migrations() -> dict:
|
|
283
|
+
"""Get the migrations dictionary.
|
|
284
|
+
|
|
285
|
+
Returns:
|
|
286
|
+
Dict mapping version strings to lists of SQL statements
|
|
287
|
+
"""
|
|
288
|
+
return MIGRATIONS
|