runbooks 0.7.6__py3-none-any.whl โ 0.7.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runbooks/__init__.py +1 -1
- runbooks/base.py +5 -1
- runbooks/cfat/__init__.py +8 -4
- runbooks/cfat/assessment/collectors.py +171 -14
- runbooks/cfat/assessment/compliance.py +871 -0
- runbooks/cfat/assessment/runner.py +122 -11
- runbooks/cfat/models.py +6 -2
- runbooks/common/logger.py +14 -0
- runbooks/common/rich_utils.py +451 -0
- runbooks/enterprise/__init__.py +68 -0
- runbooks/enterprise/error_handling.py +411 -0
- runbooks/enterprise/logging.py +439 -0
- runbooks/enterprise/multi_tenant.py +583 -0
- runbooks/finops/README.md +468 -241
- runbooks/finops/__init__.py +39 -3
- runbooks/finops/cli.py +83 -18
- runbooks/finops/cross_validation.py +375 -0
- runbooks/finops/dashboard_runner.py +812 -164
- runbooks/finops/enhanced_dashboard_runner.py +525 -0
- runbooks/finops/finops_dashboard.py +1892 -0
- runbooks/finops/helpers.py +485 -51
- runbooks/finops/optimizer.py +823 -0
- runbooks/finops/tests/__init__.py +19 -0
- runbooks/finops/tests/results_test_finops_dashboard.xml +1 -0
- runbooks/finops/tests/run_comprehensive_tests.py +421 -0
- runbooks/finops/tests/run_tests.py +305 -0
- runbooks/finops/tests/test_finops_dashboard.py +705 -0
- runbooks/finops/tests/test_integration.py +477 -0
- runbooks/finops/tests/test_performance.py +380 -0
- runbooks/finops/tests/test_performance_benchmarks.py +500 -0
- runbooks/finops/tests/test_reference_images_validation.py +867 -0
- runbooks/finops/tests/test_single_account_features.py +715 -0
- runbooks/finops/tests/validate_test_suite.py +220 -0
- runbooks/finops/types.py +1 -1
- runbooks/hitl/enhanced_workflow_engine.py +725 -0
- runbooks/inventory/artifacts/scale-optimize-status.txt +12 -0
- runbooks/inventory/collectors/aws_comprehensive.py +442 -0
- runbooks/inventory/collectors/enterprise_scale.py +281 -0
- runbooks/inventory/core/collector.py +172 -13
- runbooks/inventory/discovery.md +1 -1
- runbooks/inventory/list_ec2_instances.py +18 -20
- runbooks/inventory/list_ssm_parameters.py +31 -3
- runbooks/inventory/organizations_discovery.py +1269 -0
- runbooks/inventory/rich_inventory_display.py +393 -0
- runbooks/inventory/run_on_multi_accounts.py +35 -19
- runbooks/inventory/runbooks.security.report_generator.log +0 -0
- runbooks/inventory/runbooks.security.run_script.log +0 -0
- runbooks/inventory/vpc_flow_analyzer.py +1030 -0
- runbooks/main.py +2215 -119
- runbooks/metrics/dora_metrics_engine.py +599 -0
- runbooks/operate/__init__.py +2 -2
- runbooks/operate/base.py +122 -10
- runbooks/operate/deployment_framework.py +1032 -0
- runbooks/operate/deployment_validator.py +853 -0
- runbooks/operate/dynamodb_operations.py +10 -6
- runbooks/operate/ec2_operations.py +319 -11
- runbooks/operate/executive_dashboard.py +779 -0
- runbooks/operate/mcp_integration.py +750 -0
- runbooks/operate/nat_gateway_operations.py +1120 -0
- runbooks/operate/networking_cost_heatmap.py +685 -0
- runbooks/operate/privatelink_operations.py +940 -0
- runbooks/operate/s3_operations.py +10 -6
- runbooks/operate/vpc_endpoints.py +644 -0
- runbooks/operate/vpc_operations.py +1038 -0
- runbooks/remediation/__init__.py +2 -2
- runbooks/remediation/acm_remediation.py +1 -1
- runbooks/remediation/base.py +1 -1
- runbooks/remediation/cloudtrail_remediation.py +1 -1
- runbooks/remediation/cognito_remediation.py +1 -1
- runbooks/remediation/dynamodb_remediation.py +1 -1
- runbooks/remediation/ec2_remediation.py +1 -1
- runbooks/remediation/ec2_unattached_ebs_volumes.py +1 -1
- runbooks/remediation/kms_enable_key_rotation.py +1 -1
- runbooks/remediation/kms_remediation.py +1 -1
- runbooks/remediation/lambda_remediation.py +1 -1
- runbooks/remediation/multi_account.py +1 -1
- runbooks/remediation/rds_remediation.py +1 -1
- runbooks/remediation/s3_block_public_access.py +1 -1
- runbooks/remediation/s3_enable_access_logging.py +1 -1
- runbooks/remediation/s3_encryption.py +1 -1
- runbooks/remediation/s3_remediation.py +1 -1
- runbooks/remediation/vpc_remediation.py +475 -0
- runbooks/security/__init__.py +3 -1
- runbooks/security/compliance_automation.py +632 -0
- runbooks/security/report_generator.py +10 -0
- runbooks/security/run_script.py +31 -5
- runbooks/security/security_baseline_tester.py +169 -30
- runbooks/security/security_export.py +477 -0
- runbooks/validation/__init__.py +10 -0
- runbooks/validation/benchmark.py +484 -0
- runbooks/validation/cli.py +356 -0
- runbooks/validation/mcp_validator.py +768 -0
- runbooks/vpc/__init__.py +38 -0
- runbooks/vpc/config.py +212 -0
- runbooks/vpc/cost_engine.py +347 -0
- runbooks/vpc/heatmap_engine.py +605 -0
- runbooks/vpc/manager_interface.py +634 -0
- runbooks/vpc/networking_wrapper.py +1260 -0
- runbooks/vpc/rich_formatters.py +679 -0
- runbooks/vpc/tests/__init__.py +5 -0
- runbooks/vpc/tests/conftest.py +356 -0
- runbooks/vpc/tests/test_cli_integration.py +530 -0
- runbooks/vpc/tests/test_config.py +458 -0
- runbooks/vpc/tests/test_cost_engine.py +479 -0
- runbooks/vpc/tests/test_networking_wrapper.py +512 -0
- {runbooks-0.7.6.dist-info โ runbooks-0.7.9.dist-info}/METADATA +40 -12
- {runbooks-0.7.6.dist-info โ runbooks-0.7.9.dist-info}/RECORD +111 -50
- {runbooks-0.7.6.dist-info โ runbooks-0.7.9.dist-info}/WHEEL +0 -0
- {runbooks-0.7.6.dist-info โ runbooks-0.7.9.dist-info}/entry_points.txt +0 -0
- {runbooks-0.7.6.dist-info โ runbooks-0.7.9.dist-info}/licenses/LICENSE +0 -0
- {runbooks-0.7.6.dist-info โ runbooks-0.7.9.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,725 @@
|
|
1
|
+
#!/usr/bin/env python3
|
2
|
+
"""
|
3
|
+
Enhanced HITL Workflow Engine for Human-in-the-Loop Optimization
|
4
|
+
|
5
|
+
Issue #93: HITL System & DORA Metrics Optimization
|
6
|
+
Priority: High (Sprint 1 Improvements)
|
7
|
+
Scope: Streamlined approval workflows with ยฑ15% cross-validation accuracy
|
8
|
+
"""
|
9
|
+
|
10
|
+
import asyncio
|
11
|
+
import json
|
12
|
+
import logging
|
13
|
+
from dataclasses import asdict, dataclass
|
14
|
+
from datetime import datetime, timedelta, timezone
|
15
|
+
from enum import Enum
|
16
|
+
from pathlib import Path
|
17
|
+
from typing import Any, Callable, Dict, List, Optional
|
18
|
+
|
19
|
+
from ..metrics.dora_metrics_engine import DORAMetricsEngine
|
20
|
+
from ..utils.logger import configure_logger
|
21
|
+
|
22
|
+
logger = configure_logger(__name__)
|
23
|
+
|
24
|
+
|
25
|
+
class ApprovalStatus(Enum):
|
26
|
+
PENDING = "pending"
|
27
|
+
APPROVED = "approved"
|
28
|
+
REJECTED = "rejected"
|
29
|
+
ESCALATED = "escalated"
|
30
|
+
TIMEOUT = "timeout"
|
31
|
+
|
32
|
+
|
33
|
+
class WorkflowStep(Enum):
|
34
|
+
VALIDATION = "validation"
|
35
|
+
ARCHITECTURE_REVIEW = "architecture_review"
|
36
|
+
SECURITY_REVIEW = "security_review"
|
37
|
+
COST_ANALYSIS = "cost_analysis"
|
38
|
+
MANAGER_APPROVAL = "manager_approval"
|
39
|
+
DEPLOYMENT = "deployment"
|
40
|
+
|
41
|
+
|
42
|
+
@dataclass
|
43
|
+
class ApprovalRequest:
|
44
|
+
"""Approval request for HITL workflow"""
|
45
|
+
|
46
|
+
request_id: str
|
47
|
+
request_type: str # deployment, architecture, security, cost
|
48
|
+
title: str
|
49
|
+
description: str
|
50
|
+
requester: str
|
51
|
+
created_at: datetime
|
52
|
+
priority: str = "medium" # low, medium, high, critical
|
53
|
+
metadata: Dict = None
|
54
|
+
|
55
|
+
def __post_init__(self):
|
56
|
+
if self.metadata is None:
|
57
|
+
self.metadata = {}
|
58
|
+
|
59
|
+
|
60
|
+
@dataclass
|
61
|
+
class ApprovalDecision:
|
62
|
+
"""Approval decision record"""
|
63
|
+
|
64
|
+
request_id: str
|
65
|
+
approver: str
|
66
|
+
status: ApprovalStatus
|
67
|
+
decision_time: datetime
|
68
|
+
reason: str = ""
|
69
|
+
additional_data: Dict = None
|
70
|
+
|
71
|
+
def __post_init__(self):
|
72
|
+
if self.additional_data is None:
|
73
|
+
self.additional_data = {}
|
74
|
+
|
75
|
+
|
76
|
+
@dataclass
|
77
|
+
class WorkflowExecution:
|
78
|
+
"""Workflow execution tracking"""
|
79
|
+
|
80
|
+
workflow_id: str
|
81
|
+
request_id: str
|
82
|
+
steps: List[WorkflowStep]
|
83
|
+
current_step: int
|
84
|
+
status: str
|
85
|
+
start_time: datetime
|
86
|
+
end_time: Optional[datetime] = None
|
87
|
+
step_durations: Dict[str, float] = None
|
88
|
+
|
89
|
+
def __post_init__(self):
|
90
|
+
if self.step_durations is None:
|
91
|
+
self.step_durations = {}
|
92
|
+
|
93
|
+
|
94
|
+
class EnhancedHITLWorkflowEngine:
|
95
|
+
"""Enhanced Human-in-the-Loop workflow engine with optimization"""
|
96
|
+
|
97
|
+
def __init__(
|
98
|
+
self,
|
99
|
+
dora_engine: DORAMetricsEngine,
|
100
|
+
artifacts_dir: str = "./artifacts/hitl",
|
101
|
+
cross_validation_tolerance: float = 15.0,
|
102
|
+
):
|
103
|
+
"""
|
104
|
+
Initialize Enhanced HITL Workflow Engine
|
105
|
+
|
106
|
+
Args:
|
107
|
+
dora_engine: DORA metrics engine for performance tracking
|
108
|
+
artifacts_dir: Directory for HITL artifacts
|
109
|
+
cross_validation_tolerance: Cross-validation tolerance percentage
|
110
|
+
"""
|
111
|
+
self.dora_engine = dora_engine
|
112
|
+
self.artifacts_dir = Path(artifacts_dir)
|
113
|
+
self.artifacts_dir.mkdir(parents=True, exist_ok=True)
|
114
|
+
|
115
|
+
self.tolerance = cross_validation_tolerance
|
116
|
+
|
117
|
+
# Workflow data
|
118
|
+
self.approval_requests: Dict[str, ApprovalRequest] = {}
|
119
|
+
self.approval_decisions: Dict[str, List[ApprovalDecision]] = {}
|
120
|
+
self.workflow_executions: Dict[str, WorkflowExecution] = {}
|
121
|
+
|
122
|
+
# Approval matrix from CLAUDE.md
|
123
|
+
self.approval_matrix = {
|
124
|
+
"production_changes": {
|
125
|
+
"required": True,
|
126
|
+
"approvers": ["management"],
|
127
|
+
"timeout_minutes": 30,
|
128
|
+
"escalation": "slack",
|
129
|
+
},
|
130
|
+
"cost_impact_high": { # >$1000
|
131
|
+
"required": True,
|
132
|
+
"approvers": ["management", "finops"],
|
133
|
+
"timeout_minutes": 45,
|
134
|
+
"analysis_required": True,
|
135
|
+
},
|
136
|
+
"security_changes": {
|
137
|
+
"required": True,
|
138
|
+
"compliance": ["SOC2", "PCI-DSS", "HIPAA"],
|
139
|
+
"approvers": ["management", "security"],
|
140
|
+
"evidence_required": True,
|
141
|
+
"timeout_minutes": 60,
|
142
|
+
},
|
143
|
+
"architecture_changes": {
|
144
|
+
"required": True,
|
145
|
+
"scope": ["multi-account", "cross-region"],
|
146
|
+
"approvers": ["management", "architect"],
|
147
|
+
"documentation_required": True,
|
148
|
+
"timeout_minutes": 45,
|
149
|
+
},
|
150
|
+
}
|
151
|
+
|
152
|
+
# Performance optimization settings
|
153
|
+
self.optimization_config = {
|
154
|
+
"parallel_approvals": True,
|
155
|
+
"auto_escalation": True,
|
156
|
+
"smart_routing": True,
|
157
|
+
"cross_validation": True,
|
158
|
+
"bottleneck_detection": True,
|
159
|
+
}
|
160
|
+
|
161
|
+
# Workflow templates
|
162
|
+
self.workflow_templates = {
|
163
|
+
"deployment_approval": [
|
164
|
+
WorkflowStep.VALIDATION,
|
165
|
+
WorkflowStep.ARCHITECTURE_REVIEW,
|
166
|
+
WorkflowStep.SECURITY_REVIEW,
|
167
|
+
WorkflowStep.COST_ANALYSIS,
|
168
|
+
WorkflowStep.MANAGER_APPROVAL,
|
169
|
+
WorkflowStep.DEPLOYMENT,
|
170
|
+
],
|
171
|
+
"architecture_review": [
|
172
|
+
WorkflowStep.VALIDATION,
|
173
|
+
WorkflowStep.ARCHITECTURE_REVIEW,
|
174
|
+
WorkflowStep.SECURITY_REVIEW,
|
175
|
+
WorkflowStep.MANAGER_APPROVAL,
|
176
|
+
],
|
177
|
+
"cost_optimization": [WorkflowStep.VALIDATION, WorkflowStep.COST_ANALYSIS, WorkflowStep.MANAGER_APPROVAL],
|
178
|
+
}
|
179
|
+
|
180
|
+
async def submit_approval_request(
|
181
|
+
self,
|
182
|
+
request_type: str,
|
183
|
+
title: str,
|
184
|
+
description: str,
|
185
|
+
requester: str,
|
186
|
+
priority: str = "medium",
|
187
|
+
metadata: Dict = None,
|
188
|
+
) -> str:
|
189
|
+
"""Submit new approval request"""
|
190
|
+
|
191
|
+
request_id = f"{request_type}_{int(datetime.now().timestamp())}"
|
192
|
+
|
193
|
+
request = ApprovalRequest(
|
194
|
+
request_id=request_id,
|
195
|
+
request_type=request_type,
|
196
|
+
title=title,
|
197
|
+
description=description,
|
198
|
+
requester=requester,
|
199
|
+
created_at=datetime.now(timezone.utc),
|
200
|
+
priority=priority,
|
201
|
+
metadata=metadata or {},
|
202
|
+
)
|
203
|
+
|
204
|
+
self.approval_requests[request_id] = request
|
205
|
+
|
206
|
+
# Start workflow execution
|
207
|
+
await self._start_workflow(request)
|
208
|
+
|
209
|
+
logger.info(f"๐ Approval request submitted: {request_id} - {title}")
|
210
|
+
|
211
|
+
return request_id
|
212
|
+
|
213
|
+
async def _start_workflow(self, request: ApprovalRequest):
|
214
|
+
"""Start workflow execution for approval request"""
|
215
|
+
|
216
|
+
# Determine workflow template
|
217
|
+
if request.request_type in self.workflow_templates:
|
218
|
+
steps = self.workflow_templates[request.request_type]
|
219
|
+
else:
|
220
|
+
steps = self.workflow_templates["deployment_approval"] # Default
|
221
|
+
|
222
|
+
workflow_id = f"wf_{request.request_id}"
|
223
|
+
|
224
|
+
execution = WorkflowExecution(
|
225
|
+
workflow_id=workflow_id,
|
226
|
+
request_id=request.request_id,
|
227
|
+
steps=steps,
|
228
|
+
current_step=0,
|
229
|
+
status="running",
|
230
|
+
start_time=datetime.now(timezone.utc),
|
231
|
+
)
|
232
|
+
|
233
|
+
self.workflow_executions[workflow_id] = execution
|
234
|
+
|
235
|
+
# Start first step
|
236
|
+
await self._execute_workflow_step(execution)
|
237
|
+
|
238
|
+
async def _execute_workflow_step(self, execution: WorkflowExecution):
|
239
|
+
"""Execute current workflow step"""
|
240
|
+
|
241
|
+
if execution.current_step >= len(execution.steps):
|
242
|
+
# Workflow completed
|
243
|
+
execution.status = "completed"
|
244
|
+
execution.end_time = datetime.now(timezone.utc)
|
245
|
+
|
246
|
+
# Record workflow completion time
|
247
|
+
total_duration = (execution.end_time - execution.start_time).total_seconds() / 60 # minutes
|
248
|
+
self.dora_engine.record_approval_time(total_duration, "complete_workflow")
|
249
|
+
|
250
|
+
logger.info(f"โ
Workflow completed: {execution.workflow_id}")
|
251
|
+
return
|
252
|
+
|
253
|
+
current_step = execution.steps[execution.current_step]
|
254
|
+
step_start = datetime.now(timezone.utc)
|
255
|
+
|
256
|
+
logger.info(f"๐ Executing step: {current_step.value} for {execution.workflow_id}")
|
257
|
+
|
258
|
+
# Execute step based on type
|
259
|
+
step_result = await self._execute_step_logic(current_step, execution)
|
260
|
+
|
261
|
+
step_duration = (datetime.now(timezone.utc) - step_start).total_seconds() / 60 # minutes
|
262
|
+
execution.step_durations[current_step.value] = step_duration
|
263
|
+
|
264
|
+
# Record step timing
|
265
|
+
self.dora_engine.record_approval_time(step_duration, current_step.value)
|
266
|
+
|
267
|
+
if step_result["success"]:
|
268
|
+
# Move to next step
|
269
|
+
execution.current_step += 1
|
270
|
+
await self._execute_workflow_step(execution)
|
271
|
+
else:
|
272
|
+
# Handle step failure
|
273
|
+
execution.status = "failed"
|
274
|
+
execution.end_time = datetime.now(timezone.utc)
|
275
|
+
logger.error(f"โ Workflow step failed: {current_step.value} - {step_result.get('error', 'Unknown error')}")
|
276
|
+
|
277
|
+
async def _execute_step_logic(self, step: WorkflowStep, execution: WorkflowExecution) -> Dict:
|
278
|
+
"""Execute specific step logic"""
|
279
|
+
|
280
|
+
request = self.approval_requests[execution.request_id]
|
281
|
+
|
282
|
+
if step == WorkflowStep.VALIDATION:
|
283
|
+
return await self._validate_request(request)
|
284
|
+
elif step == WorkflowStep.ARCHITECTURE_REVIEW:
|
285
|
+
return await self._architecture_review(request)
|
286
|
+
elif step == WorkflowStep.SECURITY_REVIEW:
|
287
|
+
return await self._security_review(request)
|
288
|
+
elif step == WorkflowStep.COST_ANALYSIS:
|
289
|
+
return await self._cost_analysis(request)
|
290
|
+
elif step == WorkflowStep.MANAGER_APPROVAL:
|
291
|
+
return await self._manager_approval(request)
|
292
|
+
elif step == WorkflowStep.DEPLOYMENT:
|
293
|
+
return await self._execute_deployment(request)
|
294
|
+
else:
|
295
|
+
return {"success": False, "error": f"Unknown step: {step}"}
|
296
|
+
|
297
|
+
async def _validate_request(self, request: ApprovalRequest) -> Dict:
|
298
|
+
"""Validate approval request"""
|
299
|
+
|
300
|
+
# Basic validation
|
301
|
+
if not request.title or not request.description:
|
302
|
+
return {"success": False, "error": "Missing required fields"}
|
303
|
+
|
304
|
+
# Cross-validation if enabled
|
305
|
+
if self.optimization_config["cross_validation"]:
|
306
|
+
validation_result = await self._cross_validate_request(request)
|
307
|
+
if not validation_result["valid"]:
|
308
|
+
return {"success": False, "error": f"Cross-validation failed: {validation_result['error']}"}
|
309
|
+
|
310
|
+
logger.info(f"โ
Request validated: {request.request_id}")
|
311
|
+
return {"success": True, "validation_result": "passed"}
|
312
|
+
|
313
|
+
async def _cross_validate_request(self, request: ApprovalRequest) -> Dict:
|
314
|
+
"""Cross-validate request with ยฑ15% tolerance"""
|
315
|
+
|
316
|
+
# Simulate cross-validation logic
|
317
|
+
# In real implementation, this would validate against external systems
|
318
|
+
|
319
|
+
if request.request_type == "cost_optimization":
|
320
|
+
expected_savings = request.metadata.get("expected_savings", 0)
|
321
|
+
actual_estimate = request.metadata.get("actual_estimate", 0)
|
322
|
+
|
323
|
+
if expected_savings > 0:
|
324
|
+
variance = abs(expected_savings - actual_estimate) / expected_savings * 100
|
325
|
+
if variance <= self.tolerance:
|
326
|
+
return {"valid": True, "variance": variance}
|
327
|
+
else:
|
328
|
+
return {
|
329
|
+
"valid": False,
|
330
|
+
"error": f"Cost variance {variance:.1f}% exceeds tolerance {self.tolerance}%",
|
331
|
+
"variance": variance,
|
332
|
+
}
|
333
|
+
|
334
|
+
return {"valid": True, "variance": 0}
|
335
|
+
|
336
|
+
async def _architecture_review(self, request: ApprovalRequest) -> Dict:
|
337
|
+
"""Execute architecture review step"""
|
338
|
+
|
339
|
+
# Check if architecture review is required
|
340
|
+
if request.request_type in ["deployment_approval", "architecture_review"]:
|
341
|
+
# Simulate architecture review (would integrate with actual review process)
|
342
|
+
review_score = 95 # From CLAUDE.md - "Architecture approved (95/100)"
|
343
|
+
|
344
|
+
if review_score >= 90:
|
345
|
+
logger.info(f"โ
Architecture review passed: {request.request_id} - Score: {review_score}/100")
|
346
|
+
return {"success": True, "review_score": review_score}
|
347
|
+
else:
|
348
|
+
return {"success": False, "error": f"Architecture review failed - Score: {review_score}/100"}
|
349
|
+
|
350
|
+
# Skip if not required
|
351
|
+
return {"success": True, "skipped": "not_required"}
|
352
|
+
|
353
|
+
async def _security_review(self, request: ApprovalRequest) -> Dict:
|
354
|
+
"""Execute security review step"""
|
355
|
+
|
356
|
+
# Check security requirements
|
357
|
+
if request.request_type in ["deployment_approval", "security_changes"]:
|
358
|
+
# Simulate security compliance check
|
359
|
+
compliance_frameworks = ["SOC2", "PCI-DSS", "AWS Well-Architected"]
|
360
|
+
compliance_scores = {fw: 98 for fw in compliance_frameworks} # High compliance scores
|
361
|
+
|
362
|
+
all_passed = all(score >= 95 for score in compliance_scores.values())
|
363
|
+
|
364
|
+
if all_passed:
|
365
|
+
logger.info(f"โ
Security review passed: {request.request_id}")
|
366
|
+
return {"success": True, "compliance_scores": compliance_scores}
|
367
|
+
else:
|
368
|
+
failing_frameworks = [fw for fw, score in compliance_scores.items() if score < 95]
|
369
|
+
return {"success": False, "error": f"Security compliance failed: {failing_frameworks}"}
|
370
|
+
|
371
|
+
return {"success": True, "skipped": "not_required"}
|
372
|
+
|
373
|
+
async def _cost_analysis(self, request: ApprovalRequest) -> Dict:
|
374
|
+
"""Execute cost analysis step"""
|
375
|
+
|
376
|
+
# Check if cost analysis is required
|
377
|
+
cost_impact = request.metadata.get("cost_impact", 0)
|
378
|
+
|
379
|
+
if cost_impact > 1000: # >$1000 threshold from CLAUDE.md
|
380
|
+
# Simulate cost analysis
|
381
|
+
analysis_result = {
|
382
|
+
"cost_impact": cost_impact,
|
383
|
+
"monthly_increase": cost_impact,
|
384
|
+
"annual_impact": cost_impact * 12,
|
385
|
+
"optimization_potential": cost_impact * 0.25, # 25% savings target
|
386
|
+
"recommendation": "approved_with_monitoring",
|
387
|
+
}
|
388
|
+
|
389
|
+
logger.info(f"โ
Cost analysis completed: {request.request_id} - Impact: ${cost_impact}")
|
390
|
+
return {"success": True, "analysis_result": analysis_result}
|
391
|
+
|
392
|
+
return {"success": True, "skipped": "low_cost_impact"}
|
393
|
+
|
394
|
+
async def _manager_approval(self, request: ApprovalRequest) -> Dict:
|
395
|
+
"""Execute manager approval step"""
|
396
|
+
|
397
|
+
# Check if manual approval is pending
|
398
|
+
request_decisions = self.approval_decisions.get(request.request_id, [])
|
399
|
+
manager_decisions = [d for d in request_decisions if "manager" in d.approver.lower()]
|
400
|
+
|
401
|
+
if manager_decisions:
|
402
|
+
latest_decision = max(manager_decisions, key=lambda d: d.decision_time)
|
403
|
+
|
404
|
+
if latest_decision.status == ApprovalStatus.APPROVED:
|
405
|
+
logger.info(f"โ
Manager approval granted: {request.request_id}")
|
406
|
+
return {"success": True, "approver": latest_decision.approver}
|
407
|
+
else:
|
408
|
+
return {"success": False, "error": f"Manager approval {latest_decision.status.value}"}
|
409
|
+
|
410
|
+
# Wait for manual approval (in real implementation, would trigger notification)
|
411
|
+
logger.info(f"โณ Waiting for manager approval: {request.request_id}")
|
412
|
+
|
413
|
+
# For simulation, auto-approve after short delay
|
414
|
+
await asyncio.sleep(1)
|
415
|
+
|
416
|
+
# Simulate manager approval
|
417
|
+
auto_decision = ApprovalDecision(
|
418
|
+
request_id=request.request_id,
|
419
|
+
approver="management_auto",
|
420
|
+
status=ApprovalStatus.APPROVED,
|
421
|
+
decision_time=datetime.now(timezone.utc),
|
422
|
+
reason="Auto-approved for simulation",
|
423
|
+
)
|
424
|
+
|
425
|
+
if request.request_id not in self.approval_decisions:
|
426
|
+
self.approval_decisions[request.request_id] = []
|
427
|
+
self.approval_decisions[request.request_id].append(auto_decision)
|
428
|
+
|
429
|
+
return {"success": True, "approver": "management_auto"}
|
430
|
+
|
431
|
+
async def _execute_deployment(self, request: ApprovalRequest) -> Dict:
|
432
|
+
"""Execute deployment step"""
|
433
|
+
|
434
|
+
if request.request_type == "deployment_approval":
|
435
|
+
# Create deployment record in DORA metrics
|
436
|
+
deployment_id = f"deploy_{request.request_id}"
|
437
|
+
|
438
|
+
deployment = self.dora_engine.record_deployment(
|
439
|
+
deployment_id=deployment_id,
|
440
|
+
environment=request.metadata.get("environment", "production"),
|
441
|
+
service_name=request.metadata.get("service_name", "unknown"),
|
442
|
+
version=request.metadata.get("version", "unknown"),
|
443
|
+
commit_sha=request.metadata.get("commit_sha", ""),
|
444
|
+
approver=request.metadata.get("approver", request.requester),
|
445
|
+
)
|
446
|
+
|
447
|
+
# Simulate deployment (would trigger actual deployment)
|
448
|
+
await asyncio.sleep(2) # Simulate deployment time
|
449
|
+
|
450
|
+
# Mark deployment as successful (90% success rate from CLAUDE.md targets)
|
451
|
+
success_rate = 0.95
|
452
|
+
deployment_success = hash(deployment_id) % 100 < (success_rate * 100)
|
453
|
+
|
454
|
+
status = "success" if deployment_success else "failed"
|
455
|
+
self.dora_engine.complete_deployment(deployment_id, status)
|
456
|
+
|
457
|
+
if deployment_success:
|
458
|
+
logger.info(f"โ
Deployment successful: {deployment_id}")
|
459
|
+
return {"success": True, "deployment_id": deployment_id}
|
460
|
+
else:
|
461
|
+
logger.error(f"โ Deployment failed: {deployment_id}")
|
462
|
+
return {"success": False, "error": "Deployment failed", "deployment_id": deployment_id}
|
463
|
+
|
464
|
+
return {"success": True, "skipped": "not_deployment"}
|
465
|
+
|
466
|
+
def record_manual_approval(self, request_id: str, approver: str, status: ApprovalStatus, reason: str = "") -> bool:
|
467
|
+
"""Record manual approval decision"""
|
468
|
+
|
469
|
+
if request_id not in self.approval_requests:
|
470
|
+
logger.warning(f"โ ๏ธ Approval request not found: {request_id}")
|
471
|
+
return False
|
472
|
+
|
473
|
+
decision = ApprovalDecision(
|
474
|
+
request_id=request_id,
|
475
|
+
approver=approver,
|
476
|
+
status=status,
|
477
|
+
decision_time=datetime.now(timezone.utc),
|
478
|
+
reason=reason,
|
479
|
+
)
|
480
|
+
|
481
|
+
if request_id not in self.approval_decisions:
|
482
|
+
self.approval_decisions[request_id] = []
|
483
|
+
self.approval_decisions[request_id].append(decision)
|
484
|
+
|
485
|
+
# Record approval time
|
486
|
+
request = self.approval_requests[request_id]
|
487
|
+
approval_duration = (decision.decision_time - request.created_at).total_seconds() / 60 # minutes
|
488
|
+
self.dora_engine.record_approval_time(approval_duration, "manual_approval")
|
489
|
+
|
490
|
+
logger.info(f"๐ Manual approval recorded: {request_id} - {status.value} by {approver}")
|
491
|
+
|
492
|
+
return True
|
493
|
+
|
494
|
+
def identify_workflow_bottlenecks(self) -> Dict:
|
495
|
+
"""Identify workflow bottlenecks for optimization"""
|
496
|
+
|
497
|
+
bottlenecks = {
|
498
|
+
"analysis_timestamp": datetime.now(timezone.utc).isoformat(),
|
499
|
+
"step_performance": {},
|
500
|
+
"recommendations": [],
|
501
|
+
}
|
502
|
+
|
503
|
+
# Analyze step durations across all workflows
|
504
|
+
all_step_durations = {}
|
505
|
+
|
506
|
+
for execution in self.workflow_executions.values():
|
507
|
+
for step, duration in execution.step_durations.items():
|
508
|
+
if step not in all_step_durations:
|
509
|
+
all_step_durations[step] = []
|
510
|
+
all_step_durations[step].append(duration)
|
511
|
+
|
512
|
+
# Calculate average durations and identify bottlenecks
|
513
|
+
for step, durations in all_step_durations.items():
|
514
|
+
if durations:
|
515
|
+
avg_duration = sum(durations) / len(durations)
|
516
|
+
max_duration = max(durations)
|
517
|
+
|
518
|
+
bottlenecks["step_performance"][step] = {
|
519
|
+
"average_duration_minutes": avg_duration,
|
520
|
+
"max_duration_minutes": max_duration,
|
521
|
+
"executions_count": len(durations),
|
522
|
+
"is_bottleneck": avg_duration > 30, # >30 minutes considered bottleneck
|
523
|
+
}
|
524
|
+
|
525
|
+
if avg_duration > 30:
|
526
|
+
bottlenecks["recommendations"].append(
|
527
|
+
f"Optimize '{step}' step - average duration {avg_duration:.1f} minutes exceeds target"
|
528
|
+
)
|
529
|
+
|
530
|
+
# Overall recommendations
|
531
|
+
if not bottlenecks["recommendations"]:
|
532
|
+
bottlenecks["recommendations"].append("All workflow steps performing within target times")
|
533
|
+
|
534
|
+
logger.info(f"๐ Workflow bottlenecks analysis completed")
|
535
|
+
|
536
|
+
return bottlenecks
|
537
|
+
|
538
|
+
def generate_hitl_optimization_report(self) -> Dict:
|
539
|
+
"""Generate comprehensive HITL optimization report"""
|
540
|
+
|
541
|
+
logger.info("๐ Generating HITL optimization report")
|
542
|
+
|
543
|
+
# Get DORA metrics
|
544
|
+
dora_report = self.dora_engine.generate_comprehensive_report()
|
545
|
+
|
546
|
+
# Analyze bottlenecks
|
547
|
+
bottlenecks = self.identify_workflow_bottlenecks()
|
548
|
+
|
549
|
+
# Calculate workflow statistics
|
550
|
+
completed_workflows = [w for w in self.workflow_executions.values() if w.status == "completed"]
|
551
|
+
failed_workflows = [w for w in self.workflow_executions.values() if w.status == "failed"]
|
552
|
+
|
553
|
+
workflow_stats = {
|
554
|
+
"total_workflows": len(self.workflow_executions),
|
555
|
+
"completed_workflows": len(completed_workflows),
|
556
|
+
"failed_workflows": len(failed_workflows),
|
557
|
+
"success_rate": len(completed_workflows) / len(self.workflow_executions) if self.workflow_executions else 0,
|
558
|
+
}
|
559
|
+
|
560
|
+
# Calculate average workflow duration
|
561
|
+
if completed_workflows:
|
562
|
+
workflow_durations = []
|
563
|
+
for workflow in completed_workflows:
|
564
|
+
if workflow.end_time:
|
565
|
+
duration = (workflow.end_time - workflow.start_time).total_seconds() / 60 # minutes
|
566
|
+
workflow_durations.append(duration)
|
567
|
+
|
568
|
+
avg_workflow_duration = sum(workflow_durations) / len(workflow_durations) if workflow_durations else 0
|
569
|
+
else:
|
570
|
+
avg_workflow_duration = 0
|
571
|
+
|
572
|
+
# Cross-validation accuracy
|
573
|
+
cross_validation_stats = {
|
574
|
+
"tolerance_percentage": self.tolerance,
|
575
|
+
"validation_enabled": self.optimization_config["cross_validation"],
|
576
|
+
}
|
577
|
+
|
578
|
+
report = {
|
579
|
+
"report_type": "hitl_optimization",
|
580
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
581
|
+
"workflow_statistics": workflow_stats,
|
582
|
+
"average_workflow_duration_minutes": avg_workflow_duration,
|
583
|
+
"cross_validation": cross_validation_stats,
|
584
|
+
"bottleneck_analysis": bottlenecks,
|
585
|
+
"dora_metrics_integration": {
|
586
|
+
"lead_time_hours": dora_report["dora_metrics"]["lead_time"]["value"],
|
587
|
+
"approval_time_minutes": dora_report["hitl_metrics"].get("approval_time", {}).get("value", 0),
|
588
|
+
},
|
589
|
+
"optimization_recommendations": self._generate_optimization_recommendations(workflow_stats, bottlenecks),
|
590
|
+
"performance_grade": dora_report["performance_analysis"]["performance_grade"],
|
591
|
+
}
|
592
|
+
|
593
|
+
# Save report
|
594
|
+
report_file = self.artifacts_dir / f"hitl_optimization_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
|
595
|
+
with open(report_file, "w") as f:
|
596
|
+
json.dump(report, f, indent=2, default=str)
|
597
|
+
|
598
|
+
logger.info(f"โ
HITL optimization report saved to: {report_file}")
|
599
|
+
|
600
|
+
return report
|
601
|
+
|
602
|
+
def _generate_optimization_recommendations(self, workflow_stats: Dict, bottlenecks: Dict) -> List[str]:
|
603
|
+
"""Generate optimization recommendations"""
|
604
|
+
|
605
|
+
recommendations = []
|
606
|
+
|
607
|
+
# Success rate recommendations
|
608
|
+
success_rate = workflow_stats["success_rate"]
|
609
|
+
if success_rate < 0.95: # <95% target from CLAUDE.md
|
610
|
+
recommendations.append(f"๐ Improve workflow success rate: Currently {success_rate:.1%}, target >95%")
|
611
|
+
|
612
|
+
# Bottleneck recommendations
|
613
|
+
for step, perf in bottlenecks["step_performance"].items():
|
614
|
+
if perf["is_bottleneck"]:
|
615
|
+
recommendations.append(
|
616
|
+
f"โก Optimize '{step}' step: Average {perf['average_duration_minutes']:.1f} minutes exceeds 30-minute target"
|
617
|
+
)
|
618
|
+
|
619
|
+
# Cross-validation recommendations
|
620
|
+
if self.tolerance > 10:
|
621
|
+
recommendations.append(
|
622
|
+
f"๐ฏ Tighten cross-validation tolerance: Current {self.tolerance}% could be reduced for better accuracy"
|
623
|
+
)
|
624
|
+
|
625
|
+
# General optimizations
|
626
|
+
if not recommendations:
|
627
|
+
recommendations.extend(
|
628
|
+
[
|
629
|
+
"โ
All HITL metrics within target ranges",
|
630
|
+
"๐ Consider implementing advanced workflow optimizations:",
|
631
|
+
" - Parallel approval processing",
|
632
|
+
" - AI-assisted pre-validation",
|
633
|
+
" - Predictive bottleneck detection",
|
634
|
+
]
|
635
|
+
)
|
636
|
+
|
637
|
+
return recommendations
|
638
|
+
|
639
|
+
|
640
|
+
# Async integration functions
|
641
|
+
async def simulate_hitl_workflow_optimization(duration_minutes: int = 10) -> Dict:
|
642
|
+
"""Simulate HITL workflow optimization for demonstration"""
|
643
|
+
|
644
|
+
from ..metrics.dora_metrics_engine import DORAMetricsEngine
|
645
|
+
|
646
|
+
dora_engine = DORAMetricsEngine()
|
647
|
+
hitl_engine = EnhancedHITLWorkflowEngine(dora_engine)
|
648
|
+
|
649
|
+
logger.info(f"๐งช Starting {duration_minutes}-minute HITL optimization simulation")
|
650
|
+
|
651
|
+
# Simulate various approval requests
|
652
|
+
requests = [
|
653
|
+
(
|
654
|
+
"deployment_approval",
|
655
|
+
"Deploy VPC Wrapper v1.2.0",
|
656
|
+
"Deploy enhanced VPC wrapper with Rich integration",
|
657
|
+
"developer",
|
658
|
+
"high",
|
659
|
+
{"environment": "production", "service_name": "vpc-wrapper", "version": "v1.2.0", "cost_impact": 500},
|
660
|
+
),
|
661
|
+
(
|
662
|
+
"architecture_review",
|
663
|
+
"Multi-Account Organizations API",
|
664
|
+
"Review Organizations API integration architecture",
|
665
|
+
"architect",
|
666
|
+
"medium",
|
667
|
+
{"scope": "multi-account", "compliance_required": True},
|
668
|
+
),
|
669
|
+
(
|
670
|
+
"cost_optimization",
|
671
|
+
"FinOps Dashboard Enhancement",
|
672
|
+
"Cost optimization for FinOps dashboard",
|
673
|
+
"finops",
|
674
|
+
"medium",
|
675
|
+
{"expected_savings": 2500, "actual_estimate": 2400, "cost_impact": 1200},
|
676
|
+
),
|
677
|
+
]
|
678
|
+
|
679
|
+
# Submit approval requests
|
680
|
+
for req_type, title, desc, requester, priority, metadata in requests:
|
681
|
+
request_id = await hitl_engine.submit_approval_request(req_type, title, desc, requester, priority, metadata)
|
682
|
+
|
683
|
+
# Simulate manual approval for some requests
|
684
|
+
if "deployment" in req_type:
|
685
|
+
hitl_engine.record_manual_approval(
|
686
|
+
request_id, "management", ApprovalStatus.APPROVED, "Approved for production deployment"
|
687
|
+
)
|
688
|
+
|
689
|
+
# Allow workflows to complete
|
690
|
+
await asyncio.sleep(3)
|
691
|
+
|
692
|
+
# Generate optimization report
|
693
|
+
optimization_report = hitl_engine.generate_hitl_optimization_report()
|
694
|
+
|
695
|
+
return optimization_report
|
696
|
+
|
697
|
+
|
698
|
+
if __name__ == "__main__":
|
699
|
+
# CLI execution
|
700
|
+
import argparse
|
701
|
+
|
702
|
+
parser = argparse.ArgumentParser(description="Enhanced HITL Workflow Engine")
|
703
|
+
parser.add_argument("--simulate", action="store_true", help="Run optimization simulation")
|
704
|
+
parser.add_argument("--duration", type=int, default=10, help="Simulation duration in minutes")
|
705
|
+
parser.add_argument("--output", "-o", default="./artifacts/hitl", help="Output directory")
|
706
|
+
|
707
|
+
args = parser.parse_args()
|
708
|
+
|
709
|
+
async def main():
|
710
|
+
if args.simulate:
|
711
|
+
report = await simulate_hitl_workflow_optimization(args.duration)
|
712
|
+
print("โ
HITL optimization simulation completed")
|
713
|
+
print(f"๐ Performance grade: {report['performance_grade']}")
|
714
|
+
print(f"โก Average workflow duration: {report['average_workflow_duration_minutes']:.1f} minutes")
|
715
|
+
print(f"๐ฏ Success rate: {report['workflow_statistics']['success_rate']:.1%}")
|
716
|
+
else:
|
717
|
+
from ..metrics.dora_metrics_engine import DORAMetricsEngine
|
718
|
+
|
719
|
+
dora_engine = DORAMetricsEngine(args.output)
|
720
|
+
hitl_engine = EnhancedHITLWorkflowEngine(dora_engine, args.output)
|
721
|
+
report = hitl_engine.generate_hitl_optimization_report()
|
722
|
+
print("โ
HITL optimization report generated")
|
723
|
+
print(f"๐ Report saved to: {hitl_engine.artifacts_dir}")
|
724
|
+
|
725
|
+
asyncio.run(main())
|