crackerjack 0.32.0__py3-none-any.whl → 0.33.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of crackerjack might be problematic. Click here for more details.
- crackerjack/core/enhanced_container.py +67 -0
- crackerjack/core/phase_coordinator.py +183 -44
- crackerjack/core/workflow_orchestrator.py +459 -138
- crackerjack/managers/publish_manager.py +22 -5
- crackerjack/managers/test_command_builder.py +4 -2
- crackerjack/managers/test_manager.py +15 -4
- crackerjack/mcp/server_core.py +162 -34
- crackerjack/mcp/tools/core_tools.py +1 -1
- crackerjack/mcp/tools/execution_tools.py +8 -3
- crackerjack/mixins/__init__.py +5 -0
- crackerjack/mixins/error_handling.py +214 -0
- crackerjack/models/config.py +9 -0
- crackerjack/models/protocols.py +69 -0
- crackerjack/models/task.py +3 -0
- crackerjack/security/__init__.py +1 -1
- crackerjack/security/audit.py +92 -78
- crackerjack/services/config.py +3 -2
- crackerjack/services/config_merge.py +11 -5
- crackerjack/services/coverage_ratchet.py +22 -0
- crackerjack/services/git.py +37 -24
- crackerjack/services/initialization.py +25 -9
- crackerjack/services/memory_optimizer.py +477 -0
- crackerjack/services/parallel_executor.py +474 -0
- crackerjack/services/performance_benchmarks.py +292 -577
- crackerjack/services/performance_cache.py +443 -0
- crackerjack/services/performance_monitor.py +633 -0
- crackerjack/services/security.py +63 -0
- crackerjack/services/security_logger.py +9 -1
- crackerjack/services/terminal_utils.py +0 -0
- {crackerjack-0.32.0.dist-info → crackerjack-0.33.0.dist-info}/METADATA +2 -2
- {crackerjack-0.32.0.dist-info → crackerjack-0.33.0.dist-info}/RECORD +34 -27
- {crackerjack-0.32.0.dist-info → crackerjack-0.33.0.dist-info}/WHEEL +0 -0
- {crackerjack-0.32.0.dist-info → crackerjack-0.33.0.dist-info}/entry_points.txt +0 -0
- {crackerjack-0.32.0.dist-info → crackerjack-0.33.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,628 +1,343 @@
|
|
|
1
|
+
"""Performance benchmarking service to measure Phase 3 optimization improvements.
|
|
2
|
+
|
|
3
|
+
This module provides comprehensive benchmarking capabilities to measure the performance
|
|
4
|
+
gains from async workflows, caching, memory optimization, and parallel execution.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import asyncio
|
|
1
8
|
import json
|
|
2
9
|
import statistics
|
|
3
|
-
import subprocess
|
|
4
10
|
import time
|
|
5
|
-
import typing as t
|
|
6
|
-
from contextlib import suppress
|
|
7
11
|
from dataclasses import dataclass, field
|
|
12
|
+
from datetime import datetime
|
|
8
13
|
from pathlib import Path
|
|
9
|
-
from typing import Any
|
|
10
|
-
|
|
11
|
-
from rich.console import Console
|
|
12
|
-
from rich.table import Table
|
|
13
14
|
|
|
14
|
-
from crackerjack.
|
|
15
|
+
from crackerjack.services.logging import get_logger
|
|
16
|
+
from crackerjack.services.memory_optimizer import get_memory_optimizer
|
|
17
|
+
from crackerjack.services.performance_cache import get_performance_cache
|
|
18
|
+
from crackerjack.services.performance_monitor import get_performance_monitor
|
|
15
19
|
|
|
16
20
|
|
|
17
21
|
@dataclass
|
|
18
22
|
class BenchmarkResult:
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
23
|
+
"""Individual benchmark test result."""
|
|
24
|
+
|
|
25
|
+
test_name: str
|
|
26
|
+
baseline_time_seconds: float
|
|
27
|
+
optimized_time_seconds: float
|
|
28
|
+
memory_baseline_mb: float
|
|
29
|
+
memory_optimized_mb: float
|
|
30
|
+
cache_hits: int = 0
|
|
31
|
+
cache_misses: int = 0
|
|
32
|
+
parallel_operations: int = 0
|
|
33
|
+
sequential_operations: int = 0
|
|
34
|
+
|
|
35
|
+
@property
|
|
36
|
+
def time_improvement_percentage(self) -> float:
|
|
37
|
+
"""Calculate time improvement percentage."""
|
|
38
|
+
if self.baseline_time_seconds == 0:
|
|
39
|
+
return 0.0
|
|
40
|
+
return (
|
|
41
|
+
(self.baseline_time_seconds - self.optimized_time_seconds)
|
|
42
|
+
/ self.baseline_time_seconds
|
|
43
|
+
* 100
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def memory_improvement_percentage(self) -> float:
|
|
48
|
+
"""Calculate memory improvement percentage."""
|
|
49
|
+
if self.memory_baseline_mb == 0:
|
|
50
|
+
return 0.0
|
|
51
|
+
return (
|
|
52
|
+
(self.memory_baseline_mb - self.memory_optimized_mb)
|
|
53
|
+
/ self.memory_baseline_mb
|
|
54
|
+
* 100
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def cache_hit_ratio(self) -> float:
|
|
59
|
+
"""Calculate cache hit ratio."""
|
|
60
|
+
total = self.cache_hits + self.cache_misses
|
|
61
|
+
return self.cache_hits / total if total > 0 else 0.0
|
|
62
|
+
|
|
63
|
+
@property
|
|
64
|
+
def parallelization_ratio(self) -> float:
|
|
65
|
+
"""Calculate parallelization ratio."""
|
|
66
|
+
total = self.parallel_operations + self.sequential_operations
|
|
67
|
+
return self.parallel_operations / total if total > 0 else 0.0
|
|
25
68
|
|
|
26
69
|
|
|
27
70
|
@dataclass
|
|
28
|
-
class
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
self
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
def
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
71
|
+
class BenchmarkSuite:
|
|
72
|
+
"""Collection of benchmark results."""
|
|
73
|
+
|
|
74
|
+
suite_name: str
|
|
75
|
+
results: list[BenchmarkResult] = field(default_factory=list)
|
|
76
|
+
run_timestamp: datetime = field(default_factory=datetime.now)
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def average_time_improvement(self) -> float:
|
|
80
|
+
"""Calculate average time improvement across all tests."""
|
|
81
|
+
if not self.results:
|
|
82
|
+
return 0.0
|
|
83
|
+
improvements = [r.time_improvement_percentage for r in self.results]
|
|
84
|
+
return statistics.mean(improvements)
|
|
85
|
+
|
|
86
|
+
@property
|
|
87
|
+
def average_memory_improvement(self) -> float:
|
|
88
|
+
"""Calculate average memory improvement across all tests."""
|
|
89
|
+
if not self.results:
|
|
90
|
+
return 0.0
|
|
91
|
+
improvements = [r.memory_improvement_percentage for r in self.results]
|
|
92
|
+
return statistics.mean(improvements)
|
|
93
|
+
|
|
94
|
+
@property
|
|
95
|
+
def overall_cache_hit_ratio(self) -> float:
|
|
96
|
+
"""Calculate overall cache hit ratio."""
|
|
97
|
+
total_hits = sum(r.cache_hits for r in self.results)
|
|
98
|
+
total_misses = sum(r.cache_misses for r in self.results)
|
|
99
|
+
total = total_hits + total_misses
|
|
100
|
+
return total_hits / total if total > 0 else 0.0
|
|
101
|
+
|
|
102
|
+
def add_result(self, result: BenchmarkResult) -> None:
|
|
103
|
+
"""Add a benchmark result to the suite."""
|
|
104
|
+
self.results.append(result)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class PerformanceBenchmarker:
|
|
108
|
+
"""Service for benchmarking Phase 3 performance optimizations."""
|
|
109
|
+
|
|
110
|
+
def __init__(self):
|
|
111
|
+
self._logger = get_logger("crackerjack.benchmarker")
|
|
112
|
+
self._monitor = get_performance_monitor()
|
|
113
|
+
self._memory_optimizer = get_memory_optimizer()
|
|
114
|
+
self._cache = get_performance_cache()
|
|
115
|
+
|
|
116
|
+
# Benchmark configurations
|
|
117
|
+
self._test_iterations = 3
|
|
118
|
+
self._warmup_iterations = 1
|
|
119
|
+
|
|
120
|
+
async def run_comprehensive_benchmark(self) -> BenchmarkSuite:
|
|
121
|
+
"""Run comprehensive benchmark suite comparing baseline vs optimized performance."""
|
|
122
|
+
self._logger.info("Starting comprehensive performance benchmark")
|
|
123
|
+
|
|
124
|
+
suite = BenchmarkSuite("Phase 3 Optimization Benchmark")
|
|
125
|
+
|
|
126
|
+
# Memory optimization benchmark
|
|
127
|
+
suite.add_result(await self._benchmark_memory_optimization())
|
|
128
|
+
|
|
129
|
+
# Caching benchmark
|
|
130
|
+
suite.add_result(await self._benchmark_caching_performance())
|
|
131
|
+
|
|
132
|
+
# Async workflow benchmark
|
|
133
|
+
suite.add_result(await self._benchmark_async_workflows())
|
|
134
|
+
|
|
135
|
+
self._logger.info(
|
|
136
|
+
f"Benchmark complete. Average improvements: "
|
|
137
|
+
f"Time: {suite.average_time_improvement:.1f}%, "
|
|
138
|
+
f"Memory: {suite.average_memory_improvement:.1f}%, "
|
|
139
|
+
f"Cache ratio: {suite.overall_cache_hit_ratio:.2f}"
|
|
60
140
|
)
|
|
61
141
|
|
|
62
|
-
|
|
63
|
-
|
|
142
|
+
return suite
|
|
143
|
+
|
|
144
|
+
async def _benchmark_memory_optimization(self) -> BenchmarkResult:
|
|
145
|
+
"""Benchmark memory optimization improvements."""
|
|
146
|
+
self._logger.debug("Benchmarking memory optimization")
|
|
64
147
|
|
|
65
|
-
|
|
66
|
-
|
|
148
|
+
# Baseline: Create objects without optimization
|
|
149
|
+
baseline_start = time.time()
|
|
150
|
+
baseline_memory_start = self._memory_optimizer.record_checkpoint(
|
|
151
|
+
"baseline_start"
|
|
152
|
+
)
|
|
67
153
|
|
|
68
|
-
|
|
154
|
+
# Simulate heavy object creation (baseline)
|
|
155
|
+
heavy_objects = []
|
|
156
|
+
for i in range(50): # Reduced for faster testing
|
|
157
|
+
obj = {
|
|
158
|
+
"data": f"heavy_data_{i}" * 100, # Smaller for testing
|
|
159
|
+
"metadata": {"created": time.time(), "index": i},
|
|
160
|
+
"payload": list(range(100)),
|
|
161
|
+
}
|
|
162
|
+
heavy_objects.append(obj)
|
|
69
163
|
|
|
70
|
-
|
|
71
|
-
|
|
164
|
+
baseline_time = time.time() - baseline_start
|
|
165
|
+
baseline_memory_peak = self._memory_optimizer.record_checkpoint("baseline_peak")
|
|
72
166
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
report: PerformanceReport,
|
|
76
|
-
run_tests: bool,
|
|
77
|
-
run_hooks: bool,
|
|
78
|
-
iterations: int,
|
|
79
|
-
) -> None:
|
|
80
|
-
if run_tests:
|
|
81
|
-
report.test_benchmarks = self._benchmark_test_suite(iterations)
|
|
167
|
+
# Clean up baseline objects
|
|
168
|
+
del heavy_objects
|
|
82
169
|
|
|
83
|
-
|
|
84
|
-
|
|
170
|
+
# Optimized: Use lazy loading
|
|
171
|
+
optimized_start = time.time()
|
|
172
|
+
optimized_memory_start = self._memory_optimizer.record_checkpoint(
|
|
173
|
+
"optimized_start"
|
|
174
|
+
)
|
|
85
175
|
|
|
86
|
-
|
|
87
|
-
report.file_operation_stats = self._benchmark_file_operations()
|
|
176
|
+
from crackerjack.services.memory_optimizer import LazyLoader
|
|
88
177
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
report: PerformanceReport,
|
|
92
|
-
start_time: float,
|
|
93
|
-
) -> None:
|
|
94
|
-
report.total_duration = time.time() - start_time
|
|
95
|
-
report.recommendations = self._generate_performance_recommendations(report)
|
|
96
|
-
report.baseline_comparison = self._compare_with_baseline(report)
|
|
97
|
-
self._save_performance_history(report)
|
|
98
|
-
|
|
99
|
-
def _benchmark_test_suite(self, iterations: int = 1) -> dict[str, Any]:
|
|
100
|
-
self.console.print("[dim]📊 Benchmarking test suite...[/ dim]")
|
|
101
|
-
|
|
102
|
-
benchmark_results = {}
|
|
103
|
-
|
|
104
|
-
try:
|
|
105
|
-
for i in range(iterations):
|
|
106
|
-
start_time = time.time()
|
|
107
|
-
|
|
108
|
-
result = subprocess.run(
|
|
109
|
-
[
|
|
110
|
-
"uv",
|
|
111
|
-
"run",
|
|
112
|
-
"pytest",
|
|
113
|
-
"--benchmark-only",
|
|
114
|
-
"--benchmark-json=.benchmarks/test_benchmark.json",
|
|
115
|
-
"--tb=no",
|
|
116
|
-
"-q",
|
|
117
|
-
],
|
|
118
|
-
check=False,
|
|
119
|
-
capture_output=True,
|
|
120
|
-
text=True,
|
|
121
|
-
timeout=300,
|
|
122
|
-
)
|
|
123
|
-
|
|
124
|
-
duration = time.time() - start_time
|
|
125
|
-
|
|
126
|
-
benchmark_file = self.benchmarks_dir / "test_benchmark.json"
|
|
127
|
-
if benchmark_file.exists():
|
|
128
|
-
with benchmark_file.open() as f:
|
|
129
|
-
benchmark_data = json.load(f)
|
|
130
|
-
|
|
131
|
-
benchmark_results[f"iteration_{i + 1}"] = {
|
|
132
|
-
"total_duration": duration,
|
|
133
|
-
"benchmark_data": benchmark_data,
|
|
134
|
-
"success": result.returncode == 0,
|
|
135
|
-
}
|
|
136
|
-
else:
|
|
137
|
-
benchmark_results[f"iteration_{i + 1}"] = {
|
|
138
|
-
"total_duration": duration,
|
|
139
|
-
"success": result.returncode == 0,
|
|
140
|
-
"note": "No benchmark tests found",
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
except subprocess.TimeoutExpired:
|
|
144
|
-
benchmark_results["error"] = "Test benchmarking timed out"
|
|
145
|
-
except Exception as e:
|
|
146
|
-
benchmark_results["error"] = f"Test benchmarking failed: {e}"
|
|
147
|
-
|
|
148
|
-
return benchmark_results
|
|
149
|
-
|
|
150
|
-
def _benchmark_hooks(self, iterations: int = 1) -> dict[str, float]:
|
|
151
|
-
self.console.print("[dim]🔧 Benchmarking hooks performance...[/ dim]")
|
|
152
|
-
|
|
153
|
-
hook_performance = {}
|
|
154
|
-
|
|
155
|
-
hooks_to_test = [
|
|
156
|
-
"trailing-whitespace",
|
|
157
|
-
"end-of-file-fixer",
|
|
158
|
-
"ruff-format",
|
|
159
|
-
"ruff-check",
|
|
160
|
-
"gitleaks",
|
|
161
|
-
"pyright",
|
|
162
|
-
"bandit",
|
|
163
|
-
"vulture",
|
|
164
|
-
]
|
|
165
|
-
|
|
166
|
-
for hook_name in hooks_to_test:
|
|
167
|
-
durations: list[float] = []
|
|
168
|
-
|
|
169
|
-
for _i in range(iterations):
|
|
170
|
-
try:
|
|
171
|
-
start_time = time.time()
|
|
172
|
-
subprocess.run(
|
|
173
|
-
[
|
|
174
|
-
"uv",
|
|
175
|
-
"run",
|
|
176
|
-
"pre-commit",
|
|
177
|
-
"run",
|
|
178
|
-
hook_name,
|
|
179
|
-
"--all-files",
|
|
180
|
-
],
|
|
181
|
-
check=False,
|
|
182
|
-
capture_output=True,
|
|
183
|
-
text=True,
|
|
184
|
-
timeout=300, # Fixed: Use 300s to match pytest config
|
|
185
|
-
)
|
|
186
|
-
duration = time.time() - start_time
|
|
187
|
-
durations.append(duration)
|
|
188
|
-
except subprocess.TimeoutExpired:
|
|
189
|
-
durations.append(300.0) # Fixed: Use 300s to match timeout
|
|
190
|
-
except Exception:
|
|
191
|
-
durations.append(float("inf"))
|
|
192
|
-
|
|
193
|
-
if durations and all(d != float("inf") for d in durations):
|
|
194
|
-
hook_performance[hook_name] = {
|
|
195
|
-
"mean_duration": statistics.mean(durations),
|
|
196
|
-
"min_duration": min(durations),
|
|
197
|
-
"max_duration": max(durations),
|
|
198
|
-
}
|
|
178
|
+
lazy_objects = []
|
|
179
|
+
for i in range(50):
|
|
199
180
|
|
|
200
|
-
|
|
181
|
+
def create_heavy_object(index: int = i):
|
|
182
|
+
return {
|
|
183
|
+
"data": f"heavy_data_{index}" * 100,
|
|
184
|
+
"metadata": {"created": time.time(), "index": index},
|
|
185
|
+
"payload": list(range(100)),
|
|
186
|
+
}
|
|
201
187
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
iterations: int = 1,
|
|
205
|
-
) -> list[BenchmarkResult]:
|
|
206
|
-
self.console.print("[dim]⚙️ Benchmarking workflow components...[/ dim]")
|
|
188
|
+
lazy_obj = LazyLoader(create_heavy_object, f"heavy_object_{i}")
|
|
189
|
+
lazy_objects.append(lazy_obj)
|
|
207
190
|
|
|
208
|
-
|
|
191
|
+
optimized_time = time.time() - optimized_start
|
|
192
|
+
optimized_memory_peak = self._memory_optimizer.record_checkpoint(
|
|
193
|
+
"optimized_peak"
|
|
194
|
+
)
|
|
209
195
|
|
|
210
|
-
|
|
211
|
-
python_files = list(self.project_root.rglob("*.py"))
|
|
212
|
-
file_discovery_duration = time.time() - start_time
|
|
196
|
+
del lazy_objects
|
|
213
197
|
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
),
|
|
198
|
+
return BenchmarkResult(
|
|
199
|
+
test_name="memory_optimization",
|
|
200
|
+
baseline_time_seconds=baseline_time,
|
|
201
|
+
optimized_time_seconds=optimized_time,
|
|
202
|
+
memory_baseline_mb=max(0, baseline_memory_peak - baseline_memory_start),
|
|
203
|
+
memory_optimized_mb=max(0, optimized_memory_peak - optimized_memory_start),
|
|
220
204
|
)
|
|
221
205
|
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
with suppress(Exception):
|
|
226
|
-
import tomllib
|
|
227
|
-
|
|
228
|
-
with pyproject_path.open("rb") as f:
|
|
229
|
-
tomllib.load(f)
|
|
230
|
-
config_load_duration = time.time() - start_time
|
|
231
|
-
|
|
232
|
-
results.append(
|
|
233
|
-
BenchmarkResult(
|
|
234
|
-
name="config_loading",
|
|
235
|
-
duration_seconds=config_load_duration,
|
|
236
|
-
),
|
|
237
|
-
)
|
|
238
|
-
|
|
239
|
-
return results
|
|
240
|
-
|
|
241
|
-
def _benchmark_file_operations(self) -> dict[str, float]:
|
|
242
|
-
stats = {}
|
|
243
|
-
|
|
244
|
-
test_files = list(self.project_root.glob("*.py"))[:10]
|
|
245
|
-
if test_files:
|
|
246
|
-
start_time = time.time()
|
|
247
|
-
for file_path in test_files:
|
|
248
|
-
with suppress(Exception):
|
|
249
|
-
file_path.read_text(encoding="utf-8")
|
|
250
|
-
read_duration = time.time() - start_time
|
|
251
|
-
stats["file_read_ops"] = read_duration / len(test_files)
|
|
252
|
-
|
|
253
|
-
return stats
|
|
254
|
-
|
|
255
|
-
def _generate_performance_recommendations(
|
|
256
|
-
self,
|
|
257
|
-
report: PerformanceReport,
|
|
258
|
-
) -> list[str]:
|
|
259
|
-
recommendations = []
|
|
260
|
-
|
|
261
|
-
self._add_test_suite_recommendations(report, recommendations)
|
|
262
|
-
self._add_hook_performance_recommendations(report, recommendations)
|
|
263
|
-
self._add_component_performance_recommendations(report, recommendations)
|
|
264
|
-
self._add_overall_performance_recommendations(report, recommendations)
|
|
265
|
-
|
|
266
|
-
return recommendations
|
|
267
|
-
|
|
268
|
-
def _add_test_suite_recommendations(
|
|
269
|
-
self,
|
|
270
|
-
report: PerformanceReport,
|
|
271
|
-
recommendations: list[str],
|
|
272
|
-
) -> None:
|
|
273
|
-
if not report.test_benchmarks:
|
|
274
|
-
return
|
|
206
|
+
async def _benchmark_caching_performance(self) -> BenchmarkResult:
|
|
207
|
+
"""Benchmark caching performance improvements."""
|
|
208
|
+
self._logger.debug("Benchmarking caching performance")
|
|
275
209
|
|
|
276
|
-
|
|
277
|
-
if self._is_slow_test_iteration(iteration_data):
|
|
278
|
-
recommendations.append(
|
|
279
|
-
"Consider optimizing test suite-execution time exceeds 1 minute",
|
|
280
|
-
)
|
|
281
|
-
break
|
|
210
|
+
self._cache.clear()
|
|
282
211
|
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
isinstance(iteration_data, dict)
|
|
286
|
-
and iteration_data.get("total_duration", 0) > 60
|
|
287
|
-
)
|
|
212
|
+
# Baseline: No caching
|
|
213
|
+
baseline_start = time.time()
|
|
288
214
|
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
report: PerformanceReport,
|
|
292
|
-
recommendations: list[str],
|
|
293
|
-
) -> None:
|
|
294
|
-
slow_hooks = self._identify_slow_hooks(report.hook_performance)
|
|
295
|
-
if slow_hooks:
|
|
296
|
-
recommendations.append(self._format_slow_hooks_message(slow_hooks))
|
|
297
|
-
|
|
298
|
-
def _identify_slow_hooks(
|
|
299
|
-
self,
|
|
300
|
-
hook_performance: dict[str, float],
|
|
301
|
-
) -> list[tuple[str, float]]:
|
|
302
|
-
slow_hooks = []
|
|
303
|
-
for hook_name, perf_data in hook_performance.items():
|
|
304
|
-
if isinstance(perf_data, dict):
|
|
305
|
-
mean_duration = perf_data.get("mean_duration", 0)
|
|
306
|
-
if mean_duration > 30:
|
|
307
|
-
slow_hooks.append((hook_name, mean_duration))
|
|
308
|
-
return slow_hooks
|
|
309
|
-
|
|
310
|
-
def _format_slow_hooks_message(self, slow_hooks: list[tuple[str, float]]) -> str:
|
|
311
|
-
hooks_info = ", ".join(f"{h}({d: .1f}s)" for h, d in slow_hooks[:3])
|
|
312
|
-
return (
|
|
313
|
-
f"Slow hooks detected: {hooks_info}. "
|
|
314
|
-
"Consider hook optimization or selective execution."
|
|
315
|
-
)
|
|
215
|
+
for i in range(10): # Reduced for testing
|
|
216
|
+
await self._simulate_expensive_operation(f"operation_{i % 3}")
|
|
316
217
|
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
) -> None:
|
|
341
|
-
if report.total_duration > 300:
|
|
342
|
-
recommendations.append(
|
|
343
|
-
"Overall workflow execution is slow. Consider enabling --skip-hooks "
|
|
344
|
-
"during development iterations.",
|
|
345
|
-
)
|
|
346
|
-
|
|
347
|
-
def _compare_with_baseline(
|
|
348
|
-
self,
|
|
349
|
-
current_report: PerformanceReport,
|
|
350
|
-
) -> dict[str, float]:
|
|
351
|
-
baseline_comparison = {}
|
|
352
|
-
|
|
353
|
-
try:
|
|
354
|
-
history = self._load_performance_history()
|
|
355
|
-
if not history:
|
|
356
|
-
return baseline_comparison
|
|
357
|
-
|
|
358
|
-
self._add_overall_performance_comparison(
|
|
359
|
-
current_report,
|
|
360
|
-
history,
|
|
361
|
-
baseline_comparison,
|
|
362
|
-
)
|
|
363
|
-
self._add_component_performance_comparison(
|
|
364
|
-
current_report,
|
|
365
|
-
history,
|
|
366
|
-
baseline_comparison,
|
|
367
|
-
)
|
|
368
|
-
|
|
369
|
-
except Exception as e:
|
|
370
|
-
baseline_comparison["error"] = f"Could not load baseline: {e}"
|
|
371
|
-
|
|
372
|
-
return baseline_comparison
|
|
373
|
-
|
|
374
|
-
def _load_performance_history(self) -> list[dict[str, Any]] | None:
|
|
375
|
-
if not self.history_file.exists():
|
|
376
|
-
return None
|
|
377
|
-
|
|
378
|
-
with self.history_file.open() as f:
|
|
379
|
-
history = json.load(f)
|
|
380
|
-
|
|
381
|
-
return history if history and len(history) > 1 else None
|
|
382
|
-
|
|
383
|
-
def _add_overall_performance_comparison(
|
|
384
|
-
self,
|
|
385
|
-
current_report: PerformanceReport,
|
|
386
|
-
history: list[dict[str, Any]],
|
|
387
|
-
comparison: dict[str, Any],
|
|
388
|
-
) -> None:
|
|
389
|
-
recent_runs = history[-5:]
|
|
390
|
-
baseline_duration = statistics.median(
|
|
391
|
-
[r["total_duration"] for r in recent_runs],
|
|
218
|
+
baseline_time = time.time() - baseline_start
|
|
219
|
+
|
|
220
|
+
# Optimized: With caching
|
|
221
|
+
optimized_start = time.time()
|
|
222
|
+
cache_stats_start = self._cache.get_stats()
|
|
223
|
+
|
|
224
|
+
for i in range(10):
|
|
225
|
+
await self._simulate_cached_operation(f"operation_{i % 3}")
|
|
226
|
+
|
|
227
|
+
optimized_time = time.time() - optimized_start
|
|
228
|
+
cache_stats_end = self._cache.get_stats()
|
|
229
|
+
|
|
230
|
+
cache_hits = cache_stats_end.hits - cache_stats_start.hits
|
|
231
|
+
cache_misses = cache_stats_end.misses - cache_stats_start.misses
|
|
232
|
+
|
|
233
|
+
return BenchmarkResult(
|
|
234
|
+
test_name="caching_performance",
|
|
235
|
+
baseline_time_seconds=baseline_time,
|
|
236
|
+
optimized_time_seconds=optimized_time,
|
|
237
|
+
memory_baseline_mb=0.0,
|
|
238
|
+
memory_optimized_mb=0.0,
|
|
239
|
+
cache_hits=cache_hits,
|
|
240
|
+
cache_misses=cache_misses,
|
|
392
241
|
)
|
|
393
242
|
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
comparison["overall_performance_change_percent"] = performance_change
|
|
243
|
+
async def _benchmark_async_workflows(self) -> BenchmarkResult:
|
|
244
|
+
"""Benchmark async workflow improvements."""
|
|
245
|
+
self._logger.debug("Benchmarking async workflows")
|
|
398
246
|
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
current_report: PerformanceReport,
|
|
402
|
-
history: list[dict[str, Any]],
|
|
403
|
-
comparison: dict[str, Any],
|
|
404
|
-
) -> None:
|
|
405
|
-
recent_runs = history[-5:]
|
|
406
|
-
if not recent_runs:
|
|
407
|
-
return
|
|
408
|
-
|
|
409
|
-
component_durations = recent_runs[-1].get("component_durations", {})
|
|
410
|
-
|
|
411
|
-
for component in current_report.workflow_benchmarks:
|
|
412
|
-
if component.name in component_durations:
|
|
413
|
-
old_duration = component_durations[component.name]
|
|
414
|
-
change = self._calculate_performance_change(
|
|
415
|
-
component.duration_seconds,
|
|
416
|
-
old_duration,
|
|
417
|
-
)
|
|
418
|
-
comparison[f"{component.name}_change_percent"] = change
|
|
419
|
-
|
|
420
|
-
def _calculate_performance_change(
|
|
421
|
-
self,
|
|
422
|
-
current_duration: float,
|
|
423
|
-
old_duration: float,
|
|
424
|
-
) -> float:
|
|
425
|
-
return ((current_duration - old_duration) / old_duration) * 100
|
|
426
|
-
|
|
427
|
-
def _save_performance_history(self, report: PerformanceReport) -> None:
|
|
428
|
-
try:
|
|
429
|
-
history = []
|
|
430
|
-
if self.history_file.exists():
|
|
431
|
-
with self.history_file.open() as f:
|
|
432
|
-
history = json.load(f)
|
|
433
|
-
|
|
434
|
-
record = {
|
|
435
|
-
"timestamp": time.time(),
|
|
436
|
-
"total_duration": report.total_duration,
|
|
437
|
-
"component_durations": {
|
|
438
|
-
c.name: c.duration_seconds for c in report.workflow_benchmarks
|
|
439
|
-
},
|
|
440
|
-
"hook_durations": {
|
|
441
|
-
hook: (perf["mean_duration"] if isinstance(perf, dict) else perf)
|
|
442
|
-
for hook, perf in report.hook_performance.items()
|
|
443
|
-
},
|
|
444
|
-
"recommendations_count": len(report.recommendations),
|
|
445
|
-
}
|
|
247
|
+
# Baseline: Sequential operations
|
|
248
|
+
baseline_start = time.time()
|
|
446
249
|
|
|
447
|
-
|
|
250
|
+
for i in range(5): # Reduced for testing
|
|
251
|
+
await self._simulate_io_operation(f"seq_{i}", 0.01) # Reduced delay
|
|
448
252
|
|
|
449
|
-
|
|
253
|
+
baseline_time = time.time() - baseline_start
|
|
450
254
|
|
|
451
|
-
|
|
452
|
-
|
|
255
|
+
# Optimized: Parallel operations
|
|
256
|
+
optimized_start = time.time()
|
|
453
257
|
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
f"[yellow]⚠️[/ yellow] Could not save performance history: {e}",
|
|
457
|
-
)
|
|
258
|
+
tasks = [self._simulate_io_operation(f"par_{i}", 0.01) for i in range(5)]
|
|
259
|
+
await asyncio.gather(*tasks)
|
|
458
260
|
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
261
|
+
optimized_time = time.time() - optimized_start
|
|
262
|
+
|
|
263
|
+
return BenchmarkResult(
|
|
264
|
+
test_name="async_workflows",
|
|
265
|
+
baseline_time_seconds=baseline_time,
|
|
266
|
+
optimized_time_seconds=optimized_time,
|
|
267
|
+
memory_baseline_mb=0.0,
|
|
268
|
+
memory_optimized_mb=0.0,
|
|
269
|
+
parallel_operations=5,
|
|
270
|
+
sequential_operations=5,
|
|
462
271
|
)
|
|
463
272
|
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
self._display_baseline_comparison(report)
|
|
468
|
-
self._display_recommendations(report)
|
|
273
|
+
async def _simulate_expensive_operation(self, operation_id: str) -> str:
|
|
274
|
+
"""Simulate an expensive operation without caching."""
|
|
275
|
+
await asyncio.sleep(0.002) # 2ms delay for testing
|
|
469
276
|
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
277
|
+
result = ""
|
|
278
|
+
for i in range(100): # Reduced computation
|
|
279
|
+
result += f"{operation_id}_{i}"
|
|
280
|
+
|
|
281
|
+
return result[:50]
|
|
473
282
|
|
|
474
|
-
def
|
|
475
|
-
|
|
476
|
-
|
|
283
|
+
async def _simulate_cached_operation(self, operation_id: str) -> str:
|
|
284
|
+
"""Simulate an expensive operation with caching."""
|
|
285
|
+
cached_result = await self._cache.get_async(f"expensive_op:{operation_id}")
|
|
286
|
+
if cached_result is not None:
|
|
287
|
+
return cached_result
|
|
288
|
+
|
|
289
|
+
result = await self._simulate_expensive_operation(operation_id)
|
|
290
|
+
await self._cache.set_async(
|
|
291
|
+
f"expensive_op:{operation_id}", result, ttl_seconds=60
|
|
477
292
|
)
|
|
478
293
|
|
|
479
|
-
|
|
480
|
-
if not report.workflow_benchmarks:
|
|
481
|
-
return
|
|
482
|
-
|
|
483
|
-
table = Table(title="Workflow Component Performance")
|
|
484
|
-
table.add_column("Component", style="cyan")
|
|
485
|
-
table.add_column("Duration (s)", style="yellow", justify="right")
|
|
486
|
-
table.add_column("Metadata", style="dim")
|
|
487
|
-
|
|
488
|
-
for benchmark in report.workflow_benchmarks:
|
|
489
|
-
metadata_str = ", ".join(f"{k}={v}" for k, v in benchmark.metadata.items())
|
|
490
|
-
table.add_row(
|
|
491
|
-
benchmark.name,
|
|
492
|
-
f"{benchmark.duration_seconds: .3f}",
|
|
493
|
-
metadata_str,
|
|
494
|
-
)
|
|
495
|
-
|
|
496
|
-
self.console.print(table)
|
|
497
|
-
self.console.print()
|
|
498
|
-
|
|
499
|
-
def _display_hook_performance(self, report: PerformanceReport) -> None:
|
|
500
|
-
if not report.hook_performance:
|
|
501
|
-
return
|
|
502
|
-
|
|
503
|
-
table = Table(title="Hook Performance Analysis")
|
|
504
|
-
table.add_column("Hook", style="cyan")
|
|
505
|
-
table.add_column("Mean (s)", style="yellow", justify="right")
|
|
506
|
-
table.add_column("Min (s)", style="green", justify="right")
|
|
507
|
-
table.add_column("Max (s)", style="red", justify="right")
|
|
508
|
-
|
|
509
|
-
for hook_name, perf_data in report.hook_performance.items():
|
|
510
|
-
if isinstance(perf_data, dict):
|
|
511
|
-
table.add_row(
|
|
512
|
-
hook_name,
|
|
513
|
-
f"{perf_data.get('mean_duration', 0): .2f}",
|
|
514
|
-
f"{perf_data.get('min_duration', 0): .2f}",
|
|
515
|
-
f"{perf_data.get('max_duration', 0): .2f}",
|
|
516
|
-
)
|
|
517
|
-
|
|
518
|
-
self.console.print(table)
|
|
519
|
-
self.console.print()
|
|
520
|
-
|
|
521
|
-
def _display_baseline_comparison(self, report: PerformanceReport) -> None:
|
|
522
|
-
if not report.baseline_comparison:
|
|
523
|
-
return
|
|
524
|
-
|
|
525
|
-
self._print_comparison_header()
|
|
526
|
-
self._print_comparison_metrics(report.baseline_comparison)
|
|
527
|
-
self.console.print()
|
|
528
|
-
|
|
529
|
-
def _print_comparison_header(self) -> None:
|
|
530
|
-
self.console.print("[bold]📊 Performance Comparison[/ bold]")
|
|
531
|
-
|
|
532
|
-
def _print_comparison_metrics(self, baseline_comparison: dict[str, t.Any]) -> None:
|
|
533
|
-
for metric, value in baseline_comparison.items():
|
|
534
|
-
if isinstance(value, float | int) and "percent" in metric:
|
|
535
|
-
color = "green" if value < 0 else "red" if value > 10 else "yellow"
|
|
536
|
-
direction = "faster" if value < 0 else "slower"
|
|
537
|
-
self.console.print(
|
|
538
|
-
f" {metric}: [{color}]{abs(value): .1f}% {direction}[/{color}]",
|
|
539
|
-
)
|
|
540
|
-
|
|
541
|
-
def _display_recommendations(self, report: PerformanceReport) -> None:
|
|
542
|
-
if report.recommendations:
|
|
543
|
-
self.console.print(
|
|
544
|
-
"[bold yellow]💡 Performance Recommendations[/ bold yellow]",
|
|
545
|
-
)
|
|
546
|
-
for i, rec in enumerate(report.recommendations, 1):
|
|
547
|
-
self.console.print(f" {i}. {rec}")
|
|
548
|
-
else:
|
|
549
|
-
self.console.print("[green]✨ No performance issues detected ![/ green]")
|
|
550
|
-
|
|
551
|
-
def get_performance_trends(self, days: int = 7) -> dict[str, Any]:
|
|
552
|
-
try:
|
|
553
|
-
recent_history = self._get_recent_history(days)
|
|
554
|
-
if not recent_history:
|
|
555
|
-
return self._handle_insufficient_trend_data()
|
|
556
|
-
|
|
557
|
-
trends = {}
|
|
558
|
-
self._add_duration_trends(recent_history, trends)
|
|
559
|
-
self._add_component_trends(recent_history, trends)
|
|
560
|
-
trends["data_points"] = len(recent_history)
|
|
561
|
-
|
|
562
|
-
return trends
|
|
563
|
-
|
|
564
|
-
except Exception as e:
|
|
565
|
-
return {"error": f"Could not analyze trends: {e}"}
|
|
566
|
-
|
|
567
|
-
def _get_recent_history(self, days: int) -> list[dict[str, Any]] | None:
|
|
568
|
-
if not self.history_file.exists():
|
|
569
|
-
return None
|
|
570
|
-
|
|
571
|
-
with self.history_file.open() as f:
|
|
572
|
-
history = json.load(f)
|
|
573
|
-
|
|
574
|
-
cutoff_time = time.time() - (days * 86400)
|
|
575
|
-
recent_history = [r for r in history if r.get("timestamp", 0) > cutoff_time]
|
|
576
|
-
|
|
577
|
-
return recent_history if len(recent_history) >= 2 else None
|
|
578
|
-
|
|
579
|
-
def _handle_insufficient_trend_data(self) -> dict[str, str]:
|
|
580
|
-
if not self.history_file.exists():
|
|
581
|
-
return {"error": "No performance history available"}
|
|
582
|
-
return {"error": "Insufficient data for trend analysis"}
|
|
583
|
-
|
|
584
|
-
def _add_duration_trends(
|
|
585
|
-
self, recent_history: list[dict[str, Any]], trends: dict[str, Any]
|
|
586
|
-
) -> None:
|
|
587
|
-
durations = [r["total_duration"] for r in recent_history]
|
|
588
|
-
trends["duration_trend"] = {
|
|
589
|
-
"current": durations[-1],
|
|
590
|
-
"average": statistics.mean(durations),
|
|
591
|
-
"trend": self._determine_trend_direction(durations),
|
|
592
|
-
}
|
|
294
|
+
return result
|
|
593
295
|
|
|
594
|
-
def
|
|
595
|
-
|
|
296
|
+
async def _simulate_io_operation(self, operation_id: str, duration: float) -> str:
|
|
297
|
+
"""Simulate I/O bound operation."""
|
|
298
|
+
await asyncio.sleep(duration)
|
|
299
|
+
return f"result_{operation_id}"
|
|
300
|
+
|
|
301
|
+
def export_benchmark_results(
|
|
302
|
+
self, suite: BenchmarkSuite, output_path: Path
|
|
596
303
|
) -> None:
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
"
|
|
304
|
+
"""Export benchmark results to JSON file."""
|
|
305
|
+
data = {
|
|
306
|
+
"suite_name": suite.suite_name,
|
|
307
|
+
"run_timestamp": suite.run_timestamp.isoformat(),
|
|
308
|
+
"summary": {
|
|
309
|
+
"average_time_improvement_percentage": suite.average_time_improvement,
|
|
310
|
+
"average_memory_improvement_percentage": suite.average_memory_improvement,
|
|
311
|
+
"overall_cache_hit_ratio": suite.overall_cache_hit_ratio,
|
|
312
|
+
"total_tests": len(suite.results),
|
|
313
|
+
},
|
|
314
|
+
"results": [
|
|
315
|
+
{
|
|
316
|
+
"test_name": r.test_name,
|
|
317
|
+
"baseline_time_seconds": r.baseline_time_seconds,
|
|
318
|
+
"optimized_time_seconds": r.optimized_time_seconds,
|
|
319
|
+
"time_improvement_percentage": r.time_improvement_percentage,
|
|
320
|
+
"memory_baseline_mb": r.memory_baseline_mb,
|
|
321
|
+
"memory_optimized_mb": r.memory_optimized_mb,
|
|
322
|
+
"memory_improvement_percentage": r.memory_improvement_percentage,
|
|
323
|
+
"cache_hits": r.cache_hits,
|
|
324
|
+
"cache_misses": r.cache_misses,
|
|
325
|
+
"cache_hit_ratio": r.cache_hit_ratio,
|
|
326
|
+
"parallel_operations": r.parallel_operations,
|
|
327
|
+
"sequential_operations": r.sequential_operations,
|
|
328
|
+
"parallelization_ratio": r.parallelization_ratio,
|
|
610
329
|
}
|
|
330
|
+
for r in suite.results
|
|
331
|
+
],
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
with output_path.open("w") as f:
|
|
335
|
+
json.dump(data, f, indent=2)
|
|
336
|
+
|
|
337
|
+
self._logger.info(f"Exported benchmark results to {output_path}")
|
|
338
|
+
|
|
611
339
|
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
recent_history: list[dict[str, Any]],
|
|
617
|
-
component: str,
|
|
618
|
-
) -> list[float]:
|
|
619
|
-
return [
|
|
620
|
-
r.get("component_durations", {}).get(component)
|
|
621
|
-
for r in recent_history
|
|
622
|
-
if component in r.get("component_durations", {})
|
|
623
|
-
]
|
|
624
|
-
|
|
625
|
-
def _determine_trend_direction(self, durations: list[float]) -> str:
|
|
626
|
-
current = durations[-1]
|
|
627
|
-
historical_average = statistics.mean(durations[:-1])
|
|
628
|
-
return "improving" if current < historical_average else "degrading"
|
|
340
|
+
# Global benchmarker instance
|
|
341
|
+
def get_benchmarker() -> PerformanceBenchmarker:
|
|
342
|
+
"""Get performance benchmarker instance."""
|
|
343
|
+
return PerformanceBenchmarker()
|