ivolatility-backtesting 1.9.0__py3-none-any.whl → 1.11.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ivolatility-backtesting might be problematic. Click here for more details.

@@ -0,0 +1,4222 @@
1
+ # ============================================================
2
+ # ivolatility_backtesting.py - ENHANCED VERSION
3
+ #
4
+ # NEW FEATURES:
5
+ # 1. Combined stop-loss (requires BOTH conditions)
6
+ # 2. Parameter optimization framework
7
+ # 3. Optimization results visualization
8
+ # ============================================================
9
+
10
+ import pandas as pd
11
+ import numpy as np
12
+ import matplotlib.pyplot as plt
13
+ import seaborn as sns
14
+ from datetime import datetime, timedelta
15
+ import ivolatility as ivol
16
+ import os
17
+ import time
18
+ import psutil
19
+ import warnings
20
+ from itertools import product
21
+ import sys
22
+ from typing import Dict, List, Optional, Tuple, Union, Any
23
+ warnings.filterwarnings('ignore', category=pd.errors.SettingWithCopyWarning)
24
+ warnings.filterwarnings('ignore', message='.*SettingWithCopyWarning.*')
25
+ warnings.filterwarnings('ignore', category=FutureWarning)
26
+ warnings.filterwarnings('ignore', category=DeprecationWarning)
27
+
28
+ sns.set_style('darkgrid')
29
+ plt.rcParams['figure.figsize'] = (15, 8)
30
+
31
+ def create_optimization_folder(base_dir='optimization_results'):
32
+ """
33
+ Create timestamped folder for optimization run
34
+ Returns: folder path (e.g., 'optimization_results/20250122_143025')
35
+ """
36
+ from pathlib import Path
37
+ timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
38
+ folder_path = Path(base_dir) / timestamp
39
+ folder_path.mkdir(parents=True, exist_ok=True)
40
+ print(f"\n📁 Created optimization folder: {folder_path}")
41
+ return str(folder_path)
42
+
43
+ # ============================================================
44
+ # RESOURCE MONITOR
45
+ # ============================================================
46
+ class ResourceMonitor:
47
+ """Monitor CPU and RAM with container support"""
48
+
49
+ def __init__(self, show_container_total=False):
50
+ self.process = psutil.Process()
51
+ self.cpu_count = psutil.cpu_count()
52
+ self.last_cpu_time = None
53
+ self.last_check_time = None
54
+ self.use_cgroups = self._check_cgroups_v2()
55
+ self.show_container_total = show_container_total
56
+ self.cpu_history = []
57
+ self.cpu_history_max = 5
58
+
59
+ if self.use_cgroups:
60
+ quota = self._read_cpu_quota()
61
+ if quota and quota > 0:
62
+ self.cpu_count = quota
63
+
64
+ self.context = "Container" if self.use_cgroups else "Host"
65
+
66
+ def _read_cpu_quota(self):
67
+ try:
68
+ with open('/sys/fs/cgroup/cpu.max', 'r') as f:
69
+ line = f.read().strip()
70
+ if line == 'max':
71
+ return None
72
+ parts = line.split()
73
+ if len(parts) == 2:
74
+ quota = int(parts[0])
75
+ period = int(parts[1])
76
+ return quota / period
77
+ except:
78
+ pass
79
+ return None
80
+
81
+ def get_context_info(self):
82
+ if self.use_cgroups:
83
+ current, max_mem = self._read_cgroup_memory()
84
+ ram_info = ""
85
+ if max_mem:
86
+ max_mem_gb = max_mem / (1024**3)
87
+ ram_info = f", {max_mem_gb:.1f}GB limit"
88
+
89
+ mem_type = "container total" if self.show_container_total else "process only"
90
+ return f"Container (CPU: {self.cpu_count:.1f} cores{ram_info}) - RAM: {mem_type}"
91
+ else:
92
+ total_ram_gb = psutil.virtual_memory().total / (1024**3)
93
+ return f"Host ({self.cpu_count} cores, {total_ram_gb:.0f}GB RAM) - RAM: process"
94
+
95
+ def _check_cgroups_v2(self):
96
+ try:
97
+ return os.path.exists('/sys/fs/cgroup/cpu.stat') and \
98
+ os.path.exists('/sys/fs/cgroup/memory.current')
99
+ except:
100
+ return False
101
+
102
+ def _read_cgroup_cpu(self):
103
+ try:
104
+ with open('/sys/fs/cgroup/cpu.stat', 'r') as f:
105
+ for line in f:
106
+ if line.startswith('usage_usec'):
107
+ return int(line.split()[1])
108
+ except:
109
+ pass
110
+ return None
111
+
112
+ def _read_cgroup_memory(self):
113
+ try:
114
+ with open('/sys/fs/cgroup/memory.current', 'r') as f:
115
+ current = int(f.read().strip())
116
+ with open('/sys/fs/cgroup/memory.max', 'r') as f:
117
+ max_mem = f.read().strip()
118
+ if max_mem == 'max':
119
+ max_mem = psutil.virtual_memory().total
120
+ else:
121
+ max_mem = int(max_mem)
122
+ return current, max_mem
123
+ except:
124
+ pass
125
+ return None, None
126
+
127
+ def get_cpu_percent(self):
128
+ if self.use_cgroups:
129
+ current_time = time.time()
130
+ current_cpu = self._read_cgroup_cpu()
131
+
132
+ if current_cpu and self.last_cpu_time and self.last_check_time:
133
+ time_delta = current_time - self.last_check_time
134
+ cpu_delta = current_cpu - self.last_cpu_time
135
+
136
+ if time_delta > 0:
137
+ cpu_percent = (cpu_delta / (time_delta * 1_000_000)) * 100
138
+ cpu_percent = min(cpu_percent, 100 * self.cpu_count)
139
+
140
+ self.cpu_history.append(cpu_percent)
141
+ if len(self.cpu_history) > self.cpu_history_max:
142
+ self.cpu_history.pop(0)
143
+
144
+ self.last_cpu_time = current_cpu
145
+ self.last_check_time = current_time
146
+
147
+ return round(sum(self.cpu_history) / len(self.cpu_history), 1)
148
+
149
+ self.last_cpu_time = current_cpu
150
+ self.last_check_time = current_time
151
+
152
+ try:
153
+ cpu = self.process.cpu_percent(interval=0.1)
154
+ if cpu == 0:
155
+ cpu = psutil.cpu_percent(interval=0.1)
156
+
157
+ self.cpu_history.append(cpu)
158
+ if len(self.cpu_history) > self.cpu_history_max:
159
+ self.cpu_history.pop(0)
160
+
161
+ return round(sum(self.cpu_history) / len(self.cpu_history), 1)
162
+ except:
163
+ return 0.0
164
+
165
+ def get_memory_info(self):
166
+ try:
167
+ mem = self.process.memory_info()
168
+ process_mb = mem.rss / (1024 * 1024)
169
+
170
+ if self.use_cgroups:
171
+ current, max_mem = self._read_cgroup_memory()
172
+ if max_mem:
173
+ process_percent = (mem.rss / max_mem) * 100
174
+
175
+ if current:
176
+ container_mb = current / (1024 * 1024)
177
+ container_percent = (current / max_mem) * 100
178
+ return (
179
+ round(process_mb, 1),
180
+ round(process_percent, 1),
181
+ round(container_mb, 1),
182
+ round(container_percent, 1)
183
+ )
184
+
185
+ return (
186
+ round(process_mb, 1),
187
+ round(process_percent, 1),
188
+ round(process_mb, 1),
189
+ round(process_percent, 1)
190
+ )
191
+
192
+ total = psutil.virtual_memory().total
193
+ percent = (mem.rss / total) * 100
194
+
195
+ return (
196
+ round(process_mb, 1),
197
+ round(percent, 1),
198
+ round(process_mb, 1),
199
+ round(percent, 1)
200
+ )
201
+
202
+ except:
203
+ return 0.0, 0.0, 0.0, 0.0
204
+
205
+
206
+ def create_progress_bar(reuse_existing=None):
207
+ """Create or reuse enhanced progress bar"""
208
+ if reuse_existing is not None:
209
+ progress_bar, status_label, monitor, start_time = reuse_existing
210
+ progress_bar.value = 0
211
+ progress_bar.bar_style = 'info'
212
+ status_label.value = "<b style='color:#0066cc'>Starting...</b>"
213
+ return progress_bar, status_label, monitor, time.time()
214
+
215
+ try:
216
+ from IPython.display import display
217
+ import ipywidgets as widgets
218
+
219
+ progress_bar = widgets.FloatProgress(
220
+ value=0, min=0, max=100,
221
+ description='Progress:',
222
+ bar_style='info',
223
+ style={'bar_color': '#00ff00'},
224
+ layout=widgets.Layout(width='100%', height='30px')
225
+ )
226
+
227
+ status_label = widgets.HTML(
228
+ value="<b style='color:#0066cc'>Starting...</b>"
229
+ )
230
+
231
+ display(widgets.VBox([progress_bar, status_label]))
232
+
233
+ monitor = ResourceMonitor()
234
+ start_time = time.time()
235
+
236
+ return progress_bar, status_label, monitor, start_time
237
+ except ImportError:
238
+ print("Warning: ipywidgets not available. Progress bar disabled.")
239
+ return None, None, ResourceMonitor(), time.time()
240
+
241
+
242
+ def update_progress(progress_bar, status_label, monitor, current, total, start_time, message="Processing"):
243
+ """Update progress bar with ETA, CPU%, RAM"""
244
+ if progress_bar is None or status_label is None:
245
+ return
246
+
247
+ progress = (current / total) * 100
248
+ progress_bar.value = progress
249
+
250
+ elapsed = time.time() - start_time
251
+ if current > 0:
252
+ eta_seconds = (elapsed / current) * (total - current)
253
+ eta_str = format_time(eta_seconds)
254
+ else:
255
+ eta_str = "calculating..."
256
+
257
+ cpu = monitor.get_cpu_percent()
258
+ process_mb, process_pct, container_mb, container_pct = monitor.get_memory_info()
259
+
260
+ if abs(container_mb - process_mb) > 10:
261
+ ram_display = (
262
+ f"RAM: <span style='color:#4CAF50'>{process_mb}MB ({process_pct}%)</span> Python | "
263
+ f"<span style='color:#2196F3'>{container_mb}MB ({container_pct}%)</span> Container"
264
+ )
265
+ else:
266
+ ram_display = f"RAM: {process_mb}MB ({process_pct}%)"
267
+
268
+ context_info = monitor.get_context_info()
269
+
270
+ elapsed_str = format_time(elapsed)
271
+ start_time_str = datetime.fromtimestamp(start_time).strftime('%H:%M:%S')
272
+
273
+ status_label.value = (
274
+ f"<b style='color:#0066cc'>{message} ({current}/{total})</b><br>"
275
+ f"<span style='color:#666'>⏱️ Elapsed: {elapsed_str} | ETA: {eta_str} | Started: {start_time_str}</span><br>"
276
+ f"<span style='color:#666'>CPU: {cpu}% | {ram_display}</span><br>"
277
+ f"<span style='color:#999;font-size:10px'>{context_info}</span>"
278
+ )
279
+
280
+
281
+ def format_time(seconds):
282
+ """Format seconds to human readable time"""
283
+ if seconds < 60:
284
+ return f"{int(seconds)}s"
285
+ elif seconds < 3600:
286
+ return f"{int(seconds // 60)}m {int(seconds % 60)}s"
287
+ else:
288
+ hours = int(seconds // 3600)
289
+ minutes = int((seconds % 3600) // 60)
290
+ return f"{hours}h {minutes}m"
291
+
292
+
293
+ # ============================================================
294
+ # API HELPER
295
+ # ============================================================
296
+ class APIHelper:
297
+ """Normalizes API responses"""
298
+
299
+ @staticmethod
300
+ def normalize_response(response, debug=False):
301
+ if response is None:
302
+ if debug:
303
+ print("[APIHelper] Response is None")
304
+ return None
305
+
306
+ if isinstance(response, dict):
307
+ if 'data' in response:
308
+ if debug:
309
+ print(f"[APIHelper] Dict response: {len(response['data'])} records")
310
+ return response
311
+ else:
312
+ if debug:
313
+ print("[APIHelper] Dict without 'data' key")
314
+ return None
315
+
316
+ if isinstance(response, pd.DataFrame):
317
+ if response.empty:
318
+ if debug:
319
+ print("[APIHelper] Empty DataFrame")
320
+ return None
321
+
322
+ records = response.to_dict('records')
323
+ if debug:
324
+ print(f"[APIHelper] DataFrame converted: {len(records)} records")
325
+ return {'data': records, 'status': 'success'}
326
+
327
+ if debug:
328
+ print(f"[APIHelper] Unexpected type: {type(response)}")
329
+ return None
330
+
331
+
332
+ class APIManager:
333
+ """Centralized API key management"""
334
+ _api_key = None
335
+ _methods = {}
336
+
337
+ @classmethod
338
+ def initialize(cls, api_key):
339
+ if not api_key:
340
+ raise ValueError("API key cannot be empty")
341
+ cls._api_key = api_key
342
+ ivol.setLoginParams(apiKey=api_key)
343
+ print(f"[API] Initialized: {api_key[:10]}...{api_key[-5:]}")
344
+
345
+ @classmethod
346
+ def get_method(cls, endpoint):
347
+ if cls._api_key is None:
348
+ api_key = os.getenv("API_KEY")
349
+ if not api_key:
350
+ raise ValueError("API key not set. Call init_api(key) first")
351
+ cls.initialize(api_key)
352
+
353
+ if endpoint not in cls._methods:
354
+ ivol.setLoginParams(apiKey=cls._api_key)
355
+ cls._methods[endpoint] = ivol.setMethod(endpoint)
356
+
357
+ return cls._methods[endpoint]
358
+
359
+
360
+ def init_api(api_key=None):
361
+ """Initialize IVolatility API"""
362
+ if api_key is None:
363
+ api_key = os.getenv("API_KEY")
364
+ APIManager.initialize(api_key)
365
+
366
+
367
+ def api_call(endpoint, cache_config=None, debug=False, **kwargs):
368
+ """
369
+ Make API call with automatic response normalization and caching
370
+
371
+ Args:
372
+ endpoint: API endpoint path
373
+ cache_config: Cache configuration dict (optional, enables caching if provided)
374
+ debug: Debug mode flag
375
+ **kwargs: API parameters
376
+
377
+ Returns:
378
+ Normalized API response or None
379
+ """
380
+ try:
381
+ # Check if caching is enabled
382
+ use_cache = cache_config is not None and (
383
+ cache_config.get('disk_enabled', False) or
384
+ cache_config.get('memory_enabled', False)
385
+ )
386
+
387
+ cache_manager = None
388
+ cache_key = None
389
+ data_type = None
390
+
391
+ if use_cache:
392
+ # Initialize cache manager
393
+ cache_manager = UniversalCacheManager(cache_config)
394
+
395
+ # Create cache key from endpoint and params (human-readable)
396
+ # Determine data type based on endpoint (supports EOD + INTRADAY for both STOCK + OPTIONS)
397
+ is_intraday = 'intraday' in endpoint
398
+ is_options = 'options' in endpoint
399
+ is_stock = 'stock' in endpoint
400
+
401
+ if is_intraday and is_options:
402
+ # Intraday options data: /equities/intraday/options-rawiv
403
+ data_type = 'options_intraday'
404
+ symbol = kwargs.get('symbol', 'UNKNOWN')
405
+ date = kwargs.get('date', 'UNKNOWN')
406
+ cache_key = f"{symbol}_{date}"
407
+ elif is_intraday and is_stock:
408
+ # Intraday stock data: /equities/intraday/stock-prices
409
+ data_type = 'stock_intraday'
410
+ symbol = kwargs.get('symbol', 'UNKNOWN')
411
+ date = kwargs.get('date', 'UNKNOWN')
412
+ cache_key = f"{symbol}_{date}"
413
+ elif is_options:
414
+ # EOD options data: /equities/eod/options-rawiv
415
+ data_type = 'options_eod'
416
+ symbol = kwargs.get('symbol', 'UNKNOWN')
417
+ from_date = kwargs.get('from_', kwargs.get('date', 'UNKNOWN'))
418
+ to_date = kwargs.get('to', from_date)
419
+ if from_date != to_date:
420
+ cache_key = f"{symbol}_{from_date}_{to_date}"
421
+ else:
422
+ cache_key = f"{symbol}_{from_date}"
423
+ elif is_stock:
424
+ # EOD stock data: /equities/eod/stock-prices
425
+ data_type = 'stock_eod'
426
+ symbol = kwargs.get('symbol', 'UNKNOWN')
427
+ from_date = kwargs.get('from_', kwargs.get('date', 'UNKNOWN'))
428
+ to_date = kwargs.get('to', from_date)
429
+ if from_date != to_date:
430
+ cache_key = f"{symbol}_{from_date}_{to_date}"
431
+ else:
432
+ cache_key = f"{symbol}_{from_date}"
433
+ else:
434
+ # Fallback for other endpoints
435
+ sorted_params = sorted([(k, v) for k, v in kwargs.items()])
436
+ param_hash = abs(hash(str(sorted_params)))
437
+ cache_key = f"{endpoint.replace('/', '_')}_{param_hash}"
438
+ data_type = 'default'
439
+
440
+ # Try to get from cache
441
+ cached_data = cache_manager.get(cache_key, data_type)
442
+ if cached_data is not None:
443
+ if debug or cache_config.get('debug', False):
444
+ print(f"[CACHE] ✓ Cache hit: {endpoint} ({len(cached_data) if hasattr(cached_data, '__len__') else '?'} records)")
445
+ # Return in same format as API (dict with 'data' key)
446
+ if isinstance(cached_data, pd.DataFrame):
447
+ return {'data': cached_data.to_dict('records'), 'status': 'success'}
448
+ return cached_data
449
+
450
+ # Cache miss or caching disabled - make API call
451
+ if debug and APIManager._api_key:
452
+ base_url = "https://restapi.ivolatility.com"
453
+ url_params = {}
454
+ for key, value in kwargs.items():
455
+ clean_key = key.rstrip('_') if key.endswith('_') else key
456
+ url_params[clean_key] = value
457
+
458
+ params_str = "&".join([f"{k}={v}" for k, v in url_params.items()])
459
+ full_url = f"{base_url}{endpoint}?apiKey={APIManager._api_key}&{params_str}"
460
+ print(f"\n[API] Full URL:")
461
+ print(f"[API] {full_url}\n")
462
+
463
+ method = APIManager.get_method(endpoint)
464
+ response = method(**kwargs)
465
+
466
+ normalized = APIHelper.normalize_response(response, debug=debug)
467
+
468
+ if normalized is None and debug:
469
+ print(f"[api_call] Failed to get data")
470
+ print(f"[api_call] Endpoint: {endpoint}")
471
+ print(f"[api_call] Params: {kwargs}")
472
+
473
+ # Save to cache if enabled and data is valid
474
+ if use_cache and normalized is not None and cache_manager is not None:
475
+ # Convert dict response to DataFrame for caching
476
+ if isinstance(normalized, dict) and 'data' in normalized:
477
+ try:
478
+ cache_data = pd.DataFrame(normalized['data'])
479
+ if len(cache_data) > 0: # Only cache non-empty data
480
+ cache_manager.set(cache_key, cache_data, data_type)
481
+ if debug or cache_config.get('debug', False):
482
+ print(f"[CACHE] 💾 Saved to cache: {endpoint} ({len(cache_data)} records)")
483
+ else:
484
+ if debug or cache_config.get('debug', False):
485
+ print(f"[CACHE] ⚠️ Skipped caching empty data: {endpoint}")
486
+ except Exception as e:
487
+ if debug or cache_config.get('debug', False):
488
+ print(f"[CACHE] ❌ Error converting to cache format: {e}")
489
+
490
+ return normalized
491
+
492
+ except Exception as e:
493
+ if debug:
494
+ print(f"[api_call] Exception: {e}")
495
+ print(f"[api_call] Endpoint: {endpoint}")
496
+ print(f"[api_call] Params: {kwargs}")
497
+ return None
498
+
499
+
500
+ # ============================================================
501
+ # BACKTEST RESULTS
502
+ # ============================================================
503
+ class BacktestResults:
504
+ """Universal container for backtest results"""
505
+
506
+ def __init__(self, equity_curve, equity_dates, trades, initial_capital,
507
+ config, benchmark_prices=None, benchmark_symbol='SPY',
508
+ daily_returns=None, debug_info=None):
509
+
510
+ self.equity_curve = equity_curve
511
+ self.equity_dates = equity_dates
512
+ self.trades = trades
513
+ self.initial_capital = initial_capital
514
+ self.final_capital = equity_curve[-1] if len(equity_curve) > 0 else initial_capital
515
+ self.config = config
516
+ self.benchmark_prices = benchmark_prices
517
+ self.benchmark_symbol = benchmark_symbol
518
+ self.debug_info = debug_info if debug_info else []
519
+
520
+ if daily_returns is None and len(equity_curve) > 1:
521
+ self.daily_returns = [
522
+ (equity_curve[i] - equity_curve[i-1]) / equity_curve[i-1]
523
+ for i in range(1, len(equity_curve))
524
+ ]
525
+ else:
526
+ self.daily_returns = daily_returns if daily_returns else []
527
+
528
+ self.max_drawdown = self._calculate_max_drawdown()
529
+
530
+ def _calculate_max_drawdown(self):
531
+ if len(self.equity_curve) < 2:
532
+ return 0
533
+ running_max = np.maximum.accumulate(self.equity_curve)
534
+ drawdowns = (np.array(self.equity_curve) - running_max) / running_max * 100
535
+ return abs(np.min(drawdowns))
536
+
537
+
538
+ # ============================================================
539
+ # STOP-LOSS MANAGER (ENHANCED VERSION WITH COMBINED STOP)
540
+ # ============================================================
541
+ class StopLossManager:
542
+ """
543
+ Enhanced stop-loss manager with COMBINED STOP support
544
+
545
+ NEW STOP TYPE:
546
+ - combined: Requires BOTH pl_loss AND directional conditions (from code 2)
547
+ """
548
+
549
+ def __init__(self):
550
+ self.positions = {}
551
+
552
+ def add_position(self, position_id, entry_price, entry_date, stop_type='fixed_pct',
553
+ stop_value=0.05, atr=None, trailing_distance=None, use_pnl_pct=False,
554
+ is_short_bias=False, **kwargs):
555
+ """
556
+ Add position with stop-loss
557
+
558
+ NEW for combined stop:
559
+ stop_type='combined'
560
+ stop_value={'pl_loss': 0.05, 'directional': 0.03}
561
+ """
562
+ self.positions[position_id] = {
563
+ 'entry_price': entry_price,
564
+ 'entry_date': entry_date,
565
+ 'stop_type': stop_type,
566
+ 'stop_value': stop_value,
567
+ 'atr': atr,
568
+ 'trailing_distance': trailing_distance,
569
+ 'highest_price': entry_price if not use_pnl_pct else 0,
570
+ 'lowest_price': entry_price if not use_pnl_pct else 0,
571
+ 'max_profit': 0,
572
+ 'use_pnl_pct': use_pnl_pct,
573
+ 'is_short_bias': is_short_bias,
574
+ **kwargs # Store additional parameters for combined stop
575
+ }
576
+
577
+ def check_stop(self, position_id, current_price, current_date, position_type='LONG', **kwargs):
578
+ """
579
+ Check if stop-loss triggered
580
+
581
+ NEW: Supports 'combined' stop type
582
+ """
583
+ if position_id not in self.positions:
584
+ return False, None, None
585
+
586
+ pos = self.positions[position_id]
587
+ stop_type = pos['stop_type']
588
+ use_pnl_pct = pos.get('use_pnl_pct', False)
589
+
590
+ # Update tracking
591
+ if use_pnl_pct:
592
+ pnl_pct = current_price
593
+ pos['highest_price'] = max(pos['highest_price'], pnl_pct)
594
+ pos['lowest_price'] = min(pos['lowest_price'], pnl_pct)
595
+ pos['max_profit'] = max(pos['max_profit'], pnl_pct)
596
+ else:
597
+ if position_type == 'LONG':
598
+ pos['highest_price'] = max(pos['highest_price'], current_price)
599
+ current_profit = current_price - pos['entry_price']
600
+ else:
601
+ pos['lowest_price'] = min(pos['lowest_price'], current_price)
602
+ current_profit = pos['entry_price'] - current_price
603
+
604
+ pos['max_profit'] = max(pos['max_profit'], current_profit)
605
+
606
+ # Route to appropriate check method
607
+ if stop_type == 'fixed_pct':
608
+ if use_pnl_pct:
609
+ return self._check_fixed_pct_stop_pnl(pos, current_price)
610
+ else:
611
+ return self._check_fixed_pct_stop(pos, current_price, position_type)
612
+
613
+ elif stop_type == 'trailing':
614
+ if use_pnl_pct:
615
+ return self._check_trailing_stop_pnl(pos, current_price)
616
+ else:
617
+ return self._check_trailing_stop(pos, current_price, position_type)
618
+
619
+ elif stop_type == 'time_based':
620
+ return self._check_time_stop(pos, current_date)
621
+
622
+ elif stop_type == 'volatility':
623
+ return self._check_volatility_stop(pos, current_price, position_type)
624
+
625
+ elif stop_type == 'pl_loss':
626
+ return self._check_pl_loss_stop(pos, kwargs)
627
+
628
+ elif stop_type == 'directional':
629
+ return self._check_directional_stop(pos, kwargs)
630
+
631
+ # NEW: COMBINED STOP (requires BOTH conditions)
632
+ elif stop_type == 'combined':
633
+ return self._check_combined_stop(pos, kwargs)
634
+
635
+ else:
636
+ return False, None, None
637
+
638
+ # ========================================================
639
+ # EXISTING STOP METHODS (unchanged)
640
+ # ========================================================
641
+
642
+ def _check_fixed_pct_stop(self, pos, current_price, position_type):
643
+ """Fixed percentage stop-loss (price-based)"""
644
+ entry = pos['entry_price']
645
+ stop_pct = pos['stop_value']
646
+
647
+ if position_type == 'LONG':
648
+ stop_level = entry * (1 - stop_pct)
649
+ triggered = current_price <= stop_level
650
+ else:
651
+ stop_level = entry * (1 + stop_pct)
652
+ triggered = current_price >= stop_level
653
+
654
+ return triggered, stop_level, 'fixed_pct'
655
+
656
+ def _check_fixed_pct_stop_pnl(self, pos, pnl_pct):
657
+ """Fixed percentage stop-loss (P&L%-based for options)"""
658
+ stop_pct = pos['stop_value']
659
+ stop_level = -stop_pct * 100
660
+
661
+ triggered = pnl_pct <= stop_level
662
+
663
+ return triggered, stop_level, 'fixed_pct'
664
+
665
+ def _check_trailing_stop(self, pos, current_price, position_type):
666
+ """Trailing stop-loss (price-based)"""
667
+ if pos['trailing_distance'] is None:
668
+ pos['trailing_distance'] = pos['stop_value']
669
+
670
+ distance = pos['trailing_distance']
671
+
672
+ if position_type == 'LONG':
673
+ stop_level = pos['highest_price'] * (1 - distance)
674
+ triggered = current_price <= stop_level
675
+ else:
676
+ stop_level = pos['lowest_price'] * (1 + distance)
677
+ triggered = current_price >= stop_level
678
+
679
+ return triggered, stop_level, 'trailing'
680
+
681
+ def _check_trailing_stop_pnl(self, pos, pnl_pct):
682
+ """Trailing stop-loss (P&L%-based for options)"""
683
+ if pos['trailing_distance'] is None:
684
+ pos['trailing_distance'] = pos['stop_value']
685
+
686
+ distance = pos['trailing_distance'] * 100
687
+
688
+ stop_level = pos['highest_price'] - distance
689
+
690
+ triggered = pnl_pct <= stop_level
691
+
692
+ return triggered, stop_level, 'trailing'
693
+
694
+ def _check_time_stop(self, pos, current_date):
695
+ """Time-based stop"""
696
+ days_held = (current_date - pos['entry_date']).days
697
+ max_days = pos['stop_value']
698
+
699
+ triggered = days_held >= max_days
700
+ return triggered, None, 'time_based'
701
+
702
+ def _check_volatility_stop(self, pos, current_price, position_type):
703
+ """ATR-based stop"""
704
+ if pos['atr'] is None:
705
+ return False, None, None
706
+
707
+ entry = pos['entry_price']
708
+ atr_multiplier = pos['stop_value']
709
+ stop_distance = pos['atr'] * atr_multiplier
710
+
711
+ if position_type == 'LONG':
712
+ stop_level = entry - stop_distance
713
+ triggered = current_price <= stop_level
714
+ else:
715
+ stop_level = entry + stop_distance
716
+ triggered = current_price >= stop_level
717
+
718
+ return triggered, stop_level, 'volatility'
719
+
720
+ def _check_pl_loss_stop(self, pos, kwargs):
721
+ """Stop-loss based on actual P&L"""
722
+ pnl_pct = kwargs.get('pnl_pct')
723
+
724
+ if pnl_pct is None:
725
+ current_pnl = kwargs.get('current_pnl', 0)
726
+ total_cost = kwargs.get('total_cost', pos.get('total_cost', 1))
727
+
728
+ if total_cost > 0:
729
+ pnl_pct = (current_pnl / total_cost) * 100
730
+ else:
731
+ pnl_pct = 0
732
+
733
+ stop_threshold = -pos['stop_value'] * 100
734
+ triggered = pnl_pct <= stop_threshold
735
+
736
+ return triggered, stop_threshold, 'pl_loss'
737
+
738
+ def _check_directional_stop(self, pos, kwargs):
739
+ """Stop-loss based on underlying price movement"""
740
+ underlying_change_pct = kwargs.get('underlying_change_pct')
741
+
742
+ if underlying_change_pct is None:
743
+ current = kwargs.get('underlying_price')
744
+ entry = kwargs.get('underlying_entry_price', pos.get('underlying_entry_price'))
745
+
746
+ if current is not None and entry is not None and entry != 0:
747
+ underlying_change_pct = ((current - entry) / entry) * 100
748
+ else:
749
+ underlying_change_pct = 0
750
+
751
+ threshold = pos['stop_value'] * 100
752
+ is_short_bias = pos.get('is_short_bias', False)
753
+
754
+ if is_short_bias:
755
+ triggered = underlying_change_pct >= threshold
756
+ else:
757
+ triggered = underlying_change_pct <= -threshold
758
+
759
+ return triggered, threshold, 'directional'
760
+
761
+ # ========================================================
762
+ # NEW: COMBINED STOP (REQUIRES BOTH CONDITIONS)
763
+ # ========================================================
764
+
765
+ def _check_combined_stop(self, pos, kwargs):
766
+ """
767
+ Combined stop: Requires BOTH pl_loss AND directional conditions
768
+
769
+ This is the key feature from code 2:
770
+ - Must have P&L loss > threshold
771
+ - AND underlying must move adversely > threshold
772
+
773
+ Args:
774
+ pos: Position dict with stop_value = {'pl_loss': 0.05, 'directional': 0.03}
775
+ kwargs: Must contain pnl_pct and underlying_change_pct
776
+
777
+ Returns:
778
+ tuple: (triggered, thresholds_dict, 'combined')
779
+ """
780
+ stop_config = pos['stop_value']
781
+
782
+ if not isinstance(stop_config, dict):
783
+ # Fallback: treat as simple fixed stop
784
+ return False, None, 'combined'
785
+
786
+ pl_threshold = stop_config.get('pl_loss', 0.05)
787
+ dir_threshold = stop_config.get('directional', 0.03)
788
+
789
+ # Check P&L condition
790
+ pnl_pct = kwargs.get('pnl_pct', 0)
791
+ is_losing = pnl_pct <= (-pl_threshold * 100)
792
+
793
+ # Check directional condition
794
+ underlying_change_pct = kwargs.get('underlying_change_pct')
795
+
796
+ if underlying_change_pct is None:
797
+ current = kwargs.get('underlying_price')
798
+ entry = kwargs.get('underlying_entry_price', pos.get('underlying_entry_price'))
799
+
800
+ if current is not None and entry is not None and entry != 0:
801
+ underlying_change_pct = ((current - entry) / entry) * 100
802
+ else:
803
+ underlying_change_pct = 0
804
+
805
+ is_short_bias = pos.get('is_short_bias', False)
806
+
807
+ if is_short_bias:
808
+ # Bearish position: adverse move is UP
809
+ adverse_move = underlying_change_pct >= (dir_threshold * 100)
810
+ else:
811
+ # Bullish position: adverse move is DOWN
812
+ adverse_move = underlying_change_pct <= (-dir_threshold * 100)
813
+
814
+ # CRITICAL: Both conditions must be true
815
+ triggered = is_losing and adverse_move
816
+
817
+ # Return detailed thresholds for reporting
818
+ thresholds = {
819
+ 'pl_threshold': -pl_threshold * 100,
820
+ 'dir_threshold': dir_threshold * 100,
821
+ 'actual_pnl_pct': pnl_pct,
822
+ 'actual_underlying_change': underlying_change_pct,
823
+ 'pl_condition': is_losing,
824
+ 'dir_condition': adverse_move
825
+ }
826
+
827
+ return triggered, thresholds, 'combined'
828
+
829
+ # ========================================================
830
+ # UTILITY METHODS
831
+ # ========================================================
832
+
833
+ def remove_position(self, position_id):
834
+ """Remove position from tracking"""
835
+ if position_id in self.positions:
836
+ del self.positions[position_id]
837
+
838
+ def get_position_info(self, position_id):
839
+ """Get position stop-loss info"""
840
+ if position_id not in self.positions:
841
+ return None
842
+
843
+ pos = self.positions[position_id]
844
+ return {
845
+ 'stop_type': pos['stop_type'],
846
+ 'stop_value': pos['stop_value'],
847
+ 'max_profit_before_stop': pos['max_profit']
848
+ }
849
+
850
+
851
+ # ============================================================
852
+ # POSITION MANAGER (unchanged but compatible with combined stop)
853
+ # ============================================================
854
+ class PositionManager:
855
+ """Universal Position Manager with automatic mode detection"""
856
+
857
+ def __init__(self, config, debug=False):
858
+ self.positions = {}
859
+ self.closed_trades = []
860
+ self.config = config
861
+ self.debug = debug
862
+
863
+ # Stop-loss enable logic:
864
+ # 1) Respect explicit flag if provided
865
+ # 2) Otherwise infer from stop_loss_config.enabled for convenience
866
+ explicit_flag = config.get('stop_loss_enabled')
867
+ sl_cfg = config.get('stop_loss_config', {})
868
+ inferred_flag = bool(sl_cfg.get('enabled', False))
869
+
870
+ self.sl_enabled = explicit_flag if explicit_flag is not None else inferred_flag
871
+
872
+ if self.sl_enabled:
873
+ self.sl_config = sl_cfg
874
+ self.sl_manager = StopLossManager()
875
+ else:
876
+ self.sl_config = None
877
+ self.sl_manager = None
878
+
879
+ def open_position(self, position_id, symbol, entry_date, entry_price,
880
+ quantity, position_type='LONG', **kwargs):
881
+ """Open position with automatic stop-loss"""
882
+
883
+ if entry_price == 0 and self.sl_enabled:
884
+ if 'total_cost' not in kwargs or kwargs['total_cost'] == 0:
885
+ raise ValueError(
886
+ f"\n{'='*70}\n"
887
+ f"ERROR: P&L% mode requires 'total_cost' parameter\n"
888
+ f"{'='*70}\n"
889
+ )
890
+
891
+ position = {
892
+ 'id': position_id,
893
+ 'symbol': symbol,
894
+ 'entry_date': entry_date,
895
+ 'entry_price': entry_price,
896
+ 'quantity': quantity,
897
+ 'type': position_type,
898
+ 'highest_price': entry_price,
899
+ 'lowest_price': entry_price,
900
+ **kwargs
901
+ }
902
+
903
+ self.positions[position_id] = position
904
+
905
+ if self.sl_enabled and self.sl_manager:
906
+ sl_type = self.sl_config.get('type', 'fixed_pct')
907
+ sl_value = self.sl_config.get('value', 0.05)
908
+
909
+ use_pnl_pct = (entry_price == 0)
910
+ is_short_bias = kwargs.get('is_short_bias', False)
911
+
912
+ # Pass underlying_entry_price for combined stop
913
+ self.sl_manager.add_position(
914
+ position_id=position_id,
915
+ entry_price=entry_price,
916
+ entry_date=entry_date,
917
+ stop_type=sl_type,
918
+ stop_value=sl_value,
919
+ atr=kwargs.get('atr', None),
920
+ trailing_distance=self.sl_config.get('trailing_distance', None),
921
+ use_pnl_pct=use_pnl_pct,
922
+ is_short_bias=is_short_bias,
923
+ underlying_entry_price=kwargs.get('entry_stock_price') # For combined stop
924
+ )
925
+
926
+ if self.debug:
927
+ mode = "P&L%" if entry_price == 0 else "Price"
928
+ bias = " (SHORT BIAS)" if kwargs.get('is_short_bias') else ""
929
+ print(f"[PositionManager] OPEN {position_id}: {symbol} @ {entry_price} (Mode: {mode}{bias})")
930
+
931
+ return position
932
+
933
+ def check_positions(self, current_date, price_data):
934
+ """Check all positions for stop-loss triggers"""
935
+ if not self.sl_enabled:
936
+ return []
937
+
938
+ to_close = []
939
+
940
+ for position_id, position in self.positions.items():
941
+ if position_id not in price_data:
942
+ continue
943
+
944
+ if isinstance(price_data[position_id], dict):
945
+ data = price_data[position_id]
946
+ current_price = data.get('price', position['entry_price'])
947
+ current_pnl = data.get('pnl', 0)
948
+ current_pnl_pct = data.get('pnl_pct', 0)
949
+
950
+ # NEW: Pass underlying data for combined stop
951
+ underlying_price = data.get('underlying_price')
952
+ underlying_entry_price = data.get('underlying_entry_price')
953
+ underlying_change_pct = data.get('underlying_change_pct')
954
+ else:
955
+ current_price = price_data[position_id]
956
+ current_pnl = (current_price - position['entry_price']) * position['quantity']
957
+ current_pnl_pct = (current_price - position['entry_price']) / position['entry_price'] if position['entry_price'] != 0 else 0
958
+ underlying_price = None
959
+ underlying_entry_price = None
960
+ underlying_change_pct = None
961
+
962
+ position['highest_price'] = max(position['highest_price'], current_price)
963
+ position['lowest_price'] = min(position['lowest_price'], current_price)
964
+
965
+ if position['entry_price'] == 0:
966
+ check_value = current_pnl_pct
967
+ else:
968
+ check_value = current_price
969
+
970
+ # Pass all data to stop manager
971
+ stop_kwargs = {
972
+ 'pnl_pct': current_pnl_pct,
973
+ 'current_pnl': current_pnl,
974
+ 'total_cost': position.get('total_cost', 1),
975
+ 'underlying_price': underlying_price,
976
+ 'underlying_entry_price': underlying_entry_price or position.get('entry_stock_price'),
977
+ 'underlying_change_pct': underlying_change_pct
978
+ }
979
+
980
+ triggered, stop_level, stop_type = self.sl_manager.check_stop(
981
+ position_id=position_id,
982
+ current_price=check_value,
983
+ current_date=current_date,
984
+ position_type=position['type'],
985
+ **stop_kwargs
986
+ )
987
+
988
+ if triggered:
989
+ to_close.append({
990
+ 'position_id': position_id,
991
+ 'symbol': position['symbol'],
992
+ 'stop_type': stop_type,
993
+ 'stop_level': stop_level,
994
+ 'current_price': current_price,
995
+ 'pnl': current_pnl,
996
+ 'pnl_pct': current_pnl_pct
997
+ })
998
+
999
+ if self.debug:
1000
+ mode = "P&L%" if position['entry_price'] == 0 else "Price"
1001
+ print(f"[PositionManager] STOP-LOSS: {position_id} ({stop_type}, {mode}) @ {check_value:.2f}")
1002
+
1003
+ return to_close
1004
+
1005
+ def close_position(self, position_id, exit_date, exit_price,
1006
+ close_reason='manual', pnl=None, **kwargs):
1007
+ """Close position"""
1008
+ if position_id not in self.positions:
1009
+ if self.debug:
1010
+ print(f"[PositionManager] WARNING: Position {position_id} not found")
1011
+ return None
1012
+
1013
+ position = self.positions.pop(position_id)
1014
+
1015
+ if pnl is None:
1016
+ pnl = (exit_price - position['entry_price']) * position['quantity']
1017
+
1018
+ if position['entry_price'] != 0:
1019
+ pnl_pct = (exit_price - position['entry_price']) / position['entry_price'] * 100
1020
+ else:
1021
+ if 'total_cost' in position and position['total_cost'] != 0:
1022
+ pnl_pct = (pnl / position['total_cost']) * 100
1023
+ elif 'total_cost' in kwargs and kwargs['total_cost'] != 0:
1024
+ pnl_pct = (pnl / kwargs['total_cost']) * 100
1025
+ else:
1026
+ pnl_pct = 0.0
1027
+
1028
+ trade = {
1029
+ 'entry_date': position['entry_date'],
1030
+ 'exit_date': exit_date,
1031
+ 'symbol': position['symbol'],
1032
+ 'signal': position['type'],
1033
+ 'entry_price': position['entry_price'],
1034
+ 'exit_price': exit_price,
1035
+ 'quantity': position['quantity'],
1036
+ 'pnl': pnl,
1037
+ 'return_pct': pnl_pct,
1038
+ 'exit_reason': close_reason,
1039
+ 'stop_type': self.sl_config.get('type', 'none') if self.sl_enabled else 'none',
1040
+ **kwargs
1041
+ }
1042
+
1043
+ for key in ['call_strike', 'put_strike', 'expiration', 'contracts',
1044
+ 'short_strike', 'long_strike', 'opt_type', 'spread_type',
1045
+ 'entry_z_score', 'is_short_bias', 'entry_lean', 'exit_lean',
1046
+ 'call_iv_entry', 'put_iv_entry', 'iv_lean_entry']:
1047
+ if key in position:
1048
+ trade[key] = position[key]
1049
+
1050
+ for key in ['short_entry_bid', 'short_entry_ask', 'short_entry_mid',
1051
+ 'long_entry_bid', 'long_entry_ask', 'long_entry_mid',
1052
+ 'underlying_entry_price']:
1053
+ if key in position:
1054
+ trade[key] = position[key]
1055
+
1056
+ for key in ['short_exit_bid', 'short_exit_ask',
1057
+ 'long_exit_bid', 'long_exit_ask',
1058
+ 'underlying_exit_price', 'underlying_change_pct',
1059
+ 'stop_threshold', 'actual_value',
1060
+ 'call_iv_exit', 'put_iv_exit', 'iv_lean_exit',
1061
+ 'spy_intraday_high', 'spy_intraday_low', 'spy_intraday_close',
1062
+ 'spy_stop_trigger_time', 'spy_stop_trigger_price',
1063
+ 'spy_stop_trigger_bid', 'spy_stop_trigger_ask', 'spy_stop_trigger_last',
1064
+ 'intraday_data_points', 'intraday_data_available', 'stop_triggered_by']:
1065
+ if key in kwargs:
1066
+ trade[key] = kwargs[key]
1067
+
1068
+ self.closed_trades.append(trade)
1069
+
1070
+ if self.sl_enabled and self.sl_manager:
1071
+ self.sl_manager.remove_position(position_id)
1072
+
1073
+ if self.debug:
1074
+ print(f"[PositionManager] CLOSE {position_id}: P&L=${pnl:.2f} ({pnl_pct:.2f}%) - {close_reason}")
1075
+
1076
+ return trade
1077
+
1078
+ def get_open_positions(self):
1079
+ return list(self.positions.values())
1080
+
1081
+ def get_closed_trades(self):
1082
+ return self.closed_trades
1083
+
1084
+ def close_all_positions(self, final_date, price_data, reason='end_of_backtest'):
1085
+ """Close all open positions at end of backtest"""
1086
+ for position_id in list(self.positions.keys()):
1087
+ if position_id in price_data:
1088
+ position = self.positions[position_id]
1089
+
1090
+ if isinstance(price_data[position_id], dict):
1091
+ data = price_data[position_id]
1092
+ exit_price = data.get('price', position['entry_price'])
1093
+ pnl = data.get('pnl', None)
1094
+ else:
1095
+ exit_price = price_data[position_id]
1096
+ pnl = None
1097
+
1098
+ if pnl is None and position['entry_price'] == 0:
1099
+ if isinstance(price_data[position_id], dict) and 'pnl' in price_data[position_id]:
1100
+ pnl = price_data[position_id]['pnl']
1101
+
1102
+ self.close_position(
1103
+ position_id=position_id,
1104
+ exit_date=final_date,
1105
+ exit_price=exit_price,
1106
+ close_reason=reason,
1107
+ pnl=pnl
1108
+ )
1109
+
1110
+
1111
+ # ============================================================
1112
+ # BACKTEST ANALYZER (unchanged)
1113
+ # ============================================================
1114
+ class BacktestAnalyzer:
1115
+ """Calculate all metrics from BacktestResults"""
1116
+
1117
+ def __init__(self, results):
1118
+ self.results = results
1119
+ self.metrics = {}
1120
+
1121
+ def calculate_all_metrics(self):
1122
+ r = self.results
1123
+
1124
+ self.metrics['initial_capital'] = r.initial_capital
1125
+ self.metrics['final_equity'] = r.final_capital
1126
+
1127
+ self.metrics['total_pnl'] = r.final_capital - r.initial_capital
1128
+ self.metrics['total_return'] = (self.metrics['total_pnl'] / r.initial_capital) * 100
1129
+
1130
+ if len(r.equity_dates) > 0:
1131
+ start_date = min(r.equity_dates)
1132
+ end_date = max(r.equity_dates)
1133
+ days_diff = (end_date - start_date).days
1134
+
1135
+ if days_diff <= 0:
1136
+ self.metrics['cagr'] = 0
1137
+ self.metrics['show_cagr'] = False
1138
+ else:
1139
+ years = days_diff / 365.25
1140
+ if years >= 1.0:
1141
+ self.metrics['cagr'] = ((r.final_capital / r.initial_capital) ** (1/years) - 1) * 100
1142
+ self.metrics['show_cagr'] = True
1143
+ else:
1144
+ self.metrics['cagr'] = self.metrics['total_return'] * (365.25 / days_diff)
1145
+ self.metrics['show_cagr'] = False
1146
+ else:
1147
+ self.metrics['cagr'] = 0
1148
+ self.metrics['show_cagr'] = False
1149
+
1150
+ self.metrics['sharpe'] = self._sharpe_ratio(r.daily_returns)
1151
+ self.metrics['sortino'] = self._sortino_ratio(r.daily_returns)
1152
+ self.metrics['max_drawdown'] = r.max_drawdown
1153
+ self.metrics['volatility'] = np.std(r.daily_returns) * np.sqrt(252) * 100 if len(r.daily_returns) > 0 else 0
1154
+ self.metrics['calmar'] = abs(self.metrics['total_return'] / r.max_drawdown) if r.max_drawdown > 0 else 0
1155
+ self.metrics['omega'] = self._omega_ratio(r.daily_returns)
1156
+ self.metrics['ulcer'] = self._ulcer_index(r.equity_curve)
1157
+
1158
+ self.metrics['var_95'], self.metrics['var_95_pct'] = self._calculate_var(r.daily_returns, 0.95)
1159
+ self.metrics['var_99'], self.metrics['var_99_pct'] = self._calculate_var(r.daily_returns, 0.99)
1160
+ self.metrics['cvar_95'], self.metrics['cvar_95_pct'] = self._calculate_cvar(r.daily_returns, 0.95)
1161
+
1162
+ avg_equity = np.mean(r.equity_curve) if len(r.equity_curve) > 0 else r.initial_capital
1163
+ self.metrics['var_95_dollar'] = self.metrics['var_95'] * avg_equity
1164
+ self.metrics['var_99_dollar'] = self.metrics['var_99'] * avg_equity
1165
+ self.metrics['cvar_95_dollar'] = self.metrics['cvar_95'] * avg_equity
1166
+
1167
+ self.metrics['tail_ratio'] = self._tail_ratio(r.daily_returns)
1168
+ self.metrics['skewness'], self.metrics['kurtosis'] = self._skewness_kurtosis(r.daily_returns)
1169
+
1170
+ self.metrics['alpha'], self.metrics['beta'], self.metrics['r_squared'] = self._alpha_beta(r)
1171
+
1172
+ if len(r.trades) > 0:
1173
+ self._calculate_trading_stats(r.trades)
1174
+ else:
1175
+ self._set_empty_trading_stats()
1176
+
1177
+ running_max = np.maximum.accumulate(r.equity_curve)
1178
+ max_dd_dollars = np.min(np.array(r.equity_curve) - running_max)
1179
+ self.metrics['recovery_factor'] = self.metrics['total_pnl'] / abs(max_dd_dollars) if max_dd_dollars != 0 else 0
1180
+
1181
+ if len(r.trades) > 0 and 'start_date' in r.config and 'end_date' in r.config:
1182
+ total_days = (pd.to_datetime(r.config['end_date']) - pd.to_datetime(r.config['start_date'])).days
1183
+ self.metrics['exposure_time'] = self._exposure_time(r.trades, total_days)
1184
+ else:
1185
+ self.metrics['exposure_time'] = 0
1186
+
1187
+ return self.metrics
1188
+
1189
+ def _calculate_trading_stats(self, trades):
1190
+ trades_df = pd.DataFrame(trades)
1191
+ winning = trades_df[trades_df['pnl'] > 0]
1192
+ losing = trades_df[trades_df['pnl'] <= 0]
1193
+
1194
+ self.metrics['total_trades'] = len(trades_df)
1195
+ self.metrics['winning_trades'] = len(winning)
1196
+ self.metrics['losing_trades'] = len(losing)
1197
+ self.metrics['win_rate'] = (len(winning) / len(trades_df)) * 100 if len(trades_df) > 0 else 0
1198
+
1199
+ wins_sum = winning['pnl'].sum() if len(winning) > 0 else 0
1200
+ losses_sum = abs(losing['pnl'].sum()) if len(losing) > 0 else 0
1201
+ self.metrics['profit_factor'] = wins_sum / losses_sum if losses_sum > 0 else float('inf')
1202
+
1203
+ self.metrics['avg_win'] = winning['pnl'].mean() if len(winning) > 0 else 0
1204
+ self.metrics['avg_loss'] = losing['pnl'].mean() if len(losing) > 0 else 0
1205
+ self.metrics['best_trade'] = trades_df['pnl'].max()
1206
+ self.metrics['worst_trade'] = trades_df['pnl'].min()
1207
+
1208
+ if len(winning) > 0 and len(losing) > 0 and self.metrics['avg_loss'] != 0:
1209
+ self.metrics['avg_win_loss_ratio'] = abs(self.metrics['avg_win'] / self.metrics['avg_loss'])
1210
+ else:
1211
+ self.metrics['avg_win_loss_ratio'] = 0
1212
+
1213
+ self.metrics['max_win_streak'], self.metrics['max_loss_streak'] = self._win_loss_streaks(trades)
1214
+
1215
+ def _set_empty_trading_stats(self):
1216
+ self.metrics.update({
1217
+ 'total_trades': 0, 'winning_trades': 0, 'losing_trades': 0,
1218
+ 'win_rate': 0, 'profit_factor': 0, 'avg_win': 0, 'avg_loss': 0,
1219
+ 'best_trade': 0, 'worst_trade': 0, 'avg_win_loss_ratio': 0,
1220
+ 'max_win_streak': 0, 'max_loss_streak': 0
1221
+ })
1222
+
1223
+ def _sharpe_ratio(self, returns):
1224
+ if len(returns) < 2:
1225
+ return 0
1226
+ return np.sqrt(252) * np.mean(returns) / np.std(returns) if np.std(returns) > 0 else 0
1227
+
1228
+ def _sortino_ratio(self, returns):
1229
+ if len(returns) < 2:
1230
+ return 0
1231
+ returns_array = np.array(returns)
1232
+ downside = returns_array[returns_array < 0]
1233
+ if len(downside) == 0 or np.std(downside) == 0:
1234
+ return 0
1235
+ return np.sqrt(252) * np.mean(returns_array) / np.std(downside)
1236
+
1237
+ def _omega_ratio(self, returns, threshold=0):
1238
+ if len(returns) < 2:
1239
+ return 0
1240
+ returns_array = np.array(returns)
1241
+ gains = np.sum(np.maximum(returns_array - threshold, 0))
1242
+ losses = np.sum(np.maximum(threshold - returns_array, 0))
1243
+ return gains / losses if losses > 0 else float('inf')
1244
+
1245
+ def _ulcer_index(self, equity_curve):
1246
+ if len(equity_curve) < 2:
1247
+ return 0
1248
+ equity_array = np.array(equity_curve)
1249
+ running_max = np.maximum.accumulate(equity_array)
1250
+ drawdown = (equity_array - running_max) / running_max
1251
+ return np.sqrt(np.mean(drawdown ** 2)) * 100
1252
+
1253
+ def _calculate_var(self, returns, confidence=0.95):
1254
+ if len(returns) < 10:
1255
+ return 0, 0
1256
+ returns_array = np.array(returns)
1257
+ returns_array = returns_array[~np.isnan(returns_array)]
1258
+ if len(returns_array) < 10:
1259
+ return 0, 0
1260
+ var_percentile = (1 - confidence) * 100
1261
+ var_return = np.percentile(returns_array, var_percentile)
1262
+ return var_return, var_return * 100
1263
+
1264
+ def _calculate_cvar(self, returns, confidence=0.95):
1265
+ if len(returns) < 10:
1266
+ return 0, 0
1267
+ returns_array = np.array(returns)
1268
+ returns_array = returns_array[~np.isnan(returns_array)]
1269
+ if len(returns_array) < 10:
1270
+ return 0, 0
1271
+ var_percentile = (1 - confidence) * 100
1272
+ var_threshold = np.percentile(returns_array, var_percentile)
1273
+ tail_losses = returns_array[returns_array <= var_threshold]
1274
+ if len(tail_losses) == 0:
1275
+ return 0, 0
1276
+ cvar_return = np.mean(tail_losses)
1277
+ return cvar_return, cvar_return * 100
1278
+
1279
+ def _tail_ratio(self, returns):
1280
+ if len(returns) < 20:
1281
+ return 0
1282
+ returns_array = np.array(returns)
1283
+ right = np.percentile(returns_array, 95)
1284
+ left = abs(np.percentile(returns_array, 5))
1285
+ return right / left if left > 0 else 0
1286
+
1287
+ def _skewness_kurtosis(self, returns):
1288
+ if len(returns) < 10:
1289
+ return 0, 0
1290
+ returns_array = np.array(returns)
1291
+ mean = np.mean(returns_array)
1292
+ std = np.std(returns_array)
1293
+ if std == 0:
1294
+ return 0, 0
1295
+ skew = np.mean(((returns_array - mean) / std) ** 3)
1296
+ kurt = np.mean(((returns_array - mean) / std) ** 4) - 3
1297
+ return skew, kurt
1298
+
1299
+ def _alpha_beta(self, results):
1300
+ if not hasattr(results, 'benchmark_prices') or not results.benchmark_prices:
1301
+ return 0, 0, 0
1302
+ if len(results.equity_dates) < 10:
1303
+ return 0, 0, 0
1304
+
1305
+ benchmark_returns = []
1306
+ sorted_dates = sorted(results.equity_dates)
1307
+
1308
+ for i in range(1, len(sorted_dates)):
1309
+ prev_date = sorted_dates[i-1]
1310
+ curr_date = sorted_dates[i]
1311
+
1312
+ if prev_date in results.benchmark_prices and curr_date in results.benchmark_prices:
1313
+ prev_price = results.benchmark_prices[prev_date]
1314
+ curr_price = results.benchmark_prices[curr_date]
1315
+ bench_return = (curr_price - prev_price) / prev_price
1316
+ benchmark_returns.append(bench_return)
1317
+ else:
1318
+ benchmark_returns.append(0)
1319
+
1320
+ if len(benchmark_returns) != len(results.daily_returns):
1321
+ return 0, 0, 0
1322
+
1323
+ port_ret = np.array(results.daily_returns)
1324
+ bench_ret = np.array(benchmark_returns)
1325
+
1326
+ bench_mean = np.mean(bench_ret)
1327
+ port_mean = np.mean(port_ret)
1328
+
1329
+ covariance = np.mean((bench_ret - bench_mean) * (port_ret - port_mean))
1330
+ benchmark_variance = np.mean((bench_ret - bench_mean) ** 2)
1331
+
1332
+ if benchmark_variance == 0:
1333
+ return 0, 0, 0
1334
+
1335
+ beta = covariance / benchmark_variance
1336
+ alpha_daily = port_mean - beta * bench_mean
1337
+ alpha_annualized = alpha_daily * 252 * 100
1338
+
1339
+ ss_res = np.sum((port_ret - (alpha_daily + beta * bench_ret)) ** 2)
1340
+ ss_tot = np.sum((port_ret - port_mean) ** 2)
1341
+ r_squared = 1 - (ss_res / ss_tot) if ss_tot > 0 else 0
1342
+
1343
+ return alpha_annualized, beta, r_squared
1344
+
1345
+ def _win_loss_streaks(self, trades):
1346
+ if len(trades) == 0:
1347
+ return 0, 0
1348
+ max_win = max_loss = current_win = current_loss = 0
1349
+ for trade in trades:
1350
+ if trade['pnl'] > 0:
1351
+ current_win += 1
1352
+ current_loss = 0
1353
+ max_win = max(max_win, current_win)
1354
+ else:
1355
+ current_loss += 1
1356
+ current_win = 0
1357
+ max_loss = max(max_loss, current_loss)
1358
+ return max_win, max_loss
1359
+
1360
+ def _exposure_time(self, trades, total_days):
1361
+ if total_days <= 0 or len(trades) == 0:
1362
+ return 0
1363
+ days_with_positions = set()
1364
+ for trade in trades:
1365
+ entry = pd.to_datetime(trade['entry_date'])
1366
+ exit_ = pd.to_datetime(trade['exit_date'])
1367
+ date_range = pd.date_range(start=entry, end=exit_, freq='D')
1368
+ days_with_positions.update(date_range.date)
1369
+ exposure_pct = (len(days_with_positions) / total_days) * 100
1370
+ return min(exposure_pct, 100.0)
1371
+
1372
+
1373
+ # ============================================================
1374
+ # STOP-LOSS METRICS (unchanged)
1375
+ # ============================================================
1376
+ def calculate_stoploss_metrics(analyzer):
1377
+ """Calculate stop-loss specific metrics"""
1378
+ if len(analyzer.results.trades) == 0:
1379
+ _set_empty_stoploss_metrics(analyzer)
1380
+ return analyzer.metrics
1381
+
1382
+ trades_df = pd.DataFrame(analyzer.results.trades)
1383
+
1384
+ if 'exit_reason' not in trades_df.columns:
1385
+ _set_empty_stoploss_metrics(analyzer)
1386
+ return analyzer.metrics
1387
+
1388
+ sl_trades = trades_df[trades_df['exit_reason'].str.contains('stop_loss', na=False)]
1389
+ profit_target_trades = trades_df[trades_df['exit_reason'] == 'profit_target']
1390
+
1391
+ analyzer.metrics['stoploss_count'] = len(sl_trades)
1392
+ analyzer.metrics['stoploss_pct'] = (len(sl_trades) / len(trades_df)) * 100 if len(trades_df) > 0 else 0
1393
+ analyzer.metrics['profit_target_count'] = len(profit_target_trades)
1394
+ analyzer.metrics['profit_target_pct'] = (len(profit_target_trades) / len(trades_df)) * 100 if len(trades_df) > 0 else 0
1395
+
1396
+ if len(sl_trades) > 0:
1397
+ analyzer.metrics['avg_stoploss_pnl'] = sl_trades['pnl'].mean()
1398
+ analyzer.metrics['total_stoploss_loss'] = sl_trades['pnl'].sum()
1399
+ analyzer.metrics['worst_stoploss'] = sl_trades['pnl'].min()
1400
+
1401
+ if 'return_pct' in sl_trades.columns:
1402
+ analyzer.metrics['avg_stoploss_return_pct'] = sl_trades['return_pct'].mean()
1403
+ else:
1404
+ analyzer.metrics['avg_stoploss_return_pct'] = 0
1405
+
1406
+ if 'entry_date' in sl_trades.columns and 'exit_date' in sl_trades.columns:
1407
+ sl_trades_copy = sl_trades.copy()
1408
+ sl_trades_copy['entry_date'] = pd.to_datetime(sl_trades_copy['entry_date'])
1409
+ sl_trades_copy['exit_date'] = pd.to_datetime(sl_trades_copy['exit_date'])
1410
+ sl_trades_copy['days_held'] = (sl_trades_copy['exit_date'] - sl_trades_copy['entry_date']).dt.days
1411
+ analyzer.metrics['avg_days_to_stoploss'] = sl_trades_copy['days_held'].mean()
1412
+ analyzer.metrics['min_days_to_stoploss'] = sl_trades_copy['days_held'].min()
1413
+ analyzer.metrics['max_days_to_stoploss'] = sl_trades_copy['days_held'].max()
1414
+ else:
1415
+ analyzer.metrics['avg_days_to_stoploss'] = 0
1416
+ analyzer.metrics['min_days_to_stoploss'] = 0
1417
+ analyzer.metrics['max_days_to_stoploss'] = 0
1418
+
1419
+ if 'stop_type' in sl_trades.columns:
1420
+ stop_types = sl_trades['stop_type'].value_counts().to_dict()
1421
+ analyzer.metrics['stoploss_by_type'] = stop_types
1422
+ else:
1423
+ analyzer.metrics['stoploss_by_type'] = {}
1424
+ else:
1425
+ analyzer.metrics['avg_stoploss_pnl'] = 0
1426
+ analyzer.metrics['total_stoploss_loss'] = 0
1427
+ analyzer.metrics['worst_stoploss'] = 0
1428
+ analyzer.metrics['avg_stoploss_return_pct'] = 0
1429
+ analyzer.metrics['avg_days_to_stoploss'] = 0
1430
+ analyzer.metrics['min_days_to_stoploss'] = 0
1431
+ analyzer.metrics['max_days_to_stoploss'] = 0
1432
+ analyzer.metrics['stoploss_by_type'] = {}
1433
+
1434
+ if len(profit_target_trades) > 0 and len(sl_trades) > 0:
1435
+ avg_profit_target = profit_target_trades['pnl'].mean()
1436
+ avg_stoploss = abs(sl_trades['pnl'].mean())
1437
+ analyzer.metrics['profit_to_loss_ratio'] = avg_profit_target / avg_stoploss if avg_stoploss > 0 else 0
1438
+ else:
1439
+ analyzer.metrics['profit_to_loss_ratio'] = 0
1440
+
1441
+ if 'max_profit_before_stop' in sl_trades.columns:
1442
+ early_exits = sl_trades[sl_trades['max_profit_before_stop'] > 0]
1443
+ analyzer.metrics['early_exit_count'] = len(early_exits)
1444
+ analyzer.metrics['early_exit_pct'] = (len(early_exits) / len(sl_trades)) * 100 if len(sl_trades) > 0 else 0
1445
+ if len(early_exits) > 0:
1446
+ analyzer.metrics['avg_missed_profit'] = early_exits['max_profit_before_stop'].mean()
1447
+ else:
1448
+ analyzer.metrics['avg_missed_profit'] = 0
1449
+ else:
1450
+ analyzer.metrics['early_exit_count'] = 0
1451
+ analyzer.metrics['early_exit_pct'] = 0
1452
+ analyzer.metrics['avg_missed_profit'] = 0
1453
+
1454
+ exit_reasons = trades_df['exit_reason'].value_counts().to_dict()
1455
+ analyzer.metrics['exit_reasons'] = exit_reasons
1456
+
1457
+ return analyzer.metrics
1458
+
1459
+
1460
+ def _set_empty_stoploss_metrics(analyzer):
1461
+ analyzer.metrics.update({
1462
+ 'stoploss_count': 0, 'stoploss_pct': 0,
1463
+ 'profit_target_count': 0, 'profit_target_pct': 0,
1464
+ 'avg_stoploss_pnl': 0, 'total_stoploss_loss': 0,
1465
+ 'worst_stoploss': 0, 'avg_stoploss_return_pct': 0,
1466
+ 'avg_days_to_stoploss': 0, 'min_days_to_stoploss': 0,
1467
+ 'max_days_to_stoploss': 0, 'stoploss_by_type': {},
1468
+ 'profit_to_loss_ratio': 0, 'early_exit_count': 0,
1469
+ 'early_exit_pct': 0, 'avg_missed_profit': 0,
1470
+ 'exit_reasons': {}
1471
+ })
1472
+
1473
+
1474
+ # ============================================================
1475
+ # RESULTS REPORTER (unchanged)
1476
+ # ============================================================
1477
+ class ResultsReporter:
1478
+ """Print comprehensive metrics report"""
1479
+
1480
+ @staticmethod
1481
+ def print_full_report(analyzer):
1482
+ m = analyzer.metrics
1483
+ r = analyzer.results
1484
+
1485
+ print("="*80)
1486
+ print(" "*25 + "BACKTEST RESULTS")
1487
+ print("="*80)
1488
+ print()
1489
+
1490
+ print("PROFITABILITY METRICS")
1491
+ print("-"*80)
1492
+ print(f"Initial Capital: ${r.initial_capital:>15,.2f}")
1493
+ print(f"Final Equity: ${r.final_capital:>15,.2f}")
1494
+ print(f"Total P&L: ${m['total_pnl']:>15,.2f} (absolute profit/loss)")
1495
+ print(f"Total Return: {m['total_return']:>15.2f}% (% gain/loss)")
1496
+ if m['cagr'] != 0:
1497
+ if m['show_cagr']:
1498
+ print(f"CAGR: {m['cagr']:>15.2f}% (annualized compound growth)")
1499
+ else:
1500
+ print(f"Annualized Return: {m['cagr']:>15.2f}% (extrapolated to 1 year)")
1501
+ print()
1502
+
1503
+ print("RISK METRICS")
1504
+ print("-"*80)
1505
+ print(f"Sharpe Ratio: {m['sharpe']:>15.2f} (>1 good, >2 excellent)")
1506
+ print(f"Sortino Ratio: {m['sortino']:>15.2f} (downside risk, >2 good)")
1507
+ print(f"Calmar Ratio: {m['calmar']:>15.2f} (return/drawdown, >3 good)")
1508
+ if m['omega'] != 0:
1509
+ omega_display = f"{m['omega']:.2f}" if m['omega'] < 999 else "∞"
1510
+ print(f"Omega Ratio: {omega_display:>15s} (gains/losses, >1 good)")
1511
+ print(f"Maximum Drawdown: {m['max_drawdown']:>15.2f}% (peak to trough)")
1512
+ if m['ulcer'] != 0:
1513
+ print(f"Ulcer Index: {m['ulcer']:>15.2f}% (pain of drawdowns, lower better)")
1514
+ print(f"Volatility (ann.): {m['volatility']:>15.2f}% (annualized std dev)")
1515
+
1516
+ if len(r.daily_returns) >= 10:
1517
+ print(f"VaR (95%, 1-day): {m['var_95_pct']:>15.2f}% (${m['var_95_dollar']:>,.0f}) (max loss 95% confidence)")
1518
+ print(f"VaR (99%, 1-day): {m['var_99_pct']:>15.2f}% (${m['var_99_dollar']:>,.0f}) (max loss 99% confidence)")
1519
+ print(f"CVaR (95%, 1-day): {m['cvar_95_pct']:>15.2f}% (${m['cvar_95_dollar']:>,.0f}) (avg loss in worst 5%)")
1520
+
1521
+ if m['tail_ratio'] != 0:
1522
+ print(f"Tail Ratio (95/5): {m['tail_ratio']:>15.2f} (big wins/losses, >1 good)")
1523
+
1524
+ if m['skewness'] != 0 or m['kurtosis'] != 0:
1525
+ print(f"Skewness: {m['skewness']:>15.2f} (>0 positive tail)")
1526
+ print(f"Kurtosis (excess): {m['kurtosis']:>15.2f} (>0 fat tails)")
1527
+
1528
+ if m['beta'] != 0 or m['alpha'] != 0:
1529
+ print(f"Alpha (vs {r.benchmark_symbol}): {m['alpha']:>15.2f}% (excess return)")
1530
+ print(f"Beta (vs {r.benchmark_symbol}): {m['beta']:>15.2f} (<1 defensive, >1 aggressive)")
1531
+ print(f"R² (vs {r.benchmark_symbol}): {m['r_squared']:>15.2f} (market correlation 0-1)")
1532
+
1533
+ if abs(m['total_return']) > 200 or m['volatility'] > 150:
1534
+ print()
1535
+ print("WARNING: UNREALISTIC RESULTS DETECTED")
1536
+ if abs(m['total_return']) > 200:
1537
+ print(f" Total return {m['total_return']:.1f}% is extremely high")
1538
+ if m['volatility'] > 150:
1539
+ print(f" Volatility {m['volatility']:.1f}% is higher than leveraged ETFs")
1540
+ print(" Review configuration before trusting results")
1541
+
1542
+ print()
1543
+
1544
+ print("EFFICIENCY METRICS")
1545
+ print("-"*80)
1546
+ if m['recovery_factor'] != 0:
1547
+ print(f"Recovery Factor: {m['recovery_factor']:>15.2f} (profit/max DD, >3 good)")
1548
+ if m['exposure_time'] != 0:
1549
+ print(f"Exposure Time: {m['exposure_time']:>15.1f}% (time in market)")
1550
+ print()
1551
+
1552
+ print("TRADING STATISTICS")
1553
+ print("-"*80)
1554
+ print(f"Total Trades: {m['total_trades']:>15}")
1555
+ print(f"Winning Trades: {m['winning_trades']:>15}")
1556
+ print(f"Losing Trades: {m['losing_trades']:>15}")
1557
+ print(f"Win Rate: {m['win_rate']:>15.2f}% (% profitable trades)")
1558
+ print(f"Profit Factor: {m['profit_factor']:>15.2f} (gross profit/loss, >1.5 good)")
1559
+ if m['max_win_streak'] > 0 or m['max_loss_streak'] > 0:
1560
+ print(f"Max Win Streak: {m['max_win_streak']:>15} (consecutive wins)")
1561
+ print(f"Max Loss Streak: {m['max_loss_streak']:>15} (consecutive losses)")
1562
+ print(f"Average Win: ${m['avg_win']:>15,.2f}")
1563
+ print(f"Average Loss: ${m['avg_loss']:>15,.2f}")
1564
+ print(f"Best Trade: ${m['best_trade']:>15,.2f}")
1565
+ print(f"Worst Trade: ${m['worst_trade']:>15,.2f}")
1566
+ if m['avg_win_loss_ratio'] != 0:
1567
+ print(f"Avg Win/Loss Ratio: {m['avg_win_loss_ratio']:>15.2f} (avg win / avg loss)")
1568
+ print()
1569
+ print("="*80)
1570
+
1571
+
1572
+ def print_stoploss_section(analyzer):
1573
+ """Print stop-loss analysis section"""
1574
+ m = analyzer.metrics
1575
+
1576
+ if m.get('stoploss_count', 0) == 0:
1577
+ return
1578
+
1579
+ print("STOP-LOSS ANALYSIS")
1580
+ print("-"*80)
1581
+
1582
+ print(f"Stop-Loss Trades: {m['stoploss_count']:>15} ({m['stoploss_pct']:.1f}% of total)")
1583
+ print(f"Profit Target Trades: {m['profit_target_count']:>15} ({m['profit_target_pct']:.1f}% of total)")
1584
+
1585
+ print(f"Avg Stop-Loss P&L: ${m['avg_stoploss_pnl']:>15,.2f}")
1586
+ print(f"Total Loss from SL: ${m['total_stoploss_loss']:>15,.2f}")
1587
+ print(f"Worst Stop-Loss: ${m['worst_stoploss']:>15,.2f}")
1588
+ print(f"Avg SL Return: {m['avg_stoploss_return_pct']:>15.2f}%")
1589
+
1590
+ if m['avg_days_to_stoploss'] > 0:
1591
+ print(f"Avg Days to SL: {m['avg_days_to_stoploss']:>15.1f}")
1592
+ print(f"Min/Max Days to SL: {m['min_days_to_stoploss']:>7} / {m['max_days_to_stoploss']:<7}")
1593
+
1594
+ if m['profit_to_loss_ratio'] > 0:
1595
+ print(f"Profit/Loss Ratio: {m['profit_to_loss_ratio']:>15.2f} (avg profit target / avg stop-loss)")
1596
+
1597
+ if m['early_exit_count'] > 0:
1598
+ print(f"Early Exits: {m['early_exit_count']:>15} ({m['early_exit_pct']:.1f}% of SL trades)")
1599
+ print(f"Avg Missed Profit: ${m['avg_missed_profit']:>15,.2f} (profit before stop triggered)")
1600
+
1601
+ if m['stoploss_by_type']:
1602
+ print(f"\nStop-Loss Types:")
1603
+ for stop_type, count in m['stoploss_by_type'].items():
1604
+ pct = (count / m['stoploss_count']) * 100
1605
+ print(f" {stop_type:20s} {count:>5} trades ({pct:.1f}%)")
1606
+
1607
+ if m.get('exit_reasons'):
1608
+ print(f"\nExit Reasons Distribution:")
1609
+ total_trades = sum(m['exit_reasons'].values())
1610
+ for reason, count in sorted(m['exit_reasons'].items(), key=lambda x: x[1], reverse=True):
1611
+ pct = (count / total_trades) * 100
1612
+ print(f" {reason:20s} {count:>5} trades ({pct:.1f}%)")
1613
+
1614
+ print()
1615
+ print("="*80)
1616
+
1617
+
1618
+ # ============================================================
1619
+ # CHART GENERATOR (only core charts, optimization charts separate)
1620
+ # ============================================================
1621
+ class ChartGenerator:
1622
+ """Generate 6 professional charts"""
1623
+
1624
+ @staticmethod
1625
+ def create_all_charts(analyzer, filename='backtest_results.png', show_plots=True, silent=False):
1626
+ r = analyzer.results
1627
+
1628
+ if len(r.trades) == 0:
1629
+ if not silent:
1630
+ print("No trades to visualize")
1631
+ return None
1632
+
1633
+ trades_df = pd.DataFrame(r.trades)
1634
+ fig, axes = plt.subplots(3, 2, figsize=(18, 14))
1635
+ fig.suptitle('Backtest Results', fontsize=16, fontweight='bold', y=0.995)
1636
+
1637
+ dates = pd.to_datetime(r.equity_dates)
1638
+ equity_array = np.array(r.equity_curve)
1639
+
1640
+ ax1 = axes[0, 0]
1641
+ ax1.plot(dates, equity_array, linewidth=2.5, color='#2196F3')
1642
+ ax1.axhline(y=r.initial_capital, color='gray', linestyle='--', alpha=0.7)
1643
+ ax1.fill_between(dates, r.initial_capital, equity_array,
1644
+ where=(equity_array >= r.initial_capital),
1645
+ alpha=0.3, color='green', interpolate=True)
1646
+ ax1.fill_between(dates, r.initial_capital, equity_array,
1647
+ where=(equity_array < r.initial_capital),
1648
+ alpha=0.3, color='red', interpolate=True)
1649
+ ax1.set_title('Equity Curve', fontsize=12, fontweight='bold')
1650
+ ax1.set_ylabel('Equity ($)')
1651
+ ax1.grid(True, alpha=0.3)
1652
+
1653
+ ax2 = axes[0, 1]
1654
+ running_max = np.maximum.accumulate(equity_array)
1655
+ drawdown = (equity_array - running_max) / running_max * 100
1656
+ ax2.fill_between(dates, 0, drawdown, alpha=0.6, color='#f44336')
1657
+ ax2.plot(dates, drawdown, color='#d32f2f', linewidth=2)
1658
+ ax2.set_title('Drawdown', fontsize=12, fontweight='bold')
1659
+ ax2.set_ylabel('Drawdown (%)')
1660
+ ax2.grid(True, alpha=0.3)
1661
+
1662
+ ax3 = axes[1, 0]
1663
+ pnl_values = trades_df['pnl'].values
1664
+ ax3.hist(pnl_values, bins=40, color='#4CAF50', alpha=0.7, edgecolor='black')
1665
+ ax3.axvline(x=0, color='red', linestyle='--', linewidth=2)
1666
+ ax3.set_title('P&L Distribution', fontsize=12, fontweight='bold')
1667
+ ax3.set_xlabel('P&L ($)')
1668
+ ax3.grid(True, alpha=0.3, axis='y')
1669
+
1670
+ ax4 = axes[1, 1]
1671
+ if 'signal' in trades_df.columns:
1672
+ signal_pnl = trades_df.groupby('signal')['pnl'].sum()
1673
+ colors = ['#4CAF50' if x > 0 else '#f44336' for x in signal_pnl.values]
1674
+ ax4.bar(signal_pnl.index, signal_pnl.values, color=colors, alpha=0.7)
1675
+ ax4.set_title('P&L by Signal', fontsize=12, fontweight='bold')
1676
+ else:
1677
+ ax4.text(0.5, 0.5, 'No signal data', ha='center', va='center', transform=ax4.transAxes)
1678
+ ax4.axhline(y=0, color='black', linewidth=1)
1679
+ ax4.grid(True, alpha=0.3, axis='y')
1680
+
1681
+ ax5 = axes[2, 0]
1682
+ trades_df['exit_date'] = pd.to_datetime(trades_df['exit_date'])
1683
+ trades_df['month'] = trades_df['exit_date'].dt.to_period('M')
1684
+ monthly_pnl = trades_df.groupby('month')['pnl'].sum()
1685
+ colors = ['#4CAF50' if x > 0 else '#f44336' for x in monthly_pnl.values]
1686
+ ax5.bar(range(len(monthly_pnl)), monthly_pnl.values, color=colors, alpha=0.7)
1687
+ ax5.set_title('Monthly P&L', fontsize=12, fontweight='bold')
1688
+ ax5.set_xticks(range(len(monthly_pnl)))
1689
+ ax5.set_xticklabels([str(m) for m in monthly_pnl.index], rotation=45, ha='right')
1690
+ ax5.axhline(y=0, color='black', linewidth=1)
1691
+ ax5.grid(True, alpha=0.3, axis='y')
1692
+
1693
+ ax6 = axes[2, 1]
1694
+ if 'symbol' in trades_df.columns:
1695
+ symbol_pnl = trades_df.groupby('symbol')['pnl'].sum().sort_values(ascending=True).tail(10)
1696
+ colors = ['#4CAF50' if x > 0 else '#f44336' for x in symbol_pnl.values]
1697
+ ax6.barh(range(len(symbol_pnl)), symbol_pnl.values, color=colors, alpha=0.7)
1698
+ ax6.set_yticks(range(len(symbol_pnl)))
1699
+ ax6.set_yticklabels(symbol_pnl.index, fontsize=9)
1700
+ ax6.set_title('Top Symbols', fontsize=12, fontweight='bold')
1701
+ else:
1702
+ ax6.text(0.5, 0.5, 'No symbol data', ha='center', va='center', transform=ax6.transAxes)
1703
+ ax6.axvline(x=0, color='black', linewidth=1)
1704
+ ax6.grid(True, alpha=0.3, axis='x')
1705
+
1706
+ plt.tight_layout()
1707
+ plt.savefig(filename, dpi=300, bbox_inches='tight')
1708
+
1709
+ if show_plots:
1710
+ plt.show()
1711
+ else:
1712
+ plt.close() # Close without displaying
1713
+
1714
+ if not silent:
1715
+ print(f"Chart saved: {filename}")
1716
+
1717
+ return filename
1718
+
1719
+
1720
+ def create_stoploss_charts(analyzer, filename='stoploss_analysis.png', show_plots=True):
1721
+ """Create 4 stop-loss specific charts"""
1722
+ r = analyzer.results
1723
+ m = analyzer.metrics
1724
+
1725
+ if m.get('stoploss_count', 0) == 0:
1726
+ print("No stop-loss trades to visualize")
1727
+ return
1728
+
1729
+ trades_df = pd.DataFrame(r.trades)
1730
+
1731
+ if 'exit_reason' not in trades_df.columns:
1732
+ print("No exit_reason data available")
1733
+ return
1734
+
1735
+ fig, axes = plt.subplots(2, 2, figsize=(16, 12))
1736
+ fig.suptitle('Stop-Loss Analysis', fontsize=16, fontweight='bold', y=0.995)
1737
+
1738
+ ax1 = axes[0, 0]
1739
+ if m.get('exit_reasons'):
1740
+ reasons = pd.Series(m['exit_reasons']).sort_values(ascending=True)
1741
+ colors = ['#f44336' if 'stop_loss' in str(r) else '#4CAF50' if r == 'profit_target' else '#2196F3'
1742
+ for r in reasons.index]
1743
+ ax1.barh(range(len(reasons)), reasons.values, color=colors, alpha=0.7, edgecolor='black')
1744
+ ax1.set_yticks(range(len(reasons)))
1745
+ ax1.set_yticklabels([r.replace('_', ' ').title() for r in reasons.index])
1746
+ ax1.set_title('Exit Reasons Distribution', fontsize=12, fontweight='bold')
1747
+ ax1.set_xlabel('Number of Trades')
1748
+ ax1.grid(True, alpha=0.3, axis='x')
1749
+
1750
+ total = sum(reasons.values)
1751
+ for i, v in enumerate(reasons.values):
1752
+ ax1.text(v, i, f' {(v/total)*100:.1f}%', va='center', fontweight='bold')
1753
+
1754
+ ax2 = axes[0, 1]
1755
+ sl_trades = trades_df[trades_df['exit_reason'].str.contains('stop_loss', na=False)]
1756
+ if len(sl_trades) > 0:
1757
+ ax2.hist(sl_trades['pnl'], bins=30, color='#f44336', alpha=0.7, edgecolor='black')
1758
+ ax2.axvline(x=0, color='black', linestyle='--', linewidth=2)
1759
+ ax2.axvline(x=sl_trades['pnl'].mean(), color='yellow', linestyle='--', linewidth=2, label='Mean')
1760
+ ax2.set_title('Stop-Loss P&L Distribution', fontsize=12, fontweight='bold')
1761
+ ax2.set_xlabel('P&L ($)')
1762
+ ax2.set_ylabel('Frequency')
1763
+ ax2.legend()
1764
+ ax2.grid(True, alpha=0.3, axis='y')
1765
+
1766
+ ax3 = axes[1, 0]
1767
+ if len(sl_trades) > 0 and 'entry_date' in sl_trades.columns and 'exit_date' in sl_trades.columns:
1768
+ sl_trades_copy = sl_trades.copy()
1769
+ sl_trades_copy['entry_date'] = pd.to_datetime(sl_trades_copy['entry_date'])
1770
+ sl_trades_copy['exit_date'] = pd.to_datetime(sl_trades_copy['exit_date'])
1771
+ sl_trades_copy['days_held'] = (sl_trades_copy['exit_date'] - sl_trades_copy['entry_date']).dt.days
1772
+
1773
+ ax3.hist(sl_trades_copy['days_held'], bins=30, color='#FF9800', alpha=0.7, edgecolor='black')
1774
+ ax3.axvline(x=sl_trades_copy['days_held'].mean(), color='red', linestyle='--', linewidth=2, label='Mean')
1775
+ ax3.set_title('Days Until Stop-Loss Triggered', fontsize=12, fontweight='bold')
1776
+ ax3.set_xlabel('Days Held')
1777
+ ax3.set_ylabel('Frequency')
1778
+ ax3.legend()
1779
+ ax3.grid(True, alpha=0.3, axis='y')
1780
+
1781
+ ax4 = axes[1, 1]
1782
+ if 'stop_type' in sl_trades.columns:
1783
+ stop_types = sl_trades['stop_type'].value_counts()
1784
+ colors_types = plt.cm.Set3(range(len(stop_types)))
1785
+ wedges, texts, autotexts = ax4.pie(stop_types.values, labels=stop_types.index,
1786
+ autopct='%1.1f%%', colors=colors_types,
1787
+ startangle=90)
1788
+ for autotext in autotexts:
1789
+ autotext.set_color('black')
1790
+ autotext.set_fontweight('bold')
1791
+ ax4.set_title('Stop-Loss Types', fontsize=12, fontweight='bold')
1792
+ else:
1793
+ ax4.text(0.5, 0.5, 'No stop_type data', ha='center', va='center', transform=ax4.transAxes)
1794
+
1795
+ plt.tight_layout()
1796
+ plt.savefig(filename, dpi=300, bbox_inches='tight')
1797
+
1798
+ if show_plots:
1799
+ plt.show()
1800
+ else:
1801
+ plt.close()
1802
+
1803
+ print(f"Stop-loss charts saved: {filename}")
1804
+
1805
+
1806
+ # ============================================================
1807
+ # RESULTS EXPORTER (unchanged)
1808
+ # ============================================================
1809
+ class ResultsExporter:
1810
+ """Export results to CSV"""
1811
+
1812
+ @staticmethod
1813
+ def export_all(analyzer, prefix='backtest', silent=False):
1814
+ r = analyzer.results
1815
+ m = analyzer.metrics
1816
+
1817
+ if len(r.trades) == 0:
1818
+ if not silent:
1819
+ print("No trades to export")
1820
+ return []
1821
+
1822
+ trades_df = pd.DataFrame(r.trades)
1823
+
1824
+ trades_df['entry_date'] = pd.to_datetime(trades_df['entry_date']).dt.strftime('%Y-%m-%d')
1825
+ trades_df['exit_date'] = pd.to_datetime(trades_df['exit_date']).dt.strftime('%Y-%m-%d')
1826
+
1827
+ # Round numeric columns to 5 decimal places
1828
+ numeric_columns = trades_df.select_dtypes(include=[np.number]).columns
1829
+ for col in numeric_columns:
1830
+ trades_df[col] = trades_df[col].round(5)
1831
+
1832
+ core_columns = [
1833
+ 'entry_date', 'exit_date', 'symbol', 'signal',
1834
+ 'pnl', 'return_pct', 'exit_reason', 'stop_type'
1835
+ ]
1836
+
1837
+ options_columns = [
1838
+ 'short_strike', 'long_strike', 'expiration', 'opt_type',
1839
+ 'spread_type', 'contracts'
1840
+ ]
1841
+
1842
+ bidask_columns = [
1843
+ 'short_entry_bid', 'short_entry_ask', 'short_entry_mid',
1844
+ 'long_entry_bid', 'long_entry_ask', 'long_entry_mid',
1845
+ 'short_exit_bid', 'short_exit_ask',
1846
+ 'long_exit_bid', 'long_exit_ask'
1847
+ ]
1848
+
1849
+ underlying_columns = [
1850
+ 'underlying_entry_price', 'underlying_exit_price',
1851
+ 'underlying_change_pct'
1852
+ ]
1853
+
1854
+ stop_columns = [
1855
+ 'stop_threshold', 'actual_value'
1856
+ ]
1857
+
1858
+ strategy_columns = [
1859
+ 'entry_z_score', 'is_short_bias', 'entry_price',
1860
+ 'exit_price', 'quantity', 'entry_lean', 'exit_lean',
1861
+ # IV EOD fields
1862
+ 'call_iv_entry', 'put_iv_entry', 'call_iv_exit', 'put_iv_exit',
1863
+ 'iv_lean_entry', 'iv_lean_exit'
1864
+ ]
1865
+
1866
+ # NEW: Intraday stop-loss columns
1867
+ intraday_columns = [
1868
+ 'spy_intraday_high', 'spy_intraday_low', 'spy_intraday_close',
1869
+ 'spy_stop_trigger_time', 'spy_stop_trigger_price',
1870
+ 'spy_stop_trigger_bid', 'spy_stop_trigger_ask', 'spy_stop_trigger_last',
1871
+ 'intraday_data_points', 'intraday_data_available', 'stop_triggered_by'
1872
+ ]
1873
+
1874
+ ordered_columns = []
1875
+ for col in (core_columns + options_columns + bidask_columns +
1876
+ underlying_columns + stop_columns + strategy_columns + intraday_columns):
1877
+ if col in trades_df.columns:
1878
+ ordered_columns.append(col)
1879
+
1880
+ remaining = [col for col in trades_df.columns if col not in ordered_columns]
1881
+ ordered_columns.extend(remaining)
1882
+
1883
+ trades_df = trades_df[ordered_columns]
1884
+
1885
+ # Round numeric columns to 2 decimals
1886
+ numeric_columns = trades_df.select_dtypes(include=['float64', 'float32', 'float']).columns
1887
+ for col in numeric_columns:
1888
+ trades_df[col] = trades_df[col].round(5)
1889
+
1890
+ exported_files = []
1891
+
1892
+ trades_df.to_csv(f'{prefix}_trades.csv', index=False)
1893
+ exported_files.append((f'{prefix}_trades.csv', f"({len(ordered_columns)} columns)"))
1894
+ if not silent:
1895
+ print(f"Exported: {prefix}_trades.csv ({len(ordered_columns)} columns)")
1896
+
1897
+ equity_df = pd.DataFrame({
1898
+ 'date': pd.to_datetime(r.equity_dates).strftime('%Y-%m-%d'),
1899
+ 'equity': r.equity_curve
1900
+ })
1901
+ equity_df['equity'] = equity_df['equity'].round(5)
1902
+ equity_df.to_csv(f'{prefix}_equity.csv', index=False)
1903
+ exported_files.append((f'{prefix}_equity.csv', ""))
1904
+ if not silent:
1905
+ print(f"Exported: {prefix}_equity.csv")
1906
+
1907
+ with open(f'{prefix}_summary.txt', 'w') as f:
1908
+ f.write("BACKTEST SUMMARY\n")
1909
+ f.write("="*70 + "\n\n")
1910
+ f.write(f"Strategy: {r.config.get('strategy_name', 'Unknown')}\n")
1911
+ f.write(f"Period: {r.config.get('start_date')} to {r.config.get('end_date')}\n\n")
1912
+ f.write("PERFORMANCE\n")
1913
+ f.write("-"*70 + "\n")
1914
+ f.write(f"Total Return: {m['total_return']:.2f}%\n")
1915
+ f.write(f"Sharpe: {m['sharpe']:.2f}\n")
1916
+ f.write(f"Max DD: {m['max_drawdown']:.2f}%\n")
1917
+ f.write(f"Trades: {m['total_trades']}\n")
1918
+
1919
+ exported_files.append((f'{prefix}_summary.txt', ""))
1920
+ if not silent:
1921
+ print(f"Exported: {prefix}_summary.txt")
1922
+
1923
+ # Export metrics as JSON with rounded values
1924
+ import json
1925
+ metrics_rounded = {}
1926
+ for key, value in m.items():
1927
+ if isinstance(value, (int, float)):
1928
+ metrics_rounded[key] = round(float(value), 5) if isinstance(value, float) else value
1929
+ else:
1930
+ metrics_rounded[key] = value
1931
+
1932
+ with open(f'{prefix}_metrics.json', 'w') as f:
1933
+ json.dump(metrics_rounded, f, indent=2)
1934
+
1935
+ exported_files.append((f'{prefix}_metrics.json', ""))
1936
+ if not silent:
1937
+ print(f"Exported: {prefix}_metrics.json")
1938
+
1939
+ return exported_files
1940
+
1941
+
1942
+ # ============================================================
1943
+ # RUN BACKTEST (unchanged)
1944
+ # ============================================================
1945
+ def run_backtest(strategy_function, config, print_report=True,
1946
+ create_charts=True, export_results=True,
1947
+ chart_filename='backtest_results.png',
1948
+ export_prefix='backtest',
1949
+ progress_context=None):
1950
+ """Run complete backtest"""
1951
+
1952
+ # Check if running inside optimization
1953
+ is_optimization = progress_context and progress_context.get('is_optimization', False)
1954
+
1955
+ if not progress_context and not is_optimization:
1956
+ print("="*80)
1957
+ print(" "*25 + "STARTING BACKTEST")
1958
+ print("="*80)
1959
+ print(f"Strategy: {config.get('strategy_name', 'Unknown')}")
1960
+ print(f"Period: {config.get('start_date')} to {config.get('end_date')}")
1961
+ print(f"Capital: ${config.get('initial_capital', 0):,.0f}")
1962
+ print("="*80 + "\n")
1963
+
1964
+ if progress_context:
1965
+ config['_progress_context'] = progress_context
1966
+
1967
+ results = strategy_function(config)
1968
+
1969
+ if '_progress_context' in config:
1970
+ del config['_progress_context']
1971
+
1972
+ if not is_optimization:
1973
+ print("\n[*] Calculating metrics...")
1974
+ analyzer = BacktestAnalyzer(results)
1975
+ analyzer.calculate_all_metrics()
1976
+
1977
+ if print_report:
1978
+ print("\n" + "="*80)
1979
+ ResultsReporter.print_full_report(analyzer)
1980
+
1981
+ # Store file info for later printing (in optimization mode)
1982
+ analyzer.chart_file = None
1983
+ analyzer.exported_files = []
1984
+
1985
+ # Export charts during optimization if requested
1986
+ if create_charts and len(results.trades) > 0:
1987
+ if not is_optimization:
1988
+ print(f"\n[*] Creating charts: {chart_filename}")
1989
+ try:
1990
+ # Don't show plots during optimization, just save them
1991
+ chart_file = ChartGenerator.create_all_charts(
1992
+ analyzer, chart_filename,
1993
+ show_plots=not is_optimization,
1994
+ silent=is_optimization # ← Silent in optimization
1995
+ )
1996
+ analyzer.chart_file = chart_file
1997
+ except Exception as e:
1998
+ if not is_optimization:
1999
+ print(f"[ERROR] Charts failed: {e}")
2000
+
2001
+ # Export results during optimization if requested
2002
+ if export_results and len(results.trades) > 0:
2003
+ if not is_optimization:
2004
+ print(f"\n[*] Exporting: {export_prefix}_*")
2005
+ try:
2006
+ exported = ResultsExporter.export_all(
2007
+ analyzer, export_prefix,
2008
+ silent=is_optimization # ← Silent in optimization
2009
+ )
2010
+ analyzer.exported_files = exported
2011
+ except Exception as e:
2012
+ if not is_optimization:
2013
+ print(f"[ERROR] Export failed: {e}")
2014
+
2015
+ return analyzer
2016
+
2017
+
2018
+ def run_backtest_with_stoploss(strategy_function, config, print_report=True,
2019
+ create_charts=True, export_results=True,
2020
+ chart_filename='backtest_results.png',
2021
+ export_prefix='backtest',
2022
+ create_stoploss_report=True,
2023
+ create_stoploss_charts=True,
2024
+ progress_context=None):
2025
+ """Enhanced run_backtest with stop-loss analysis"""
2026
+
2027
+ analyzer = run_backtest(
2028
+ strategy_function, config,
2029
+ print_report=False,
2030
+ create_charts=create_charts,
2031
+ export_results=export_results,
2032
+ chart_filename=chart_filename,
2033
+ export_prefix=export_prefix,
2034
+ progress_context=progress_context
2035
+ )
2036
+
2037
+ calculate_stoploss_metrics(analyzer)
2038
+
2039
+ if print_report:
2040
+ print("\n" + "="*80)
2041
+ ResultsReporter.print_full_report(analyzer)
2042
+
2043
+ if create_stoploss_report and analyzer.metrics.get('stoploss_count', 0) > 0:
2044
+ print_stoploss_section(analyzer)
2045
+
2046
+ if create_stoploss_charts and analyzer.metrics.get('stoploss_count', 0) > 0:
2047
+ print(f"\n[*] Creating stop-loss analysis charts...")
2048
+ try:
2049
+ stoploss_chart_name = chart_filename.replace('.png', '_stoploss.png') if chart_filename else 'stoploss_analysis.png'
2050
+ create_stoploss_charts(analyzer, stoploss_chart_name)
2051
+ except Exception as e:
2052
+ print(f"[ERROR] Stop-loss charts failed: {e}")
2053
+
2054
+ return analyzer
2055
+
2056
+
2057
+ # ============================================================
2058
+ # STOP-LOSS CONFIG (ENHANCED WITH COMBINED)
2059
+ # ============================================================
2060
+ class StopLossConfig:
2061
+ """
2062
+ Universal stop-loss configuration builder (ENHANCED)
2063
+
2064
+ NEW METHOD:
2065
+ - combined(): Requires BOTH pl_loss AND directional conditions
2066
+ """
2067
+
2068
+ @staticmethod
2069
+ def _normalize_pct(value):
2070
+ """Convert any number to decimal (0.30)"""
2071
+ if value >= 1:
2072
+ return value / 100
2073
+ return value
2074
+
2075
+ @staticmethod
2076
+ def _format_pct(value):
2077
+ """Format percentage for display"""
2078
+ if value >= 1:
2079
+ return f"{value:.0f}%"
2080
+ return f"{value*100:.0f}%"
2081
+
2082
+ @staticmethod
2083
+ def none():
2084
+ """No stop-loss"""
2085
+ return {
2086
+ 'enabled': False,
2087
+ 'type': 'none',
2088
+ 'value': 0,
2089
+ 'name': 'No Stop-Loss',
2090
+ 'description': 'No stop-loss protection'
2091
+ }
2092
+
2093
+ @staticmethod
2094
+ def fixed(pct):
2095
+ """Fixed percentage stop-loss"""
2096
+ decimal = StopLossConfig._normalize_pct(pct)
2097
+ display = StopLossConfig._format_pct(pct)
2098
+
2099
+ return {
2100
+ 'enabled': True,
2101
+ 'type': 'fixed_pct',
2102
+ 'value': decimal,
2103
+ 'name': f'Fixed {display}',
2104
+ 'description': f'Fixed stop at {display} loss'
2105
+ }
2106
+
2107
+ @staticmethod
2108
+ def trailing(pct, trailing_distance=None):
2109
+ """Trailing stop-loss"""
2110
+ decimal = StopLossConfig._normalize_pct(pct)
2111
+ display = StopLossConfig._format_pct(pct)
2112
+
2113
+ config = {
2114
+ 'enabled': True,
2115
+ 'type': 'trailing',
2116
+ 'value': decimal,
2117
+ 'name': f'Trailing {display}',
2118
+ 'description': f'Trailing stop at {display} from peak'
2119
+ }
2120
+
2121
+ if trailing_distance is not None:
2122
+ config['trailing_distance'] = StopLossConfig._normalize_pct(trailing_distance)
2123
+
2124
+ return config
2125
+
2126
+ @staticmethod
2127
+ def time_based(days):
2128
+ """Time-based stop"""
2129
+ return {
2130
+ 'enabled': True,
2131
+ 'type': 'time_based',
2132
+ 'value': days,
2133
+ 'name': f'Time {days}d',
2134
+ 'description': f'Exit after {days} days'
2135
+ }
2136
+
2137
+ @staticmethod
2138
+ def volatility(atr_multiplier):
2139
+ """ATR-based stop"""
2140
+ return {
2141
+ 'enabled': True,
2142
+ 'type': 'volatility',
2143
+ 'value': atr_multiplier,
2144
+ 'name': f'ATR {atr_multiplier:.1f}x',
2145
+ 'description': f'Stop at {atr_multiplier:.1f}× ATR',
2146
+ 'requires_atr': True
2147
+ }
2148
+
2149
+ @staticmethod
2150
+ def pl_loss(pct):
2151
+ """P&L-based stop using real bid/ask prices"""
2152
+ decimal = StopLossConfig._normalize_pct(pct)
2153
+ display = StopLossConfig._format_pct(pct)
2154
+
2155
+ return {
2156
+ 'enabled': True,
2157
+ 'type': 'pl_loss',
2158
+ 'value': decimal,
2159
+ 'name': f'P&L Loss {display}',
2160
+ 'description': f'Stop when P&L drops to -{display}'
2161
+ }
2162
+
2163
+ @staticmethod
2164
+ def directional(pct):
2165
+ """Directional stop based on underlying movement"""
2166
+ decimal = StopLossConfig._normalize_pct(pct)
2167
+ display = StopLossConfig._format_pct(pct)
2168
+
2169
+ return {
2170
+ 'enabled': True,
2171
+ 'type': 'directional',
2172
+ 'value': decimal,
2173
+ 'name': f'Directional {display}',
2174
+ 'description': f'Stop when underlying moves {display}'
2175
+ }
2176
+
2177
+ # ========================================================
2178
+ # NEW: COMBINED STOP (REQUIRES BOTH CONDITIONS)
2179
+ # ========================================================
2180
+
2181
+ @staticmethod
2182
+ def combined(pl_loss_pct, directional_pct):
2183
+ """
2184
+ Combined stop: Requires BOTH conditions (from code 2)
2185
+
2186
+ Args:
2187
+ pl_loss_pct: P&L loss threshold (e.g., 5 or 0.05 = -5%)
2188
+ directional_pct: Underlying move threshold (e.g., 3 or 0.03 = 3%)
2189
+
2190
+ Example:
2191
+ StopLossConfig.combined(5, 3)
2192
+ # Triggers only when BOTH:
2193
+ # 1. P&L drops to -5%
2194
+ # 2. Underlying moves 3% adversely
2195
+ """
2196
+ pl_decimal = StopLossConfig._normalize_pct(pl_loss_pct)
2197
+ dir_decimal = StopLossConfig._normalize_pct(directional_pct)
2198
+
2199
+ pl_display = StopLossConfig._format_pct(pl_loss_pct)
2200
+ dir_display = StopLossConfig._format_pct(directional_pct)
2201
+
2202
+ return {
2203
+ 'enabled': True,
2204
+ 'type': 'combined',
2205
+ 'value': {
2206
+ 'pl_loss': pl_decimal,
2207
+ 'directional': dir_decimal
2208
+ },
2209
+ 'name': f'Combined (P&L {pl_display} + Dir {dir_display})',
2210
+ 'description': f'Stop when P&L<-{pl_display} AND underlying moves {dir_display}'
2211
+ }
2212
+
2213
+ # ========================================================
2214
+ # BACKWARD COMPATIBILITY
2215
+ # ========================================================
2216
+
2217
+ @staticmethod
2218
+ def time(days):
2219
+ """Alias for time_based()"""
2220
+ return StopLossConfig.time_based(days)
2221
+
2222
+ @staticmethod
2223
+ def atr(multiplier):
2224
+ """Alias for volatility()"""
2225
+ return StopLossConfig.volatility(multiplier)
2226
+
2227
+ # ========================================================
2228
+ # PRESETS (WITH COMBINED STOPS)
2229
+ # ========================================================
2230
+
2231
+ @staticmethod
2232
+ def presets():
2233
+ """Generate all standard stop-loss presets (UPDATED WITH COMBINED)"""
2234
+ return {
2235
+ 'none': StopLossConfig.none(),
2236
+
2237
+ 'fixed_20': StopLossConfig.fixed(20),
2238
+ 'fixed_30': StopLossConfig.fixed(30),
2239
+ 'fixed_40': StopLossConfig.fixed(40),
2240
+ 'fixed_50': StopLossConfig.fixed(50),
2241
+ 'fixed_70': StopLossConfig.fixed(70),
2242
+
2243
+ 'trailing_20': StopLossConfig.trailing(20),
2244
+ 'trailing_30': StopLossConfig.trailing(30),
2245
+ 'trailing_50': StopLossConfig.trailing(50),
2246
+
2247
+ 'time_5d': StopLossConfig.time(5),
2248
+ 'time_10d': StopLossConfig.time(10),
2249
+ 'time_20d': StopLossConfig.time(20),
2250
+
2251
+ 'atr_2x': StopLossConfig.atr(2.0),
2252
+ 'atr_3x': StopLossConfig.atr(3.0),
2253
+
2254
+ 'pl_loss_5': StopLossConfig.pl_loss(5),
2255
+ 'pl_loss_10': StopLossConfig.pl_loss(10),
2256
+ 'pl_loss_15': StopLossConfig.pl_loss(15),
2257
+
2258
+ 'directional_3': StopLossConfig.directional(3),
2259
+ 'directional_5': StopLossConfig.directional(5),
2260
+ 'directional_7': StopLossConfig.directional(7),
2261
+
2262
+ # NEW: COMBINED STOPS
2263
+ 'combined_5_3': StopLossConfig.combined(5, 3),
2264
+ 'combined_7_5': StopLossConfig.combined(7, 5),
2265
+ 'combined_10_3': StopLossConfig.combined(10, 3),
2266
+ }
2267
+
2268
+ @staticmethod
2269
+ def apply(base_config, stop_config):
2270
+ """Apply stop-loss configuration to base config"""
2271
+ merged = base_config.copy()
2272
+
2273
+ merged['stop_loss_enabled'] = stop_config.get('enabled', False)
2274
+
2275
+ if merged['stop_loss_enabled']:
2276
+ sl_config = {
2277
+ 'type': stop_config['type'],
2278
+ 'value': stop_config['value']
2279
+ }
2280
+
2281
+ if 'trailing_distance' in stop_config:
2282
+ sl_config['trailing_distance'] = stop_config['trailing_distance']
2283
+
2284
+ merged['stop_loss_config'] = sl_config
2285
+
2286
+ return merged
2287
+
2288
+
2289
+ def create_stoploss_comparison_chart(results, filename='stoploss_comparison.png', show_plots=True):
2290
+ """Create comparison chart"""
2291
+ try:
2292
+ fig, axes = plt.subplots(2, 2, figsize=(16, 12))
2293
+ fig.suptitle('Stop-Loss Configuration Comparison', fontsize=16, fontweight='bold')
2294
+
2295
+ names = [r['config']['name'] for r in results.values()]
2296
+ returns = [r['total_return'] for r in results.values()]
2297
+ sharpes = [r['sharpe'] for r in results.values()]
2298
+ drawdowns = [r['max_drawdown'] for r in results.values()]
2299
+ stop_counts = [r['stoploss_count'] for r in results.values()]
2300
+
2301
+ ax1 = axes[0, 0]
2302
+ colors = ['#4CAF50' if r > 0 else '#f44336' for r in returns]
2303
+ ax1.barh(range(len(names)), returns, color=colors, alpha=0.7, edgecolor='black')
2304
+ ax1.set_yticks(range(len(names)))
2305
+ ax1.set_yticklabels(names, fontsize=9)
2306
+ ax1.set_xlabel('Total Return (%)')
2307
+ ax1.set_title('Total Return by Stop-Loss Type', fontsize=12, fontweight='bold')
2308
+ ax1.axvline(x=0, color='black', linestyle='-', linewidth=1)
2309
+ ax1.grid(True, alpha=0.3, axis='x')
2310
+
2311
+ ax2 = axes[0, 1]
2312
+ colors_sharpe = ['#4CAF50' if s > 1 else '#FF9800' if s > 0 else '#f44336' for s in sharpes]
2313
+ ax2.barh(range(len(names)), sharpes, color=colors_sharpe, alpha=0.7, edgecolor='black')
2314
+ ax2.set_yticks(range(len(names)))
2315
+ ax2.set_yticklabels(names, fontsize=9)
2316
+ ax2.set_xlabel('Sharpe Ratio')
2317
+ ax2.set_title('Sharpe Ratio by Stop-Loss Type', fontsize=12, fontweight='bold')
2318
+ ax2.axvline(x=1, color='green', linestyle='--', linewidth=1, label='Good (>1)')
2319
+ ax2.axvline(x=0, color='black', linestyle='-', linewidth=1)
2320
+ ax2.legend()
2321
+ ax2.grid(True, alpha=0.3, axis='x')
2322
+
2323
+ ax3 = axes[1, 0]
2324
+ ax3.barh(range(len(names)), drawdowns, color='#f44336', alpha=0.7, edgecolor='black')
2325
+ ax3.set_yticks(range(len(names)))
2326
+ ax3.set_yticklabels(names, fontsize=9)
2327
+ ax3.set_xlabel('Maximum Drawdown (%)')
2328
+ ax3.set_title('Maximum Drawdown (Lower is Better)', fontsize=12, fontweight='bold')
2329
+ ax3.grid(True, alpha=0.3, axis='x')
2330
+
2331
+ ax4 = axes[1, 1]
2332
+ ax4.barh(range(len(names)), stop_counts, color='#2196F3', alpha=0.7, edgecolor='black')
2333
+ ax4.set_yticks(range(len(names)))
2334
+ ax4.set_yticklabels(names, fontsize=9)
2335
+ ax4.set_xlabel('Number of Stop-Loss Exits')
2336
+ ax4.set_title('Stop-Loss Frequency', fontsize=12, fontweight='bold')
2337
+ ax4.grid(True, alpha=0.3, axis='x')
2338
+
2339
+ plt.tight_layout()
2340
+ plt.savefig(filename, dpi=300, bbox_inches='tight')
2341
+
2342
+ if show_plots:
2343
+ plt.show()
2344
+ else:
2345
+ plt.close()
2346
+
2347
+ print(f"Comparison chart saved: {filename}")
2348
+
2349
+ except Exception as e:
2350
+ print(f"Failed to create comparison chart: {e}")
2351
+
2352
+
2353
+
2354
+ # ============================================================
2355
+ # DATA PRELOADING FUNCTION (FOR OPTIMIZATION)
2356
+ # ============================================================
2357
+ def preload_options_data(config, progress_widgets=None):
2358
+ """
2359
+ Preload options data for optimization.
2360
+ Loads data ONCE and returns cache.
2361
+
2362
+ Returns:
2363
+ tuple: (lean_df, options_cache)
2364
+ - lean_df: DataFrame with IV lean history
2365
+ - options_cache: dict {date: DataFrame} with options data
2366
+ """
2367
+ if progress_widgets:
2368
+ progress_bar, status_label, monitor, start_time = progress_widgets
2369
+ status_label.value = "<b style='color:#0066cc'>🔄 Preloading options data (ONCE)...</b>"
2370
+ progress_bar.value = 5
2371
+
2372
+ # Extract config
2373
+ from datetime import datetime, timedelta
2374
+ import pandas as pd
2375
+ import numpy as np
2376
+ import gc
2377
+
2378
+ start_date = datetime.strptime(config['start_date'], '%Y-%m-%d').date()
2379
+ end_date = datetime.strptime(config['end_date'], '%Y-%m-%d').date()
2380
+ symbol = config['symbol']
2381
+ dte_target = config.get('dte_target', 30)
2382
+ lookback_period = config.get('lookback_period', 60)
2383
+ chunk_months = config.get('chunk_months', 1) # Default 1 month (~30 days), not 3
2384
+
2385
+ # Calculate date chunks
2386
+ data_start = start_date - timedelta(days=lookback_period + 60)
2387
+
2388
+ date_chunks = []
2389
+ current_chunk_start = data_start
2390
+ while current_chunk_start <= end_date:
2391
+ # Use chunk_days_options if available, otherwise chunk_months * 30
2392
+ chunk_days = config.get('chunk_days_options', chunk_months * 30)
2393
+ chunk_end = min(
2394
+ current_chunk_start + timedelta(days=chunk_days),
2395
+ end_date
2396
+ )
2397
+ date_chunks.append((current_chunk_start, chunk_end))
2398
+ current_chunk_start = chunk_end + timedelta(days=1)
2399
+
2400
+ # Store lean calculations
2401
+ lean_history = []
2402
+ all_options_data = [] # List to collect all options DataFrames
2403
+
2404
+ # Track time for ETA
2405
+ preload_start_time = time.time()
2406
+
2407
+ try:
2408
+ # Use api_call with caching instead of direct ivol API
2409
+ cache_config = config.get('cache_config')
2410
+
2411
+ # Process each chunk
2412
+ for chunk_idx, (chunk_start, chunk_end) in enumerate(date_chunks):
2413
+ if progress_widgets:
2414
+ # Use update_progress for full display with ETA, CPU, RAM
2415
+ update_progress(
2416
+ progress_bar, status_label, monitor,
2417
+ current=chunk_idx + 1,
2418
+ total=len(date_chunks),
2419
+ start_time=preload_start_time,
2420
+ message=f"🔄 Loading chunk {chunk_idx+1}/{len(date_chunks)}"
2421
+ )
2422
+
2423
+ # Use api_call with caching (supports disk + memory cache)
2424
+ raw_data = api_call(
2425
+ '/equities/eod/options-rawiv',
2426
+ cache_config,
2427
+ symbol=symbol,
2428
+ from_=chunk_start.strftime('%Y-%m-%d'),
2429
+ to=chunk_end.strftime('%Y-%m-%d'),
2430
+ debug=cache_config.get('debug', False) if cache_config else False
2431
+ )
2432
+
2433
+ if raw_data is None:
2434
+ continue
2435
+
2436
+ # api_call returns dict with 'data' key
2437
+ if isinstance(raw_data, dict) and 'data' in raw_data:
2438
+ df = pd.DataFrame(raw_data['data'])
2439
+ else:
2440
+ df = pd.DataFrame(raw_data)
2441
+
2442
+ if df.empty:
2443
+ continue
2444
+
2445
+ # Essential columns
2446
+ essential_cols = ['date', 'expiration', 'strike', 'Call/Put', 'iv', 'Adjusted close']
2447
+ if 'bid' in df.columns:
2448
+ essential_cols.append('bid')
2449
+ if 'ask' in df.columns:
2450
+ essential_cols.append('ask')
2451
+
2452
+ df = df[essential_cols].copy()
2453
+
2454
+ # Process bid/ask
2455
+ if 'bid' in df.columns:
2456
+ df['bid'] = pd.to_numeric(df['bid'], errors='coerce').astype('float32')
2457
+ else:
2458
+ df['bid'] = np.nan
2459
+
2460
+ if 'ask' in df.columns:
2461
+ df['ask'] = pd.to_numeric(df['ask'], errors='coerce').astype('float32')
2462
+ else:
2463
+ df['ask'] = np.nan
2464
+
2465
+ # Calculate mid price
2466
+ df['mid'] = (df['bid'] + df['ask']) / 2
2467
+ df['mid'] = df['mid'].fillna(df['iv'])
2468
+
2469
+ df['date'] = pd.to_datetime(df['date']).dt.date
2470
+ df['expiration'] = pd.to_datetime(df['expiration']).dt.date
2471
+ df['strike'] = pd.to_numeric(df['strike'], errors='coerce').astype('float32')
2472
+ df['iv'] = pd.to_numeric(df['iv'], errors='coerce').astype('float32')
2473
+ df['Adjusted close'] = pd.to_numeric(df['Adjusted close'], errors='coerce').astype('float32')
2474
+
2475
+ df['dte'] = (pd.to_datetime(df['expiration']) - pd.to_datetime(df['date'])).dt.days
2476
+ df['dte'] = df['dte'].astype('int16')
2477
+
2478
+ df = df.dropna(subset=['strike', 'iv', 'Adjusted close'])
2479
+
2480
+ if df.empty:
2481
+ del df
2482
+ gc.collect()
2483
+ continue
2484
+
2485
+ # Collect all options data
2486
+ all_options_data.append(df.copy())
2487
+
2488
+ # Calculate lean for this chunk
2489
+ trading_dates = sorted(df['date'].unique())
2490
+
2491
+ for current_date in trading_dates:
2492
+ day_data = df[df['date'] == current_date]
2493
+
2494
+ if day_data.empty:
2495
+ continue
2496
+
2497
+ stock_price = float(day_data['Adjusted close'].iloc[0])
2498
+
2499
+ dte_filtered = day_data[
2500
+ (day_data['dte'] >= dte_target - 7) &
2501
+ (day_data['dte'] <= dte_target + 7)
2502
+ ]
2503
+
2504
+ if dte_filtered.empty:
2505
+ continue
2506
+
2507
+ dte_filtered = dte_filtered.copy()
2508
+ dte_filtered['strike_diff'] = abs(dte_filtered['strike'] - stock_price)
2509
+ atm_idx = dte_filtered['strike_diff'].idxmin()
2510
+ atm_strike = float(dte_filtered.loc[atm_idx, 'strike'])
2511
+
2512
+ atm_options = dte_filtered[dte_filtered['strike'] == atm_strike]
2513
+ atm_call = atm_options[atm_options['Call/Put'] == 'C']
2514
+ atm_put = atm_options[atm_options['Call/Put'] == 'P']
2515
+
2516
+ if not atm_call.empty and not atm_put.empty:
2517
+ call_iv = float(atm_call['iv'].iloc[0])
2518
+ put_iv = float(atm_put['iv'].iloc[0])
2519
+
2520
+ if pd.notna(call_iv) and pd.notna(put_iv) and call_iv > 0 and put_iv > 0:
2521
+ iv_lean = call_iv - put_iv
2522
+
2523
+ lean_history.append({
2524
+ 'date': current_date,
2525
+ 'stock_price': stock_price,
2526
+ 'iv_lean': iv_lean
2527
+ })
2528
+
2529
+ del df, raw_data
2530
+ gc.collect()
2531
+
2532
+ lean_df = pd.DataFrame(lean_history)
2533
+ lean_df['stock_price'] = lean_df['stock_price'].astype('float32')
2534
+ lean_df['iv_lean'] = lean_df['iv_lean'].astype('float32')
2535
+
2536
+ # Combine all options data into single DataFrame
2537
+ if all_options_data:
2538
+ options_df = pd.concat(all_options_data, ignore_index=True)
2539
+ # Ensure date column is properly formatted
2540
+ options_df['date'] = pd.to_datetime(options_df['date']).dt.date
2541
+ options_df['expiration'] = pd.to_datetime(options_df['expiration']).dt.date
2542
+ else:
2543
+ options_df = pd.DataFrame()
2544
+
2545
+ del lean_history, all_options_data
2546
+ gc.collect()
2547
+
2548
+ if progress_widgets:
2549
+ status_label.value = f"<b style='color:#00cc00'>✓ Data preloaded: {len(lean_df)} days, {len(options_df)} options records</b>"
2550
+ progress_bar.value = 35
2551
+
2552
+ print(f"✓ Data preloaded: {len(lean_df)} days, {len(options_df)} options records")
2553
+
2554
+ return lean_df, options_df
2555
+
2556
+ except Exception as e:
2557
+ print(f"Error preloading data: {e}")
2558
+ return pd.DataFrame(), {}
2559
+
2560
+
2561
+ # ============================================================
2562
+ # UNIVERSAL DATA PRELOADER V2 (NEW!)
2563
+ # ============================================================
2564
+ def preload_data_universal(config, data_requests=None):
2565
+ """
2566
+ 🚀 TRULY UNIVERSAL DATA PRELOADER - Works with ANY API endpoint!
2567
+
2568
+ Supports:
2569
+ - EOD data: options-rawiv, stock-prices, ivs-by-delta, ivx, etc.
2570
+ - Intraday data: OPTIONS_INTRADAY, stock intraday, etc.
2571
+ - Any custom endpoint with any parameters
2572
+ - Automatic chunking for date ranges
2573
+ - Manual single-date requests
2574
+
2575
+ Args:
2576
+ config: Strategy configuration (start_date, end_date, symbol)
2577
+ data_requests: List of data requests to load. If None, tries auto-detection.
2578
+
2579
+ Format:
2580
+ [
2581
+ {
2582
+ 'name': 'options_data', # Your name for this dataset
2583
+ 'endpoint': '/equities/eod/options-rawiv',
2584
+ 'params': {...}, # Base params (symbol, etc.)
2585
+ 'chunking': { # Optional: for date-range data
2586
+ 'enabled': True,
2587
+ 'date_param': 'from_', # Param name for start date
2588
+ 'date_param_to': 'to', # Param name for end date
2589
+ 'chunk_days': 90 # Chunk size in days
2590
+ },
2591
+ 'post_process': lambda df: df, # Optional: process DataFrame
2592
+ },
2593
+ {
2594
+ 'name': 'ivx_data',
2595
+ 'endpoint': '/equities/eod/ivx',
2596
+ 'params': {
2597
+ 'symbol': config['symbol'],
2598
+ 'from_': config['start_date'],
2599
+ 'to': config['end_date']
2600
+ },
2601
+ 'chunking': {'enabled': False} # Single request
2602
+ },
2603
+ {
2604
+ 'name': 'options_intraday',
2605
+ 'endpoint': '/equities/intraday/options-rawiv',
2606
+ 'params': {
2607
+ 'symbol': config['symbol']
2608
+ },
2609
+ 'date_list': True, # Load for each date separately
2610
+ 'date_param': 'date'
2611
+ }
2612
+ ]
2613
+
2614
+ Returns:
2615
+ dict: Preloaded data with keys like:
2616
+ {
2617
+ '_preloaded_options_data': DataFrame,
2618
+ '_preloaded_ivx_data': DataFrame,
2619
+ '_preloaded_options_intraday': DataFrame,
2620
+ '_stats': {...}
2621
+ }
2622
+
2623
+ Usage in strategy:
2624
+ # Check for ANY preloaded data
2625
+ if any(k.startswith('_preloaded_') for k in config):
2626
+ options_df = config.get('_preloaded_options_data', pd.DataFrame()).copy()
2627
+ ivx_df = config.get('_preloaded_ivx_data', pd.DataFrame()).copy()
2628
+ else:
2629
+ # Load fresh
2630
+ ...
2631
+ """
2632
+
2633
+ print("\n" + "="*80)
2634
+ print("🚀 UNIVERSAL PRELOADER V2 - Supports ANY endpoint (EOD/Intraday/IVX/etc.)")
2635
+ print("="*80)
2636
+ start_time = time.time()
2637
+
2638
+ # Extract common config
2639
+ start_date = datetime.strptime(config['start_date'], '%Y-%m-%d').date()
2640
+ end_date = datetime.strptime(config['end_date'], '%Y-%m-%d').date()
2641
+ symbol = config['symbol']
2642
+ cache_config = config.get('cache_config', get_cache_config())
2643
+
2644
+ # Auto-detection if not specified
2645
+ if data_requests is None:
2646
+ data_requests = _auto_detect_requests(config)
2647
+ print(f"\n🔍 Auto-detected {len(data_requests)} data requests from config")
2648
+
2649
+ preloaded = {}
2650
+ total_rows = 0
2651
+
2652
+ # Process each data request
2653
+ for req_idx, request in enumerate(data_requests, 1):
2654
+ req_name = request['name']
2655
+ endpoint = request['endpoint']
2656
+ base_params = request.get('params', {})
2657
+ chunking = request.get('chunking', {'enabled': False})
2658
+ post_process = request.get('post_process', None)
2659
+ date_list = request.get('date_list', False)
2660
+
2661
+ print(f"\n[{req_idx}/{len(data_requests)}] 📊 Loading: {req_name}")
2662
+ print(f" Endpoint: {endpoint}")
2663
+
2664
+ all_data = []
2665
+
2666
+ # ========================================================
2667
+ # MODE 1: DATE LIST (one request per date, e.g., intraday)
2668
+ # ========================================================
2669
+ if date_list:
2670
+ date_param = request.get('date_param', 'date')
2671
+ trading_days = pd.bdate_range(start_date, end_date).date
2672
+
2673
+ print(f" Mode: Date list ({len(trading_days)} dates)")
2674
+
2675
+ for day_idx, date in enumerate(trading_days):
2676
+ params = base_params.copy()
2677
+ params[date_param] = date.strftime('%Y-%m-%d')
2678
+
2679
+ if day_idx % max(1, len(trading_days) // 10) == 0:
2680
+ print(f" Progress: {day_idx}/{len(trading_days)} dates...")
2681
+
2682
+ response = api_call(endpoint, cache_config, **params)
2683
+ if response and 'data' in response:
2684
+ df = pd.DataFrame(response['data'])
2685
+ if len(df) > 0:
2686
+ all_data.append(df)
2687
+
2688
+ # ========================================================
2689
+ # MODE 2: CHUNKED LOADING (date ranges in chunks)
2690
+ # ========================================================
2691
+ elif chunking.get('enabled', False):
2692
+ date_param_from = chunking.get('date_param', 'from_')
2693
+ date_param_to = chunking.get('date_param_to', 'to')
2694
+ chunk_days = chunking.get('chunk_days', 30)
2695
+ chunk_size = timedelta(days=chunk_days)
2696
+
2697
+ current = start_date
2698
+ chunks = []
2699
+ while current <= end_date:
2700
+ chunk_end = min(current + chunk_size, end_date)
2701
+ chunks.append((current, chunk_end))
2702
+ current = chunk_end + timedelta(days=1)
2703
+
2704
+ print(f" Mode: Chunked ({len(chunks)} chunks of {chunk_days} days)")
2705
+
2706
+ for chunk_idx, (chunk_start, chunk_end) in enumerate(chunks):
2707
+ params = base_params.copy()
2708
+ params[date_param_from] = chunk_start.strftime('%Y-%m-%d')
2709
+ params[date_param_to] = chunk_end.strftime('%Y-%m-%d')
2710
+
2711
+ if chunk_idx % max(1, len(chunks) // 5) == 0:
2712
+ print(f" Progress: {chunk_idx+1}/{len(chunks)} chunks...")
2713
+
2714
+ response = api_call(endpoint, cache_config, **params)
2715
+ if response and 'data' in response:
2716
+ df = pd.DataFrame(response['data'])
2717
+ if len(df) > 0:
2718
+ all_data.append(df)
2719
+
2720
+ # ========================================================
2721
+ # MODE 3: SINGLE REQUEST (no chunking/date list)
2722
+ # ========================================================
2723
+ else:
2724
+ print(f" Mode: Single request")
2725
+
2726
+ params = base_params.copy()
2727
+ response = api_call(endpoint, cache_config, **params)
2728
+ if response and 'data' in response:
2729
+ df = pd.DataFrame(response['data'])
2730
+ if len(df) > 0:
2731
+ all_data.append(df)
2732
+
2733
+ # ========================================================
2734
+ # COMBINE AND STORE
2735
+ # ========================================================
2736
+ if len(all_data) > 0:
2737
+ combined_df = pd.concat(all_data, ignore_index=True)
2738
+
2739
+ # Apply post-processing if provided
2740
+ if post_process is not None:
2741
+ try:
2742
+ combined_df = post_process(combined_df)
2743
+ except Exception as e:
2744
+ print(f" ⚠️ Post-processing failed: {e}")
2745
+
2746
+ # Auto-process common date columns
2747
+ combined_df = _auto_process_dates(combined_df)
2748
+
2749
+ # Store with standardized key
2750
+ key = f"_preloaded_{req_name}"
2751
+ preloaded[key] = combined_df
2752
+ total_rows += len(combined_df)
2753
+
2754
+ print(f" ✓ Loaded: {len(combined_df):,} rows → {key}")
2755
+ else:
2756
+ print(f" ⚠️ No data returned")
2757
+
2758
+ # ========================================================
2759
+ # SUMMARY
2760
+ # ========================================================
2761
+ elapsed = time.time() - start_time
2762
+
2763
+ # Collect detailed stats for each dataset
2764
+ dataset_details = {}
2765
+ for k in preloaded.keys():
2766
+ if k.startswith('_preloaded_'):
2767
+ dataset_name = k.replace('_preloaded_', '')
2768
+ df = preloaded[k]
2769
+ dataset_details[dataset_name] = {
2770
+ 'rows': len(df),
2771
+ 'endpoint': None
2772
+ }
2773
+
2774
+ # Map dataset names to endpoints from data_requests
2775
+ if data_requests:
2776
+ for req in data_requests:
2777
+ req_name = req.get('name', 'unknown')
2778
+ if req_name in dataset_details:
2779
+ dataset_details[req_name]['endpoint'] = req.get('endpoint', 'unknown')
2780
+
2781
+ preloaded['_stats'] = {
2782
+ 'load_time_seconds': int(elapsed),
2783
+ 'total_rows': total_rows,
2784
+ 'data_count': len([k for k in preloaded.keys() if k.startswith('_preloaded_')]),
2785
+ 'datasets': [k.replace('_preloaded_', '') for k in preloaded.keys() if k.startswith('_preloaded_')],
2786
+ 'dataset_details': dataset_details
2787
+ }
2788
+
2789
+ print(f"\n{'='*80}")
2790
+ print(f"✅ PRELOAD COMPLETE:")
2791
+ print(f" • Time: {int(elapsed)}s")
2792
+ print(f" • Total rows: {total_rows:,}")
2793
+ print(f" • Datasets: {preloaded['_stats']['data_count']}")
2794
+ for ds in preloaded['_stats']['datasets']:
2795
+ print(f" - {ds}")
2796
+ print(f" • Cached in RAM for 4-5x speedup! 🚀")
2797
+ print(f"{'='*80}\n")
2798
+
2799
+ return preloaded
2800
+
2801
+
2802
+ def _auto_detect_requests(config):
2803
+ """Auto-detect what data to load based on config keys"""
2804
+ requests = []
2805
+
2806
+ # Always load options data for options strategies
2807
+ requests.append({
2808
+ 'name': 'options',
2809
+ 'endpoint': '/equities/eod/options-rawiv',
2810
+ 'params': {
2811
+ 'symbol': config['symbol']
2812
+ },
2813
+ 'chunking': {
2814
+ 'enabled': True,
2815
+ 'date_param': 'from_',
2816
+ 'date_param_to': 'to',
2817
+ 'chunk_days': config.get('chunk_days_options', 30)
2818
+ },
2819
+ 'post_process': lambda df: _process_options_df(df)
2820
+ })
2821
+
2822
+ # Load IV surface if strategy uses term structure
2823
+ if any(k in config for k in ['short_tenor', 'long_tenor', 'delta_target']):
2824
+ requests.append({
2825
+ 'name': 'ivs_surface',
2826
+ 'endpoint': '/equities/eod/ivs-by-delta',
2827
+ 'params': {
2828
+ 'symbol': config['symbol'],
2829
+ 'deltaFrom': config.get('delta_target', 0.5) - 0.05,
2830
+ 'deltaTo': config.get('delta_target', 0.5) + 0.05,
2831
+ 'periodFrom': config.get('short_tenor', 30) - 7,
2832
+ 'periodTo': config.get('long_tenor', 90) + 7
2833
+ },
2834
+ 'chunking': {
2835
+ 'enabled': True,
2836
+ 'date_param': 'from_',
2837
+ 'date_param_to': 'to',
2838
+ 'chunk_days': config.get('chunk_days_options', 30)
2839
+ }
2840
+ })
2841
+
2842
+ # Load stock prices
2843
+ requests.append({
2844
+ 'name': 'stock',
2845
+ 'endpoint': '/equities/eod/stock-prices',
2846
+ 'params': {
2847
+ 'symbol': config['symbol']
2848
+ },
2849
+ 'chunking': {
2850
+ 'enabled': True,
2851
+ 'date_param': 'from_',
2852
+ 'date_param_to': 'to',
2853
+ 'chunk_days': config.get('chunk_days_stock', 180) # Stock data is lightweight
2854
+ }
2855
+ })
2856
+
2857
+ return requests
2858
+
2859
+
2860
+ def _process_options_df(df):
2861
+ """Process options DataFrame: dates + DTE + OPTIMIZATIONS (5-10x faster!)"""
2862
+ # Basic date processing
2863
+ if 'date' in df.columns:
2864
+ df['date'] = pd.to_datetime(df['date']).dt.date
2865
+ if 'expiration' in df.columns:
2866
+ df['expiration'] = pd.to_datetime(df['expiration']).dt.date
2867
+
2868
+ if 'date' in df.columns and 'expiration' in df.columns:
2869
+ df = df.copy()
2870
+ df['dte'] = (pd.to_datetime(df['expiration']) -
2871
+ pd.to_datetime(df['date'])).dt.days
2872
+
2873
+ # ========================================================
2874
+ # CRITICAL: SORT BY DATE FIRST! (Required for time-series)
2875
+ # ========================================================
2876
+ if 'date' in df.columns:
2877
+ # Check if already sorted (skip if yes, fast!)
2878
+ if not df['date'].is_monotonic_increasing:
2879
+ df = df.sort_values('date') # ✅ Sort only if needed
2880
+
2881
+ # ========================================================
2882
+ # AUTOMATIC OPTIMIZATIONS (applied by library)
2883
+ # ========================================================
2884
+
2885
+ # These optimizations are SAFE to apply automatically:
2886
+ # - Categorical types for low-cardinality columns
2887
+ # - Optimized numeric types (float32/int16 instead of float64/int64)
2888
+ #
2889
+ # NOTE: We do NOT set index on 'date' in library functions because:
2890
+ # - It breaks existing code that uses .loc with non-date indices
2891
+ # - Requires all strategies to handle Series vs scalar results
2892
+
2893
+ # Convert Call/Put to categorical (60% less RAM, 2x faster filtering)
2894
+ if 'Call/Put' in df.columns:
2895
+ df['Call/Put'] = df['Call/Put'].astype('category')
2896
+
2897
+ # Optimize data types (50% less RAM)
2898
+ # float32 for prices (4 bytes instead of 8, enough precision)
2899
+ float32_cols = ['strike', 'bid', 'ask', 'iv', 'price', 'mid', 'delta', 'gamma', 'vega', 'theta']
2900
+ for col in float32_cols:
2901
+ if col in df.columns:
2902
+ df[col] = pd.to_numeric(df[col], errors='coerce').astype('float32')
2903
+
2904
+ # int16 for DTE (2 bytes instead of 8, max 32767 days)
2905
+ if 'dte' in df.columns:
2906
+ df['dte'] = df['dte'].astype('int16')
2907
+
2908
+ return df
2909
+
2910
+
2911
+ def _auto_process_dates(df):
2912
+ """Auto-process common date columns + SORT BY DATE"""
2913
+ date_columns = ['date', 'expiration', 'trade_date', 'time']
2914
+
2915
+ for col in date_columns:
2916
+ if col in df.columns:
2917
+ try:
2918
+ if col == 'time':
2919
+ # Keep time as string or datetime
2920
+ pass
2921
+ else:
2922
+ df[col] = pd.to_datetime(df[col]).dt.date
2923
+ except:
2924
+ pass # Already in correct format or not a date
2925
+
2926
+ # ========================================================
2927
+ # CRITICAL: SORT BY DATE! (Required for time-series)
2928
+ # ========================================================
2929
+ if 'date' in df.columns:
2930
+ # Check if already sorted (O(1) check vs O(N log N) sort)
2931
+ if not df['date'].is_monotonic_increasing:
2932
+ df = df.sort_values('date') # ✅ Sort only if needed
2933
+ elif 'trade_date' in df.columns:
2934
+ if not df['trade_date'].is_monotonic_increasing:
2935
+ df = df.sort_values('trade_date') # Alternative date column
2936
+
2937
+ return df
2938
+
2939
+
2940
+ # ============================================================
2941
+ # NEW: OPTIMIZATION FRAMEWORK
2942
+ # ============================================================
2943
+ def optimize_parameters(base_config, param_grid, strategy_function,
2944
+ optimization_metric='sharpe', min_trades=5,
2945
+ max_drawdown_limit=None, parallel=False,
2946
+ export_each_combo=True, # ← NEW PARAMETER
2947
+ optimization_config=None, # ← NEW PARAMETER FOR PRESETS
2948
+ results_folder=None # ← NEW: Use existing folder or create new
2949
+ ):
2950
+ """
2951
+ Optimize strategy parameters across multiple combinations
2952
+
2953
+ Args:
2954
+ base_config: Base configuration dict
2955
+ param_grid: Dict of parameters to optimize
2956
+ Example: {'z_score_entry': [1.0, 1.5, 2.0], 'z_score_exit': [0.1, 0.3, 0.5]}
2957
+ strategy_function: Strategy function to run
2958
+ optimization_metric: Metric to optimize ('sharpe', 'total_return', 'total_pnl', 'profit_factor', 'calmar')
2959
+ min_trades: Minimum number of trades required
2960
+ max_drawdown_limit: Maximum acceptable drawdown (e.g., 0.10 for 10%)
2961
+ parallel: Use parallel processing (not implemented yet)
2962
+ export_each_combo: If True, exports files for each combination # ←
2963
+
2964
+ Returns:
2965
+ tuple: (results_df, best_params, results_folder)
2966
+ """
2967
+
2968
+ # Check if optimization_config has preset and apply it automatically
2969
+ if optimization_config and isinstance(optimization_config, dict) and 'preset' in optimization_config:
2970
+ preset = optimization_config['preset']
2971
+ print(f"🔄 Auto-applying preset: {preset}")
2972
+ apply_optimization_preset(optimization_config, preset)
2973
+ print_preset_info(optimization_config)
2974
+
2975
+ # Use preset parameters for grid and validation criteria
2976
+ param_grid = optimization_config['param_grid']
2977
+ min_trades = optimization_config['min_trades']
2978
+ max_drawdown_limit = optimization_config['max_drawdown_limit']
2979
+
2980
+ # Use optimization_config for optimization_metric if available
2981
+ if 'optimization_metric' in optimization_config:
2982
+ optimization_metric = optimization_config['optimization_metric']
2983
+
2984
+ # Use optimization_config for execution settings if available
2985
+ if 'parallel' in optimization_config:
2986
+ parallel = optimization_config['parallel']
2987
+ if 'export_each_combo' in optimization_config:
2988
+ export_each_combo = optimization_config['export_each_combo']
2989
+
2990
+ # ═══ ADD AT THE BEGINNING OF FUNCTION ═══
2991
+ # Create results folder (or use provided one)
2992
+ if results_folder is None:
2993
+ results_folder = create_optimization_folder()
2994
+ print(f"📊 Results will be saved to: {results_folder}\n")
2995
+ else:
2996
+ print(f"📊 Using existing results folder: {results_folder}\n")
2997
+
2998
+ # Record start time
2999
+ optimization_start_time = datetime.now()
3000
+ start_time_str = optimization_start_time.strftime('%Y-%m-%d %H:%M:%S')
3001
+
3002
+ print("\n" + "="*80)
3003
+ print(" "*20 + "PARAMETER OPTIMIZATION")
3004
+ print("="*80)
3005
+ print(f"Strategy: {base_config.get('strategy_name', 'Unknown')}")
3006
+ print(f"Period: {base_config.get('start_date')} to {base_config.get('end_date')}")
3007
+ print(f"Optimization Metric: {optimization_metric}")
3008
+ print(f"Min Trades: {min_trades}")
3009
+ print(f"🕐 Started: {start_time_str}")
3010
+ if max_drawdown_limit:
3011
+ print(f"Max Drawdown Limit: {max_drawdown_limit*100:.0f}%")
3012
+ print("="*80 + "\n")
3013
+
3014
+ # Generate all combinations
3015
+ param_names = list(param_grid.keys())
3016
+ param_values = list(param_grid.values())
3017
+ all_combinations = list(product(*param_values))
3018
+
3019
+ total_combinations = len(all_combinations)
3020
+ print(f"Testing {total_combinations} parameter combinations...")
3021
+ print(f"Parameters: {param_names}")
3022
+ print(f"Grid: {param_grid}\n")
3023
+
3024
+ # Create SHARED progress context for all backtests
3025
+ try:
3026
+ from IPython.display import display
3027
+ import ipywidgets as widgets
3028
+
3029
+ progress_bar = widgets.FloatProgress(
3030
+ value=0, min=0, max=100,
3031
+ description='Optimizing:',
3032
+ bar_style='info',
3033
+ layout=widgets.Layout(width='100%', height='30px')
3034
+ )
3035
+
3036
+ status_label = widgets.HTML(value="<b>Starting optimization...</b>")
3037
+ display(widgets.VBox([progress_bar, status_label]))
3038
+
3039
+ monitor = ResourceMonitor()
3040
+ opt_start_time = time.time()
3041
+
3042
+ # Create shared progress context (will suppress individual backtest progress)
3043
+ shared_progress = {
3044
+ 'progress_widgets': (progress_bar, status_label, monitor, opt_start_time),
3045
+ 'is_optimization': True
3046
+ }
3047
+ has_widgets = True
3048
+ except:
3049
+ shared_progress = None
3050
+ has_widgets = False
3051
+ print("Running optimization (no progress bar)...")
3052
+
3053
+ # ═══════════════════════════════════════════════════════════════════════════
3054
+ # DEPRECATED: optimize_parameters should NOT preload data internally!
3055
+ # Data should be preloaded BEFORE calling optimize_parameters using preload_data_universal()
3056
+ # ═══════════════════════════════════════════════════════════════════════════
3057
+ # Check if data is already preloaded
3058
+ preloaded_keys = [k for k in base_config.keys() if k.startswith('_preloaded_')]
3059
+
3060
+ # Initialize these variables for backward compatibility
3061
+ preloaded_lean_df = None
3062
+ preloaded_options_df = None
3063
+ use_legacy_preload = False
3064
+
3065
+ if not preloaded_keys:
3066
+ # Fallback: use old preload_options_data (for backward compatibility)
3067
+ print("\n" + "="*80)
3068
+ print("📥 PRELOADING OPTIONS DATA (loads ONCE, reused for all combinations)")
3069
+ print("="*80)
3070
+ print("⚠️ WARNING: Data not preloaded! Using deprecated preload_options_data()")
3071
+ print("⚠️ Recommendation: Use preload_data_universal() before calling optimize_parameters()")
3072
+ print("="*80)
3073
+
3074
+ preloaded_lean_df, preloaded_options_df = preload_options_data(
3075
+ base_config,
3076
+ progress_widgets=shared_progress['progress_widgets'] if shared_progress else None
3077
+ )
3078
+
3079
+ if preloaded_lean_df.empty:
3080
+ print("\n❌ ERROR: Failed to preload data. Cannot proceed with optimization.")
3081
+ return pd.DataFrame(), None
3082
+
3083
+ use_legacy_preload = True
3084
+ print(f"✓ Preloading complete! Data will be reused for all {total_combinations} combinations")
3085
+ print("="*80 + "\n")
3086
+ else:
3087
+ print("\n✓ Using preloaded data from preload_data_universal() (recommended method)\n")
3088
+
3089
+ # ═══════════════════════════════════════════════════════════════════════════
3090
+ # RESET PROGRESS BAR FOR OPTIMIZATION LOOP
3091
+ # ═══════════════════════════════════════════════════════════════════════════
3092
+ if has_widgets:
3093
+ progress_bar.value = 0
3094
+ progress_bar.bar_style = 'info'
3095
+ status_label.value = "<b style='color:#0066cc'>Starting optimization loop...</b>"
3096
+
3097
+ # Run backtests
3098
+ results = []
3099
+ start_time = time.time()
3100
+
3101
+ for idx, param_combo in enumerate(all_combinations, 1):
3102
+ # Create test config
3103
+ test_config = base_config.copy()
3104
+
3105
+ # Update parameters
3106
+ for param_name, param_value in zip(param_names, param_combo):
3107
+ test_config[param_name] = param_value
3108
+
3109
+ # Update name
3110
+ param_str = "_".join([f"{k}={v}" for k, v in zip(param_names, param_combo)])
3111
+ test_config['strategy_name'] = f"{base_config.get('strategy_name', 'Strategy')} [{param_str}]"
3112
+
3113
+ # ═══ ADD PRELOADED DATA TO CONFIG ═══
3114
+ # Only add legacy preloaded data if it was loaded by preload_options_data
3115
+ if use_legacy_preload:
3116
+ test_config['_preloaded_lean_df'] = preloaded_lean_df
3117
+ test_config['_preloaded_options_cache'] = preloaded_options_df
3118
+ # Otherwise, data is already in base_config from preload_data_universal
3119
+
3120
+ # Update progress
3121
+ if has_widgets:
3122
+ # Use update_progress for full display with ETA, CPU, RAM
3123
+ update_progress(
3124
+ progress_bar, status_label, monitor,
3125
+ current=idx,
3126
+ total=total_combinations,
3127
+ start_time=start_time,
3128
+ message=f"Testing: {param_str}"
3129
+ )
3130
+ else:
3131
+ if idx % max(1, total_combinations // 10) == 0:
3132
+ print(f"[{idx}/{total_combinations}] {param_str}")
3133
+
3134
+ # ═══ MODIFY run_backtest CALL (lines ~2240-2248) ═══
3135
+ try:
3136
+ # Create compact parameter string (e.g., Z1.0_E0.1_PT20)
3137
+ param_parts = []
3138
+ for name, value in zip(param_names, param_combo):
3139
+ if 'z_score_entry' in name:
3140
+ param_parts.append(f"Z{value}")
3141
+ elif 'z_score_exit' in name:
3142
+ param_parts.append(f"E{value}")
3143
+ elif 'profit_target' in name:
3144
+ if value is None:
3145
+ param_parts.append("PTNo")
3146
+ else:
3147
+ param_parts.append(f"PT{int(value*100)}")
3148
+ elif 'min_days' in name:
3149
+ param_parts.append(f"D{value}")
3150
+ else:
3151
+ # Generic short name for other params
3152
+ short_name = ''.join([c for c in name if c.isupper() or c.isdigit()])[:3]
3153
+ param_parts.append(f"{short_name}{value}")
3154
+
3155
+ compact_params = "_".join(param_parts)
3156
+
3157
+ # Create combo folder: c01_Z1.0_E0.1_PT20
3158
+ combo_folder = os.path.join(results_folder, f'c{idx:02d}_{compact_params}')
3159
+ os.makedirs(combo_folder, exist_ok=True)
3160
+
3161
+ # File prefix: c01_Z1.0_E0.1_PT20
3162
+ combo_prefix = f"c{idx:02d}_{compact_params}"
3163
+
3164
+ # Run backtest WITH EXPORT AND CHARTS (saved but not displayed)
3165
+ analyzer = run_backtest(
3166
+ strategy_function,
3167
+ test_config,
3168
+ print_report=False,
3169
+ create_charts=export_each_combo, # ← CREATE CHARTS (saved but not displayed)
3170
+ export_results=export_each_combo, # ← MODIFIED
3171
+ progress_context=shared_progress,
3172
+ chart_filename=os.path.join(combo_folder, 'equity_curve.png') if export_each_combo else None, # ← CHARTS SAVED
3173
+ export_prefix=os.path.join(combo_folder, combo_prefix) if export_each_combo else None # ← ADDED
3174
+ )
3175
+
3176
+ # Check validity
3177
+ is_valid = True
3178
+ invalid_reason = ""
3179
+
3180
+ if analyzer.metrics['total_trades'] < min_trades:
3181
+ is_valid = False
3182
+ invalid_reason = f"Too few trades ({analyzer.metrics['total_trades']})"
3183
+
3184
+ if max_drawdown_limit and analyzer.metrics['max_drawdown'] > (max_drawdown_limit * 100):
3185
+ is_valid = False
3186
+ invalid_reason = f"Excessive drawdown ({analyzer.metrics['max_drawdown']:.1f}%)"
3187
+
3188
+ # Print compact statistics for this combination
3189
+ status_symbol = "✓" if is_valid else "✗"
3190
+ status_color = "#00cc00" if is_valid else "#ff6666"
3191
+
3192
+ # Print combination header
3193
+ print(f"[{idx}/{total_combinations}] {param_str}")
3194
+ print("-" * 100)
3195
+
3196
+ # Print chart file if created
3197
+ if hasattr(analyzer, 'chart_file') and analyzer.chart_file:
3198
+ print(f"Chart saved: {analyzer.chart_file}")
3199
+
3200
+ # Print exported files
3201
+ if hasattr(analyzer, 'exported_files') and analyzer.exported_files:
3202
+ for file_path, extra_info in analyzer.exported_files:
3203
+ if extra_info:
3204
+ print(f"Exported: {file_path} {extra_info}")
3205
+ else:
3206
+ print(f"Exported: {file_path}")
3207
+
3208
+ # Print metrics with separator
3209
+ print("+" * 100)
3210
+ if is_valid:
3211
+ print(f" {status_symbol} Return: {analyzer.metrics['total_return']:>7.2f}% | "
3212
+ f"Sharpe: {analyzer.metrics['sharpe']:>6.2f} | "
3213
+ f"Max DD: {analyzer.metrics['max_drawdown']:>6.2f}% | "
3214
+ f"Trades: {analyzer.metrics['total_trades']:>3} | "
3215
+ f"Win Rate: {analyzer.metrics['win_rate']:>5.1f}% | "
3216
+ f"PF: {analyzer.metrics['profit_factor']:>5.2f}")
3217
+ else:
3218
+ print(f" {status_symbol} INVALID: {invalid_reason}")
3219
+ print("+" * 100 + "\n")
3220
+
3221
+ # Update widget status with last result
3222
+ if has_widgets:
3223
+ result_text = f"Return: {analyzer.metrics['total_return']:.1f}% | Sharpe: {analyzer.metrics['sharpe']:.2f}" if is_valid else invalid_reason
3224
+
3225
+ # Get resource usage
3226
+ cpu_pct = monitor.get_cpu_percent()
3227
+ mem_info = monitor.get_memory_info()
3228
+ ram_mb = mem_info[0] # process_mb
3229
+ resource_text = f"CPU: {cpu_pct:.0f}% | RAM: {ram_mb:.0f}MB"
3230
+
3231
+ status_label.value = (
3232
+ f"<b style='color:{status_color}'>[{idx}/{total_combinations}] {param_str}</b><br>"
3233
+ f"<span style='color:#666'>{result_text}</span><br>"
3234
+ f"<span style='color:#999;font-size:10px'>{resource_text}</span>"
3235
+ )
3236
+
3237
+ # Store results
3238
+ result = {
3239
+ 'combination_id': idx,
3240
+ 'is_valid': is_valid,
3241
+ 'invalid_reason': invalid_reason,
3242
+ **{name: value for name, value in zip(param_names, param_combo)},
3243
+ 'total_return': analyzer.metrics['total_return'],
3244
+ 'sharpe': analyzer.metrics['sharpe'],
3245
+ 'sortino': analyzer.metrics['sortino'],
3246
+ 'calmar': analyzer.metrics['calmar'],
3247
+ 'max_drawdown': analyzer.metrics['max_drawdown'],
3248
+ 'win_rate': analyzer.metrics['win_rate'],
3249
+ 'profit_factor': analyzer.metrics['profit_factor'],
3250
+ 'total_trades': analyzer.metrics['total_trades'],
3251
+ 'avg_win': analyzer.metrics['avg_win'],
3252
+ 'avg_loss': analyzer.metrics['avg_loss'],
3253
+ 'volatility': analyzer.metrics['volatility'],
3254
+ }
3255
+
3256
+ results.append(result)
3257
+
3258
+ # Show intermediate summary every 10 combinations (or at end)
3259
+ if idx % 10 == 0 or idx == total_combinations:
3260
+ valid_so_far = [r for r in results if r['is_valid']]
3261
+ if valid_so_far:
3262
+ print("\n" + "="*80)
3263
+ print(f"INTERMEDIATE SUMMARY ({idx}/{total_combinations} tested)")
3264
+ print("="*80)
3265
+
3266
+ # Sort by optimization metric
3267
+ if optimization_metric == 'sharpe':
3268
+ valid_so_far.sort(key=lambda x: x['sharpe'], reverse=True)
3269
+ elif optimization_metric == 'total_return':
3270
+ valid_so_far.sort(key=lambda x: x['total_return'], reverse=True)
3271
+ elif optimization_metric == 'total_pnl':
3272
+ valid_so_far.sort(key=lambda x: x['total_pnl'], reverse=True)
3273
+ elif optimization_metric == 'profit_factor':
3274
+ valid_so_far.sort(key=lambda x: x['profit_factor'], reverse=True)
3275
+ elif optimization_metric == 'calmar':
3276
+ valid_so_far.sort(key=lambda x: x['calmar'], reverse=True)
3277
+
3278
+ # Show top 3
3279
+ print(f"\n🏆 TOP 3 BY {optimization_metric.upper()}:")
3280
+ print("-"*80)
3281
+ for rank, res in enumerate(valid_so_far[:3], 1):
3282
+ params_display = ", ".join([f"{name}={res[name]}" for name in param_names])
3283
+ print(f" {rank}. [{params_display}]")
3284
+ print(f" Return: {res['total_return']:>7.2f}% | "
3285
+ f"Sharpe: {res['sharpe']:>6.2f} | "
3286
+ f"Max DD: {res['max_drawdown']:>6.2f}% | "
3287
+ f"Trades: {res['total_trades']:>3}")
3288
+
3289
+ print(f"\nValid: {len(valid_so_far)}/{idx} | "
3290
+ f"Invalid: {idx - len(valid_so_far)}/{idx}")
3291
+ print("="*80 + "\n")
3292
+
3293
+ except Exception as e:
3294
+ print(f"\n[{idx}/{total_combinations}] {param_str}")
3295
+ print("-" * 80)
3296
+ print(f" ✗ ERROR: {str(e)}")
3297
+ import traceback
3298
+ print(" Full traceback:")
3299
+ traceback.print_exc()
3300
+
3301
+ result = {
3302
+ 'combination_id': idx,
3303
+ 'is_valid': False,
3304
+ 'invalid_reason': f"Error: {str(e)[:50]}",
3305
+ **{name: value for name, value in zip(param_names, param_combo)},
3306
+ 'total_return': 0, 'sharpe': 0, 'sortino': 0, 'calmar': 0,
3307
+ 'max_drawdown': 0, 'win_rate': 0, 'profit_factor': 0,
3308
+ 'total_trades': 0, 'avg_win': 0, 'avg_loss': 0, 'volatility': 0
3309
+ }
3310
+ results.append(result)
3311
+
3312
+ elapsed = time.time() - start_time
3313
+
3314
+ if has_widgets:
3315
+ progress_bar.value = 100
3316
+ progress_bar.bar_style = 'success'
3317
+ status_label.value = f"<b style='color:#00cc00'>✓ Optimization complete in {int(elapsed)}s</b>"
3318
+
3319
+ # Create results DataFrame
3320
+ results_df = pd.DataFrame(results)
3321
+
3322
+ # Round numeric columns to 2 decimals
3323
+ numeric_columns = results_df.select_dtypes(include=['float64', 'float32', 'float']).columns
3324
+ for col in numeric_columns:
3325
+ results_df[col] = results_df[col].round(5)
3326
+
3327
+ # ═══ ADD SUMMARY SAVE TO FOLDER ═══
3328
+ summary_path = os.path.join(results_folder, 'optimization_summary.csv')
3329
+ results_df.to_csv(summary_path, index=False)
3330
+ print(f"\n✓ Summary saved: {summary_path}")
3331
+
3332
+ # Find best parameters
3333
+ valid_results = results_df[results_df['is_valid'] == True].copy()
3334
+
3335
+ if len(valid_results) == 0:
3336
+ print("\n" + "="*80)
3337
+ print("WARNING: No valid combinations found!")
3338
+ print("Try relaxing constraints or checking parameter ranges")
3339
+ print("="*80)
3340
+ return results_df, None, results_folder
3341
+
3342
+ # Select best based on metric
3343
+ if optimization_metric == 'sharpe':
3344
+ best_idx = valid_results['sharpe'].idxmax()
3345
+ elif optimization_metric == 'total_return':
3346
+ best_idx = valid_results['total_return'].idxmax()
3347
+ elif optimization_metric == 'total_pnl':
3348
+ best_idx = valid_results['total_pnl'].idxmax()
3349
+ elif optimization_metric == 'profit_factor':
3350
+ best_idx = valid_results['profit_factor'].idxmax()
3351
+ elif optimization_metric == 'calmar':
3352
+ best_idx = valid_results['calmar'].idxmax()
3353
+ else:
3354
+ best_idx = valid_results['sharpe'].idxmax()
3355
+
3356
+ best_result = valid_results.loc[best_idx]
3357
+
3358
+ # Extract best parameters
3359
+ best_params = {name: best_result[name] for name in param_names}
3360
+
3361
+ # Add stop_loss_pct if it exists in config (it's handled separately in notebook)
3362
+ if 'stop_loss_config' in base_config and base_config['stop_loss_config']:
3363
+ stop_loss_value = base_config['stop_loss_config'].get('value')
3364
+ if stop_loss_value is not None:
3365
+ best_params['stop_loss_pct'] = stop_loss_value
3366
+
3367
+ # Calculate total time
3368
+ optimization_end_time = datetime.now()
3369
+ total_duration = optimization_end_time - optimization_start_time
3370
+ end_time_str = optimization_end_time.strftime('%Y-%m-%d %H:%M:%S')
3371
+ duration_str = format_time(total_duration.total_seconds())
3372
+
3373
+ # Print summary
3374
+ print("\n" + "="*120)
3375
+ print(" "*31 + "🏆 OPTIMIZATION COMPLETE 🏆")
3376
+ print(" "*31 + "=========================")
3377
+ print(f" • Started : {start_time_str}")
3378
+ print(f" • Finished : {end_time_str}")
3379
+ print(f" • Total Duration : {duration_str} ({int(total_duration.total_seconds())} seconds)")
3380
+ print(f" • Average per run : {total_duration.total_seconds() / total_combinations:.1f} seconds")
3381
+ print(f" • Total combinations : {total_combinations}")
3382
+ print(f" • Valid combinations : {len(valid_results)}")
3383
+ print(f" • Invalid combinations : {len(results_df) - len(valid_results)}")
3384
+
3385
+ print(f"\n📈 OPTIMIZATION METRIC:")
3386
+ print(f" • Metric optimized : {optimization_metric.upper()}")
3387
+
3388
+ # Format best parameters in one line (with special formatting for stop_loss_pct)
3389
+ param_parts = []
3390
+ for name, value in best_params.items():
3391
+ if name == 'stop_loss_pct':
3392
+ param_parts.append(f"stop_loss={value*100:.0f}%")
3393
+ else:
3394
+ param_parts.append(f"{name}={value}")
3395
+ param_str = ", ".join(param_parts)
3396
+ print(f" • Best parameters : {param_str}")
3397
+
3398
+ # Add intraday stop-loss info if enabled
3399
+ intraday_stops = base_config.get('intraday_stops', {})
3400
+ if intraday_stops.get('enabled', False):
3401
+ intraday_pct = intraday_stops.get('stop_pct', 0.03) * 100
3402
+ intraday_days = intraday_stops.get('min_days_before_intraday', 3)
3403
+ print(f" • Intraday stop-loss : Enabled ({intraday_pct:.0f}% after {intraday_days} days)")
3404
+
3405
+ print(f"\n🏆 BEST PERFORMANCE:")
3406
+ print(f" • Total Return : {best_result['total_return']:>10.2f}%")
3407
+ print(f" • Sharpe Ratio : {best_result['sharpe']:>10.2f}")
3408
+ print(f" • Max Drawdown : {best_result['max_drawdown']:>10.2f}%")
3409
+ print(f" • Win Rate : {best_result['win_rate']:>10.1f}%")
3410
+ print(f" • Profit Factor : {best_result['profit_factor']:>10.2f}")
3411
+ print(f" • Total Trades : {best_result['total_trades']:>10.0f}")
3412
+
3413
+ print(f"\n🔌 API ENDPOINTS:")
3414
+ # Extract real endpoints from preloaded data stats
3415
+ endpoints_info = []
3416
+
3417
+ if '_stats' in base_config and 'dataset_details' in base_config['_stats']:
3418
+ dataset_details = base_config['_stats']['dataset_details']
3419
+ for dataset_name, info in dataset_details.items():
3420
+ endpoint = info.get('endpoint')
3421
+ rows = info.get('rows', 0)
3422
+ if endpoint:
3423
+ endpoints_info.append((endpoint, rows))
3424
+
3425
+ # Check if intraday stops are enabled
3426
+ intraday_stops = base_config.get('intraday_stops', {})
3427
+ if intraday_stops.get('enabled', False):
3428
+ intraday_endpoint = "/equities/intraday/stock-prices"
3429
+ if not any(ep[0] == intraday_endpoint for ep in endpoints_info):
3430
+ endpoints_info.append((intraday_endpoint, "on-demand"))
3431
+
3432
+ if endpoints_info:
3433
+ for idx, (endpoint, rows) in enumerate(endpoints_info, 1):
3434
+ if isinstance(rows, int):
3435
+ print(f" {idx}. {endpoint:<45} ({rows:>10,} rows)")
3436
+ else:
3437
+ print(f" {idx}. {endpoint:<45} ({rows})")
3438
+ else:
3439
+ # Fallback to static list if no stats available
3440
+ print(f" 1. /equities/eod/options-rawiv")
3441
+ print(f" 2. /equities/eod/stock-prices")
3442
+ if intraday_stops.get('enabled', False):
3443
+ print(f" 3. /equities/intraday/stock-prices")
3444
+
3445
+ print("="*120)
3446
+
3447
+ # ═══════════════════════════════════════════════════════════════════════════
3448
+ # NEW! FULL BACKTEST OF BEST COMBINATION WITH ALL CHARTS
3449
+ # ═══════════════════════════════════════════════════════════════════════════
3450
+ print("\n" + "="*80)
3451
+ print(" "*15 + "RUNNING FULL BACKTEST FOR BEST COMBINATION")
3452
+ print("="*80)
3453
+ print("\n📊 Creating detailed report for best combination...")
3454
+ print(f"Parameters: {', '.join([f'{k}={v}' for k, v in best_params.items()])}\n")
3455
+
3456
+ # Create config for best combination
3457
+ best_config = base_config.copy()
3458
+ best_config.update(best_params)
3459
+ if use_legacy_preload:
3460
+ best_config['_preloaded_lean_df'] = preloaded_lean_df
3461
+ best_config['_preloaded_options_cache'] = preloaded_options_df
3462
+
3463
+ # Create folder for best combination
3464
+ best_combo_folder = os.path.join(results_folder, 'best_combination')
3465
+ os.makedirs(best_combo_folder, exist_ok=True)
3466
+
3467
+ # Run FULL backtest with ALL charts and exports
3468
+ # Note: progress_context=None, so plt.show() will be called but fail due to renderer
3469
+ # We'll display charts explicitly afterwards using IPython.display.Image
3470
+ best_analyzer = run_backtest(
3471
+ strategy_function,
3472
+ best_config,
3473
+ print_report=True, # ← SHOW FULL REPORT
3474
+ create_charts=True, # ← CREATE ALL CHARTS
3475
+ export_results=True, # ← EXPORT ALL FILES
3476
+ progress_context=None, # ← Normal mode
3477
+ chart_filename=os.path.join(best_combo_folder, 'equity_curve.png'),
3478
+ export_prefix=os.path.join(best_combo_folder, 'best')
3479
+ )
3480
+
3481
+ # Save detailed metrics to optimization_metrics.csv
3482
+ metrics_data = {
3483
+ 'metric': list(best_analyzer.metrics.keys()),
3484
+ 'value': list(best_analyzer.metrics.values())
3485
+ }
3486
+ metrics_df = pd.DataFrame(metrics_data)
3487
+ metrics_path = os.path.join(results_folder, 'optimization_metrics.csv')
3488
+ metrics_df.to_csv(metrics_path, index=False)
3489
+
3490
+ print(f"\n✓ Detailed metrics saved: {metrics_path}")
3491
+ print(f"✓ Best combination results saved to: {best_combo_folder}/")
3492
+
3493
+ # ═══════════════════════════════════════════════════════════════════════════
3494
+ # DISPLAY CHARTS FOR BEST COMBINATION IN NOTEBOOK
3495
+ # ═══════════════════════════════════════════════════════════════════════════
3496
+ try:
3497
+ # Charts are displayed in the notebook, not here
3498
+ chart_file = os.path.join(best_combo_folder, 'equity_curve.png')
3499
+ if os.path.exists(chart_file):
3500
+ print(f"\n📈 Best combination charts saved to: {chart_file}")
3501
+ except Exception as e:
3502
+ print(f"\n⚠ Could not display charts (saved to {best_combo_folder}/): {e}")
3503
+
3504
+ # ═══════════════════════════════════════════════════════════════════════════
3505
+ # CREATE OPTIMIZATION COMPARISON CHARTS (save only, display in notebook manually)
3506
+ # ═══════════════════════════════════════════════════════════════════════════
3507
+ print("\n" + "="*80)
3508
+ print(" "*15 + "CREATING OPTIMIZATION COMPARISON CHARTS")
3509
+ print("="*80)
3510
+ try:
3511
+ optimization_chart_path = os.path.join(results_folder, 'optimization_results.png')
3512
+ # Save chart but don't display (show_plot=False) - display will be done in notebook for combined results
3513
+ plot_optimization_results(
3514
+ results_df,
3515
+ param_names,
3516
+ filename=optimization_chart_path,
3517
+ show_plot=False # Don't display here - will be shown in notebook for combined results
3518
+ )
3519
+ print(f"✓ Optimization comparison charts saved to: {optimization_chart_path}")
3520
+ print(" (Chart will be displayed in notebook for combined results)")
3521
+ except Exception as e:
3522
+ print(f"⚠ Could not create optimization charts: {e}")
3523
+ import traceback
3524
+ traceback.print_exc()
3525
+
3526
+ print("="*80 + "\n")
3527
+
3528
+ return results_df, best_params, results_folder
3529
+
3530
+
3531
+ def plot_optimization_results(results_df, param_names, filename='optimization_results.png', show_plot=True):
3532
+ """
3533
+ Create visualization of optimization results
3534
+
3535
+ Args:
3536
+ results_df: Results DataFrame from optimize_parameters()
3537
+ param_names: List of parameter names
3538
+ filename: Output filename
3539
+ show_plot: If True, display plot in Jupyter notebook (default: True)
3540
+ """
3541
+ import matplotlib.pyplot as plt
3542
+ import seaborn as sns
3543
+
3544
+ # Handle missing is_valid column (for combined results from multiple optimizations)
3545
+ if 'is_valid' not in results_df.columns:
3546
+ results_df = results_df.copy()
3547
+ results_df['is_valid'] = True
3548
+
3549
+ valid_results = results_df[results_df['is_valid'] == True].copy()
3550
+
3551
+ if valid_results.empty:
3552
+ print("No valid results to plot")
3553
+ return
3554
+
3555
+ sns.set_style("whitegrid")
3556
+
3557
+ fig = plt.figure(figsize=(18, 12))
3558
+
3559
+ # 1. Sharpe vs Total Return scatter
3560
+ ax1 = plt.subplot(2, 3, 1)
3561
+ scatter = ax1.scatter(
3562
+ valid_results['total_return'],
3563
+ valid_results['sharpe'],
3564
+ c=valid_results['max_drawdown'],
3565
+ s=valid_results['total_trades']*10,
3566
+ alpha=0.6,
3567
+ cmap='RdYlGn_r'
3568
+ )
3569
+ ax1.set_xlabel('Total Return (%)', fontsize=10)
3570
+ ax1.set_ylabel('Sharpe Ratio', fontsize=10)
3571
+ ax1.set_title('Sharpe vs Return (size=trades, color=drawdown)', fontsize=11, fontweight='bold')
3572
+ plt.colorbar(scatter, ax=ax1, label='Max Drawdown (%)')
3573
+ ax1.grid(True, alpha=0.3)
3574
+
3575
+ # 2. Parameter heatmap (if 2 parameters)
3576
+ if len(param_names) == 2:
3577
+ ax2 = plt.subplot(2, 3, 2)
3578
+ pivot_data = valid_results.pivot_table(
3579
+ values='sharpe',
3580
+ index=param_names[0],
3581
+ columns=param_names[1],
3582
+ aggfunc='mean'
3583
+ )
3584
+ sns.heatmap(pivot_data, annot=True, fmt='.2f', cmap='RdYlGn', ax=ax2)
3585
+ ax2.set_title(f'Sharpe Ratio Heatmap', fontsize=11, fontweight='bold')
3586
+ else:
3587
+ ax2 = plt.subplot(2, 3, 2)
3588
+ ax2.text(0.5, 0.5, 'Heatmap requires\nexactly 2 parameters',
3589
+ ha='center', va='center', fontsize=12)
3590
+ ax2.axis('off')
3591
+
3592
+ # 3. Win Rate vs Profit Factor
3593
+ ax3 = plt.subplot(2, 3, 3)
3594
+ scatter3 = ax3.scatter(
3595
+ valid_results['win_rate'],
3596
+ valid_results['profit_factor'],
3597
+ c=valid_results['sharpe'],
3598
+ s=100,
3599
+ alpha=0.6,
3600
+ cmap='viridis'
3601
+ )
3602
+ ax3.set_xlabel('Win Rate (%)', fontsize=10)
3603
+ ax3.set_ylabel('Profit Factor', fontsize=10)
3604
+ ax3.set_title('Win Rate vs Profit Factor (color=Sharpe)', fontsize=11, fontweight='bold')
3605
+ plt.colorbar(scatter3, ax=ax3, label='Sharpe Ratio')
3606
+ ax3.grid(True, alpha=0.3)
3607
+
3608
+ # 4. Distribution of Sharpe Ratios
3609
+ ax4 = plt.subplot(2, 3, 4)
3610
+ ax4.hist(valid_results['sharpe'], bins=20, color='steelblue', alpha=0.7, edgecolor='black')
3611
+ ax4.axvline(valid_results['sharpe'].mean(), color='red', linestyle='--', linewidth=2, label='Mean')
3612
+ ax4.axvline(valid_results['sharpe'].median(), color='green', linestyle='--', linewidth=2, label='Median')
3613
+ ax4.set_xlabel('Sharpe Ratio', fontsize=10)
3614
+ ax4.set_ylabel('Frequency', fontsize=10)
3615
+ ax4.set_title('Distribution of Sharpe Ratios', fontsize=11, fontweight='bold')
3616
+ ax4.legend()
3617
+ ax4.grid(True, alpha=0.3, axis='y')
3618
+
3619
+ # 5. Total Trades distribution
3620
+ ax5 = plt.subplot(2, 3, 5)
3621
+ ax5.hist(valid_results['total_trades'], bins=15, color='coral', alpha=0.7, edgecolor='black')
3622
+ ax5.set_xlabel('Total Trades', fontsize=10)
3623
+ ax5.set_ylabel('Frequency', fontsize=10)
3624
+ ax5.set_title('Distribution of Trade Counts', fontsize=11, fontweight='bold')
3625
+ ax5.grid(True, alpha=0.3, axis='y')
3626
+
3627
+ # 6. Top 10 combinations
3628
+ ax6 = plt.subplot(2, 3, 6)
3629
+ if 'combination_id' in valid_results.columns:
3630
+ top_10 = valid_results.nlargest(10, 'sharpe')[['combination_id', 'sharpe']].sort_values('sharpe')
3631
+ ax6.barh(range(len(top_10)), top_10['sharpe'], color='green', alpha=0.7)
3632
+ ax6.set_yticks(range(len(top_10)))
3633
+ ax6.set_yticklabels([f"#{int(x)}" for x in top_10['combination_id']])
3634
+ ax6.set_xlabel('Sharpe Ratio', fontsize=10)
3635
+ ax6.set_title('Top 10 Combinations by Sharpe', fontsize=11, fontweight='bold')
3636
+ else:
3637
+ # Fallback: use index as combination ID
3638
+ top_10 = valid_results.nlargest(10, 'sharpe')['sharpe'].sort_values()
3639
+ ax6.barh(range(len(top_10)), top_10.values, color='green', alpha=0.7)
3640
+ ax6.set_yticks(range(len(top_10)))
3641
+ ax6.set_yticklabels([f"#{i+1}" for i in range(len(top_10))])
3642
+ ax6.set_xlabel('Sharpe Ratio', fontsize=10)
3643
+ ax6.set_title('Top 10 Combinations by Sharpe', fontsize=11, fontweight='bold')
3644
+ ax6.grid(True, alpha=0.3, axis='x')
3645
+
3646
+ plt.tight_layout()
3647
+ plt.savefig(filename, dpi=150, bbox_inches='tight')
3648
+ print(f"\nVisualization saved: {filename}")
3649
+
3650
+ # Display plot if requested
3651
+ if show_plot:
3652
+ try:
3653
+ # First try to use IPython.display.Image (most reliable in Jupyter)
3654
+ from IPython.display import display, Image
3655
+ import os
3656
+ if os.path.exists(filename):
3657
+ display(Image(filename))
3658
+ else:
3659
+ # If file doesn't exist yet, try plt.show()
3660
+ plt.show()
3661
+ except (ImportError, NameError):
3662
+ # Not in Jupyter or IPython not available - try plt.show()
3663
+ try:
3664
+ plt.show()
3665
+ except:
3666
+ plt.close()
3667
+ except Exception:
3668
+ # Any other error - try plt.show() as fallback
3669
+ try:
3670
+ plt.show()
3671
+ except:
3672
+ plt.close()
3673
+ else:
3674
+ plt.close() # Close without displaying
3675
+
3676
+
3677
+ # ============================================================
3678
+ # CACHE CONFIGURATION (integrated from universal_backend_system.py)
3679
+ # ============================================================
3680
+ def get_cache_config(disk_enabled: bool = True, memory_enabled: bool = True,
3681
+ memory_percent: int = 10, max_age_days: int = 7,
3682
+ debug: bool = False, cache_dir: str = 'cache',
3683
+ compression: bool = True, auto_cleanup: bool = True) -> Dict[str, Any]:
3684
+ """
3685
+ Get cache configuration
3686
+
3687
+ Args:
3688
+ disk_enabled: Enable disk cache
3689
+ memory_enabled: Enable memory cache
3690
+ memory_percent: RAM percentage for cache (default 10%)
3691
+ max_age_days: Maximum cache age in days
3692
+ debug: Debug mode
3693
+ cache_dir: Cache directory
3694
+ compression: Use compression (Parquet + Snappy)
3695
+ auto_cleanup: Automatic cleanup of old cache
3696
+
3697
+ Returns:
3698
+ Dict with cache configuration
3699
+ """
3700
+ return {
3701
+ 'disk_enabled': disk_enabled,
3702
+ 'memory_enabled': memory_enabled,
3703
+ 'memory_percent': memory_percent,
3704
+ 'max_age_days': max_age_days,
3705
+ 'debug': debug,
3706
+ 'cache_dir': cache_dir,
3707
+ 'compression': compression,
3708
+ 'auto_cleanup': auto_cleanup
3709
+ }
3710
+
3711
+
3712
+ # ============================================================
3713
+ # UNIVERSAL CACHE MANAGER (integrated from universal_backend_system.py)
3714
+ # ============================================================
3715
+ class UniversalCacheManager:
3716
+ """Universal cache manager for any data types"""
3717
+
3718
+ # Mapping data types to cache directories
3719
+ DATA_TYPE_MAP = {
3720
+ 'stock_eod': 'STOCK_EOD',
3721
+ 'stock_intraday': 'STOCK_INTRADAY',
3722
+ 'options_eod': 'OPTIONS_EOD',
3723
+ 'options_intraday': 'OPTIONS_INTRADAY',
3724
+ # Backward compatibility (old naming):
3725
+ 'stock': 'STOCK_EOD',
3726
+ 'options': 'OPTIONS_EOD',
3727
+ 'intraday': 'OPTIONS_INTRADAY', # Default intraday = options
3728
+ }
3729
+
3730
+ def __init__(self, cache_config: Dict[str, Any]):
3731
+ self.cache_config = cache_config
3732
+ self.disk_enabled = cache_config.get('disk_enabled', True)
3733
+ self.memory_enabled = cache_config.get('memory_enabled', True)
3734
+ self.memory_percent = cache_config.get('memory_percent', 10)
3735
+ self.max_age_days = cache_config.get('max_age_days', 7)
3736
+ self.debug = cache_config.get('debug', False)
3737
+ self.cache_dir = cache_config.get('cache_dir', 'cache')
3738
+ self.compression = cache_config.get('compression', True)
3739
+ self.auto_cleanup = cache_config.get('auto_cleanup', True)
3740
+
3741
+ # Calculate cache size in RAM
3742
+ if self.memory_enabled:
3743
+ total_memory = psutil.virtual_memory().total
3744
+ self.max_memory_bytes = int(total_memory * self.memory_percent / 100)
3745
+ self.memory_cache = {}
3746
+ self.cache_order = []
3747
+ else:
3748
+ self.max_memory_bytes = 0
3749
+ self.memory_cache = {}
3750
+ self.cache_order = []
3751
+
3752
+ # Create cache directories
3753
+ if self.disk_enabled:
3754
+ os.makedirs(self.cache_dir, exist_ok=True)
3755
+
3756
+ def get(self, key: str, data_type: str = 'default') -> Optional[Any]:
3757
+ """Get data from cache"""
3758
+ try:
3759
+ # Check memory
3760
+ if self.memory_enabled and key in self.memory_cache:
3761
+ if self.debug:
3762
+ print(f"[CACHE] 🧠 Memory hit: {key}")
3763
+ return self.memory_cache[key]
3764
+
3765
+ # Check disk
3766
+ if self.disk_enabled:
3767
+ # Map data_type to proper directory structure using DATA_TYPE_MAP
3768
+ dir_name = self.DATA_TYPE_MAP.get(data_type, data_type.upper())
3769
+ data_dir = f"{self.cache_dir}/{dir_name}"
3770
+
3771
+ cache_file = os.path.join(data_dir, f"{key}.parquet")
3772
+ if os.path.exists(cache_file):
3773
+ if self._is_cache_valid(cache_file):
3774
+ data = self._load_from_disk(cache_file)
3775
+ if data is not None:
3776
+ # Save to memory
3777
+ if self.memory_enabled:
3778
+ self._save_to_memory(key, data)
3779
+ if self.debug:
3780
+ print(f"[CACHE] 💾 Disk hit: {key}")
3781
+ return data
3782
+
3783
+ # NEW: If exact match not found, search for overlapping cache
3784
+ # Only for date-range based cache types
3785
+ if data_type in ['stock_eod', 'options_eod', 'stock_intraday', 'options_intraday']:
3786
+ overlapping_data = self._find_overlapping_cache(key, data_type, data_dir)
3787
+ if overlapping_data is not None:
3788
+ # Save to memory for fast access
3789
+ if self.memory_enabled:
3790
+ self._save_to_memory(key, overlapping_data)
3791
+ return overlapping_data
3792
+
3793
+ if self.debug:
3794
+ print(f"[CACHE] ❌ Cache miss: {key}")
3795
+ return None
3796
+
3797
+ except Exception as e:
3798
+ if self.debug:
3799
+ print(f"[CACHE] ❌ Error getting {key}: {e}")
3800
+ return None
3801
+
3802
+ def set(self, key: str, data: Any, data_type: str = 'default') -> bool:
3803
+ """Save data to cache"""
3804
+ try:
3805
+ # Save to memory
3806
+ if self.memory_enabled:
3807
+ self._save_to_memory(key, data)
3808
+
3809
+ # Save to disk
3810
+ if self.disk_enabled:
3811
+ # Map data_type to proper directory structure using DATA_TYPE_MAP
3812
+ dir_name = self.DATA_TYPE_MAP.get(data_type, data_type.upper())
3813
+ data_dir = f"{self.cache_dir}/{dir_name}"
3814
+
3815
+ # Create directory if it doesn't exist
3816
+ os.makedirs(data_dir, exist_ok=True)
3817
+
3818
+ cache_file = os.path.join(data_dir, f"{key}.parquet")
3819
+ self._save_to_disk(cache_file, data)
3820
+
3821
+ if self.debug:
3822
+ # Count records for reporting
3823
+ record_count = len(data) if hasattr(data, '__len__') else '?'
3824
+ print(f"[CACHE] 💾 Saved: {key}")
3825
+ print(f"[CACHE] 💾 Saved to cache: {data_type.upper()} ({record_count} records)")
3826
+ return True
3827
+
3828
+ except Exception as e:
3829
+ if self.debug:
3830
+ print(f"[CACHE] ❌ Error saving {key}: {e}")
3831
+ return False
3832
+
3833
+ def _save_to_memory(self, key: str, data: Any):
3834
+ """Save to memory with LRU logic"""
3835
+ if key in self.memory_cache:
3836
+ self.cache_order.remove(key)
3837
+ else:
3838
+ # Check cache size
3839
+ while len(self.memory_cache) > 0 and self._get_memory_usage() > self.max_memory_bytes:
3840
+ oldest_key = self.cache_order.pop(0)
3841
+ del self.memory_cache[oldest_key]
3842
+
3843
+ self.memory_cache[key] = data
3844
+ self.cache_order.append(key)
3845
+
3846
+ def _save_to_disk(self, file_path: str, data: Any):
3847
+ """Save to disk"""
3848
+ try:
3849
+ # Ensure directory exists
3850
+ file_dir = os.path.dirname(file_path)
3851
+ if file_dir and not os.path.exists(file_dir):
3852
+ os.makedirs(file_dir, exist_ok=True)
3853
+
3854
+ if isinstance(data, pd.DataFrame):
3855
+ if self.compression:
3856
+ data.to_parquet(file_path, compression='snappy')
3857
+ else:
3858
+ data.to_parquet(file_path)
3859
+ elif isinstance(data, dict):
3860
+ # Convert dict to DataFrame
3861
+ df = pd.DataFrame([data])
3862
+ if self.compression:
3863
+ df.to_parquet(file_path, compression='snappy')
3864
+ else:
3865
+ df.to_parquet(file_path)
3866
+ else:
3867
+ # Try to convert to DataFrame
3868
+ df = pd.DataFrame(data)
3869
+ if self.compression:
3870
+ df.to_parquet(file_path, compression='snappy')
3871
+ else:
3872
+ df.to_parquet(file_path)
3873
+ except Exception as e:
3874
+ if self.debug:
3875
+ print(f"[CACHE] ❌ Error saving to disk: {e}")
3876
+
3877
+ def _load_from_disk(self, file_path: str) -> Optional[Any]:
3878
+ """Load from disk"""
3879
+ try:
3880
+ return pd.read_parquet(file_path)
3881
+ except Exception as e:
3882
+ if self.debug:
3883
+ print(f"[CACHE] ❌ Error loading from disk: {e}")
3884
+ return None
3885
+
3886
+ def _is_cache_valid(self, file_path: str) -> bool:
3887
+ """Check cache validity"""
3888
+ if not os.path.exists(file_path):
3889
+ return False
3890
+
3891
+ file_age = time.time() - os.path.getmtime(file_path)
3892
+ max_age_seconds = self.max_age_days * 24 * 3600
3893
+
3894
+ return file_age < max_age_seconds
3895
+
3896
+ def _get_memory_usage(self) -> int:
3897
+ """Get memory usage"""
3898
+ total_size = 0
3899
+ for key, value in self.memory_cache.items():
3900
+ try:
3901
+ if hasattr(value, 'memory_usage'):
3902
+ total_size += value.memory_usage(deep=True).sum()
3903
+ else:
3904
+ total_size += sys.getsizeof(value)
3905
+ except:
3906
+ total_size += sys.getsizeof(value)
3907
+ return total_size
3908
+
3909
+ def _find_overlapping_cache(self, key: str, data_type: str, data_dir: str) -> Optional[Any]:
3910
+ """
3911
+ Find cache files with overlapping date ranges
3912
+
3913
+ Args:
3914
+ key: Cache key (format: SYMBOL_START_END or SYMBOL_DATE)
3915
+ data_type: Data type (stock_eod, options_eod, etc.)
3916
+ data_dir: Cache directory
3917
+
3918
+ Returns:
3919
+ Filtered data if overlapping cache found, None otherwise
3920
+ """
3921
+ try:
3922
+ import re
3923
+ import glob
3924
+ from datetime import datetime
3925
+
3926
+ # Parse symbol and dates from key
3927
+ # Format: "SPY_2024-07-01_2025-10-29" or "SPY_2024-07-01"
3928
+ match = re.search(r'^([A-Z]+)_(\d{4}-\d{2}-\d{2})(?:_(\d{4}-\d{2}-\d{2}))?$', key)
3929
+ if not match:
3930
+ return None
3931
+
3932
+ symbol = match.group(1)
3933
+ start_date_str = match.group(2)
3934
+ end_date_str = match.group(3) if match.group(3) else start_date_str
3935
+
3936
+ # Parse dates
3937
+ start_date = datetime.strptime(start_date_str, '%Y-%m-%d').date()
3938
+ end_date = datetime.strptime(end_date_str, '%Y-%m-%d').date()
3939
+
3940
+ # Find all cache files for this symbol
3941
+ if not os.path.exists(data_dir):
3942
+ return None
3943
+
3944
+ pattern = os.path.join(data_dir, f"{symbol}_*.parquet")
3945
+ cache_files = glob.glob(pattern)
3946
+
3947
+ if not cache_files:
3948
+ return None
3949
+
3950
+ # Search for best overlapping cache
3951
+ best_match = None
3952
+ best_size = float('inf') # Prefer smallest file that covers range
3953
+
3954
+ for cache_file in cache_files:
3955
+ # Skip if cache is not valid
3956
+ if not self._is_cache_valid(cache_file):
3957
+ continue
3958
+
3959
+ # Parse dates from filename
3960
+ filename = os.path.basename(cache_file)
3961
+ file_match = re.search(r'(\d{4}-\d{2}-\d{2})(?:_(\d{4}-\d{2}-\d{2}))?', filename)
3962
+
3963
+ if not file_match:
3964
+ continue
3965
+
3966
+ cached_start_str = file_match.group(1)
3967
+ cached_end_str = file_match.group(2) if file_match.group(2) else cached_start_str
3968
+
3969
+ cached_start = datetime.strptime(cached_start_str, '%Y-%m-%d').date()
3970
+ cached_end = datetime.strptime(cached_end_str, '%Y-%m-%d').date()
3971
+
3972
+ # Check if cached range CONTAINS requested range
3973
+ if cached_start <= start_date and cached_end >= end_date:
3974
+ # Calculate file size (prefer smaller files)
3975
+ file_size = os.path.getsize(cache_file)
3976
+
3977
+ if file_size < best_size:
3978
+ best_match = cache_file
3979
+ best_size = file_size
3980
+
3981
+ if best_match:
3982
+ if self.debug:
3983
+ print(f"[CACHE] 🔍 Found overlapping cache: {os.path.basename(best_match)}")
3984
+ print(f"[CACHE] Requested: {start_date_str} → {end_date_str}")
3985
+ print(f"[CACHE] Filtering and loading...")
3986
+
3987
+ # Load and filter data
3988
+ df = pd.read_parquet(best_match)
3989
+
3990
+ # Ensure date column is in correct format
3991
+ if 'date' in df.columns:
3992
+ if df['date'].dtype == 'object':
3993
+ df['date'] = pd.to_datetime(df['date']).dt.date
3994
+ elif pd.api.types.is_datetime64_any_dtype(df['date']):
3995
+ df['date'] = df['date'].dt.date
3996
+
3997
+ # Filter by date range
3998
+ filtered = df[(df['date'] >= start_date) & (df['date'] <= end_date)].copy()
3999
+
4000
+ if self.debug:
4001
+ print(f"[CACHE] ✓ Overlapping cache hit: {len(filtered)} records (filtered from {len(df)})")
4002
+
4003
+ return filtered
4004
+ else:
4005
+ # No date column to filter - return as is
4006
+ if self.debug:
4007
+ print(f"[CACHE] ✓ Overlapping cache hit: {len(df)} records (no date filtering)")
4008
+ return df
4009
+
4010
+ return None
4011
+
4012
+ except Exception as e:
4013
+ if self.debug:
4014
+ print(f"[CACHE] ⚠️ Error searching for overlapping cache: {e}")
4015
+ return None
4016
+
4017
+
4018
+ # Export all
4019
+ __all__ = [
4020
+ 'BacktestResults', 'BacktestAnalyzer', 'ResultsReporter',
4021
+ 'ChartGenerator', 'ResultsExporter', 'run_backtest', 'run_backtest_with_stoploss',
4022
+ 'init_api', 'api_call', 'APIHelper', 'APIManager',
4023
+ 'ResourceMonitor', 'create_progress_bar', 'update_progress', 'format_time',
4024
+ 'StopLossManager', 'PositionManager', 'StopLossConfig',
4025
+ 'calculate_stoploss_metrics', 'print_stoploss_section', 'create_stoploss_charts',
4026
+ 'create_stoploss_comparison_chart',
4027
+ 'optimize_parameters', 'plot_optimization_results',
4028
+ 'create_optimization_folder',
4029
+ 'preload_options_data',
4030
+ 'preload_data_universal', # NEW: Universal preloader V2
4031
+ # New caching functions
4032
+ # Optimization preset functions
4033
+ 'apply_optimization_preset', 'list_optimization_presets',
4034
+ 'calculate_combinations_count', 'print_preset_info',
4035
+ 'get_cache_config', 'UniversalCacheManager'
4036
+ ]
4037
+
4038
+
4039
+ # ============================================================
4040
+ # OPTIMIZATION PRESET FUNCTIONS
4041
+ # ============================================================
4042
+
4043
+ def apply_optimization_preset(config, preset='default'):
4044
+ """
4045
+ Apply built-in optimization preset to config
4046
+
4047
+ Args:
4048
+ config: Configuration dictionary (will be updated)
4049
+ preset: Preset name ('default', 'quick_test', 'aggressive', 'conservative')
4050
+
4051
+ Returns:
4052
+ dict: Updated configuration
4053
+ """
4054
+ presets = {
4055
+ 'default': {
4056
+ 'param_grid': {
4057
+ 'z_score_entry': [0.8, 1.0, 1.2, 1.5],
4058
+ 'z_score_exit': [0.05, 0.1, 0.15],
4059
+ 'lookback_period': [45, 60, 90],
4060
+ 'dte_target': [30, 45, 60]
4061
+ },
4062
+ 'optimization_metric': 'sharpe',
4063
+ 'min_trades': 5,
4064
+ 'max_drawdown_limit': 0.50,
4065
+ 'parallel': False,
4066
+ # 'export_each_combo': True, # ← Убрано, будет использоваться из основного конфига
4067
+ 'results_folder_prefix': 'optimization',
4068
+ 'chart_filename': 'optimization_analysis.png',
4069
+ 'show_progress': True,
4070
+ 'verbose': True
4071
+ },
4072
+ 'quick_test': {
4073
+ 'param_grid': {
4074
+ 'z_score_entry': [1.0, 1.5],
4075
+ 'z_score_exit': [0.1],
4076
+ 'lookback_period': [60],
4077
+ 'dte_target': [45]
4078
+ },
4079
+ 'optimization_metric': 'sharpe',
4080
+ 'min_trades': 3,
4081
+ 'max_drawdown_limit': 0.40,
4082
+ 'parallel': False,
4083
+ # 'export_each_combo': False, # ← Убрано, будет использоваться из основного конфига
4084
+ 'results_folder_prefix': 'quick_test',
4085
+ 'chart_filename': 'quick_test_analysis.png',
4086
+ 'show_progress': True,
4087
+ 'verbose': False
4088
+ },
4089
+ 'aggressive': {
4090
+ 'param_grid': {
4091
+ 'z_score_entry': [1.5, 2.0, 2.5],
4092
+ 'z_score_exit': [0.05, 0.1],
4093
+ 'lookback_period': [30, 45, 60],
4094
+ 'dte_target': [30, 45]
4095
+ },
4096
+ 'optimization_metric': 'total_return',
4097
+ 'min_trades': 10,
4098
+ 'max_drawdown_limit': 0.60,
4099
+ 'parallel': False,
4100
+ # 'export_each_combo': True, # ← Убрано, будет использоваться из основного конфига
4101
+ 'results_folder_prefix': 'aggressive',
4102
+ 'chart_filename': 'aggressive_analysis.png',
4103
+ 'show_progress': True,
4104
+ 'verbose': True
4105
+ },
4106
+ 'conservative': {
4107
+ 'param_grid': {
4108
+ 'z_score_entry': [0.8, 1.0],
4109
+ 'z_score_exit': [0.1, 0.15, 0.2],
4110
+ 'lookback_period': [60, 90, 120],
4111
+ 'dte_target': [45, 60, 90]
4112
+ },
4113
+ 'optimization_metric': 'calmar',
4114
+ 'min_trades': 8,
4115
+ 'max_drawdown_limit': 0.25,
4116
+ 'parallel': False,
4117
+ # 'export_each_combo': True, # ← Убрано, будет использоваться из основного конфига
4118
+ 'results_folder_prefix': 'conservative',
4119
+ 'chart_filename': 'conservative_analysis.png',
4120
+ 'show_progress': True,
4121
+ 'verbose': True
4122
+ }
4123
+ }
4124
+
4125
+ if preset not in presets:
4126
+ available = list(presets.keys())
4127
+ raise ValueError(f"Preset '{preset}' not found. Available: {available}")
4128
+
4129
+ # Update only specific fields from preset
4130
+ preset_data = presets[preset]
4131
+
4132
+ # CRITICAL LOGIC:
4133
+ # - If preset == 'default' → use param_grid from config (if exists)
4134
+ # - If preset != 'default' → use param_grid from preset (override config)
4135
+ user_param_grid = config.get('param_grid')
4136
+
4137
+ fields_to_update = [
4138
+ 'param_grid', 'min_trades', 'max_drawdown_limit',
4139
+ 'optimization_metric', 'parallel', 'export_each_combo',
4140
+ 'results_folder_prefix', 'chart_filename',
4141
+ 'show_progress', 'verbose'
4142
+ ]
4143
+
4144
+ for field in fields_to_update:
4145
+ if field in preset_data:
4146
+ # Special handling for param_grid based on preset type
4147
+ if field == 'param_grid':
4148
+ if preset == 'default' and user_param_grid is not None:
4149
+ # 'default' preset → preserve user's param_grid
4150
+ continue
4151
+ else:
4152
+ # Non-default preset (quick_test, aggressive, etc.) → use preset's param_grid
4153
+ config[field] = preset_data[field]
4154
+ else:
4155
+ config[field] = preset_data[field]
4156
+
4157
+ print(f"✓ Applied preset: {preset}")
4158
+ if preset == 'default' and user_param_grid is not None:
4159
+ print(f" (Using user-defined param_grid from config)")
4160
+ elif preset != 'default':
4161
+ print(f" (Using param_grid from preset, ignoring config)")
4162
+
4163
+ return config
4164
+
4165
+
4166
+ def calculate_combinations_count(param_grid):
4167
+ """
4168
+ Calculate total number of parameter combinations
4169
+
4170
+ Args:
4171
+ param_grid: Dictionary with parameter lists
4172
+
4173
+ Returns:
4174
+ int: Total number of combinations
4175
+ """
4176
+ import math
4177
+ return math.prod(len(values) for values in param_grid.values())
4178
+
4179
+
4180
+ def print_preset_info(config):
4181
+ """
4182
+ Print preset information and combination count
4183
+
4184
+ Args:
4185
+ config: Configuration dictionary with preset applied
4186
+ """
4187
+ preset = config.get('preset', 'unknown')
4188
+ combinations = calculate_combinations_count(config['param_grid'])
4189
+
4190
+ print(f"\n{'='*60}")
4191
+ print(f"OPTIMIZATION PRESET: {preset.upper()}")
4192
+ print(f"{'='*60}")
4193
+ print(f"Total combinations: {combinations}")
4194
+ print(f"Optimization metric: {config.get('optimization_metric', 'sharpe')}")
4195
+ print(f"Min trades required: {config.get('min_trades', 10)}")
4196
+ print(f"Max drawdown limit: {config.get('max_drawdown_limit', 0.50)}")
4197
+ print(f"Parallel execution: {config.get('parallel', True)}")
4198
+ print(f"Export each combo: {config.get('export_each_combo', False)}")
4199
+ print(f"{'='*60}\n")
4200
+
4201
+
4202
+ def list_optimization_presets():
4203
+ """Show available built-in presets"""
4204
+ presets = {
4205
+ 'default': 'Standard configuration (4×3×3×3 = 108 combinations)',
4206
+ 'quick_test': 'Quick test (2×1×1×1 = 2 combinations)',
4207
+ 'aggressive': 'Aggressive strategy (3×2×3×2 = 36 combinations)',
4208
+ 'conservative': 'Conservative strategy (2×3×3×3 = 54 combinations)'
4209
+ }
4210
+
4211
+ print("\n📋 AVAILABLE OPTIMIZATION PRESETS:")
4212
+ print("-" * 60)
4213
+ for name, desc in presets.items():
4214
+ print(f" {name:<12} | {desc}")
4215
+ print("-" * 60)
4216
+
4217
+
4218
+
4219
+
4220
+
4221
+
4222
+