ivolatility-backtesting 1.6.0__py3-none-any.whl → 1.7.0__py3-none-any.whl

ivolatility_backtesting/ivolatility_backtesting.py

@@ -11,13 +11,15 @@ import pandas as pd
 import numpy as np
 import matplotlib.pyplot as plt
 import seaborn as sns
-from datetime import datetime
+from datetime import datetime, timedelta
 import ivolatility as ivol
 import os
 import time
 import psutil
 import warnings
 from itertools import product
+import sys
+from typing import Dict, List, Optional, Tuple, Union, Any
 warnings.filterwarnings('ignore', category=pd.errors.SettingWithCopyWarning)
 warnings.filterwarnings('ignore', message='.*SettingWithCopyWarning.*')
 warnings.filterwarnings('ignore', category=FutureWarning)
@@ -362,9 +364,90 @@ def init_api(api_key=None):
     APIManager.initialize(api_key)
 
 
-def api_call(endpoint, debug=False, **kwargs):
-    """Make API call with automatic response normalization"""
+def api_call(endpoint, cache_config=None, debug=False, **kwargs):
+    """
+    Make API call with automatic response normalization and caching
+    
+    Args:
+        endpoint: API endpoint path
+        cache_config: Cache configuration dict (optional, enables caching if provided)
+        debug: Debug mode flag
+        **kwargs: API parameters
+    
+    Returns:
+        Normalized API response or None
+    """
     try:
+        # Check if caching is enabled
+        use_cache = cache_config is not None and (
+            cache_config.get('disk_enabled', False) or 
+            cache_config.get('memory_enabled', False)
+        )
+        
+        cache_manager = None
+        cache_key = None
+        data_type = None
+        
+        if use_cache:
+            # Initialize cache manager
+            cache_manager = UniversalCacheManager(cache_config)
+            
+            # Create cache key from endpoint and params (human-readable)
+            # Determine data type based on endpoint (supports EOD + INTRADAY for both STOCK + OPTIONS)
+            is_intraday = 'intraday' in endpoint
+            is_options = 'options' in endpoint
+            is_stock = 'stock' in endpoint
+            
+            if is_intraday and is_options:
+                # Intraday options data: /equities/intraday/options-rawiv
+                data_type = 'options_intraday'
+                symbol = kwargs.get('symbol', 'UNKNOWN')
+                date = kwargs.get('date', 'UNKNOWN')
+                cache_key = f"{symbol}_{date}"
+            elif is_intraday and is_stock:
+                # Intraday stock data: /equities/intraday/stock-prices
+                data_type = 'stock_intraday'
+                symbol = kwargs.get('symbol', 'UNKNOWN')
+                date = kwargs.get('date', 'UNKNOWN')
+                cache_key = f"{symbol}_{date}"
+            elif is_options:
+                # EOD options data: /equities/eod/options-rawiv
+                data_type = 'options_eod'
+                symbol = kwargs.get('symbol', 'UNKNOWN')
+                from_date = kwargs.get('from_', kwargs.get('date', 'UNKNOWN'))
+                to_date = kwargs.get('to', from_date)
+                if from_date != to_date:
+                    cache_key = f"{symbol}_{from_date}_{to_date}"
+                else:
+                    cache_key = f"{symbol}_{from_date}"
+            elif is_stock:
+                # EOD stock data: /equities/eod/stock-prices
+                data_type = 'stock_eod'
+                symbol = kwargs.get('symbol', 'UNKNOWN')
+                from_date = kwargs.get('from_', kwargs.get('date', 'UNKNOWN'))
+                to_date = kwargs.get('to', from_date)
+                if from_date != to_date:
+                    cache_key = f"{symbol}_{from_date}_{to_date}"
+                else:
+                    cache_key = f"{symbol}_{from_date}"
+            else:
+                # Fallback for other endpoints
+                sorted_params = sorted([(k, v) for k, v in kwargs.items()])
+                param_hash = abs(hash(str(sorted_params)))
+                cache_key = f"{endpoint.replace('/', '_')}_{param_hash}"
+                data_type = 'default'
+            
+            # Try to get from cache
+            cached_data = cache_manager.get(cache_key, data_type)
+            if cached_data is not None:
+                if debug or cache_config.get('debug', False):
+                    print(f"[CACHE] ✓ Cache hit: {endpoint} ({len(cached_data) if hasattr(cached_data, '__len__') else '?'} records)")
+                # Return in same format as API (dict with 'data' key)
+                if isinstance(cached_data, pd.DataFrame):
+                    return {'data': cached_data.to_dict('records'), 'status': 'success'}
+                return cached_data
+        
+        # Cache miss or caching disabled - make API call
         if debug and APIManager._api_key:
             base_url = "https://restapi.ivolatility.com"
             url_params = {}
@@ -387,6 +470,23 @@ def api_call(endpoint, debug=False, **kwargs):
             print(f"[api_call] Endpoint: {endpoint}")
             print(f"[api_call] Params: {kwargs}")
         
+        # Save to cache if enabled and data is valid
+        if use_cache and normalized is not None and cache_manager is not None:
+            # Convert dict response to DataFrame for caching
+            if isinstance(normalized, dict) and 'data' in normalized:
+                try:
+                    cache_data = pd.DataFrame(normalized['data'])
+                    if len(cache_data) > 0:  # Only cache non-empty data
+                        cache_manager.set(cache_key, cache_data, data_type)
+                        if debug or cache_config.get('debug', False):
+                            print(f"[CACHE] 💾 Saved to cache: {endpoint} ({len(cache_data)} records)")
+                    else:
+                        if debug or cache_config.get('debug', False):
+                            print(f"[CACHE] ⚠️ Skipped caching empty data: {endpoint}")
+                except Exception as e:
+                    if debug or cache_config.get('debug', False):
+                        print(f"[CACHE] ❌ Error converting to cache format: {e}")
+        
         return normalized
     
     except Exception as e:
@@ -760,9 +860,17 @@ class PositionManager:
         self.config = config
         self.debug = debug
         
-        self.sl_enabled = config.get('stop_loss_enabled', False)
+        # Stop-loss enable logic:
+        # 1) Respect explicit flag if provided
+        # 2) Otherwise infer from stop_loss_config.enabled for convenience
+        explicit_flag = config.get('stop_loss_enabled')
+        sl_cfg = config.get('stop_loss_config', {})
+        inferred_flag = bool(sl_cfg.get('enabled', False))
+
+        self.sl_enabled = explicit_flag if explicit_flag is not None else inferred_flag
+
         if self.sl_enabled:
-            self.sl_config = config.get('stop_loss_config', {})
+            self.sl_config = sl_cfg
             self.sl_manager = StopLossManager()
         else:
             self.sl_config = None
@@ -1097,7 +1205,7 @@ class BacktestAnalyzer:
         self.metrics['best_trade'] = trades_df['pnl'].max()
         self.metrics['worst_trade'] = trades_df['pnl'].min()
         
-        if len(winning) > 0 and len(losing) > 0:
+        if len(winning) > 0 and len(losing) > 0 and self.metrics['avg_loss'] != 0:
             self.metrics['avg_win_loss_ratio'] = abs(self.metrics['avg_win'] / self.metrics['avg_loss'])
         else:
             self.metrics['avg_win_loss_ratio'] = 0
@@ -1600,7 +1708,7 @@ class ChartGenerator:
         if show_plots:
             plt.show()
         else:
-            plt.close()  # Закрываем без показа
+            plt.close()  # Close without displaying
         
         print(f"Chart saved: {filename}")
 
@@ -2218,13 +2326,13 @@ def create_stoploss_comparison_chart(results, filename='stoploss_comparison.png'
 # ============================================================
 def preload_options_data(config, progress_widgets=None):
     """
-    Предзагрузка опционных данных для оптимизации.
-    Загружает данные ОДИН РАЗ и возвращает кеш.
+    Preload options data for optimization.
+    Loads data ONCE and returns cache.
     
     Returns:
         tuple: (lean_df, options_cache)
-            - lean_df: DataFrame с историей IV lean
-            - options_cache: dict {date: DataFrame} с опционными данными
+            - lean_df: DataFrame with IV lean history
+            - options_cache: dict {date: DataFrame} with options data
     """
     if progress_widgets:
         progress_bar, status_label, monitor, start_time = progress_widgets
@@ -2259,14 +2367,14 @@ def preload_options_data(config, progress_widgets=None):
     
     # Store lean calculations
     lean_history = []
-    options_cache = {}  # {date: DataFrame with bid/ask data}
+    all_options_data = []  # List to collect all options DataFrames
     
     # Track time for ETA
     preload_start_time = time.time()
     
     try:
-        import ivolatility as ivol
-        getOptionsData = ivol.setMethod('/equities/eod/options-rawiv')
+        # Use api_call with caching instead of direct ivol API
+        cache_config = config.get('cache_config')
         
         # Process each chunk
         for chunk_idx, (chunk_start, chunk_end) in enumerate(date_chunks):
@@ -2280,16 +2388,24 @@ def preload_options_data(config, progress_widgets=None):
                     message=f"🔄 Loading chunk {chunk_idx+1}/{len(date_chunks)}"
                 )
             
-            raw_data = getOptionsData(
+            # Use api_call with caching (supports disk + memory cache)
+            raw_data = api_call(
+                '/equities/eod/options-rawiv',
+                cache_config,
                 symbol=symbol,
                 from_=chunk_start.strftime('%Y-%m-%d'),
-                to=chunk_end.strftime('%Y-%m-%d')
+                to=chunk_end.strftime('%Y-%m-%d'),
+                debug=cache_config.get('debug', False) if cache_config else False
             )
             
             if raw_data is None:
                 continue
             
-            df = pd.DataFrame(raw_data)
+            # api_call returns dict with 'data' key
+            if isinstance(raw_data, dict) and 'data' in raw_data:
+                df = pd.DataFrame(raw_data['data'])
+            else:
+                df = pd.DataFrame(raw_data)
             
             if df.empty:
                 continue
@@ -2334,10 +2450,8 @@ def preload_options_data(config, progress_widgets=None):
                 gc.collect()
                 continue
 
-            # Cache options data for position tracking
-            for date_val in df['date'].unique():
-                if date_val not in options_cache:
-                    options_cache[date_val] = df[df['date'] == date_val].copy()
+            # Collect all options data
+            all_options_data.append(df.copy())
             
             # Calculate lean for this chunk
             trading_dates = sorted(df['date'].unique())
@@ -2387,29 +2501,419 @@ def preload_options_data(config, progress_widgets=None):
         lean_df['stock_price'] = lean_df['stock_price'].astype('float32')
         lean_df['iv_lean'] = lean_df['iv_lean'].astype('float32')
         
-        del lean_history
+        # Combine all options data into single DataFrame
+        if all_options_data:
+            options_df = pd.concat(all_options_data, ignore_index=True)
+            # Ensure date column is properly formatted
+            options_df['date'] = pd.to_datetime(options_df['date']).dt.date
+            options_df['expiration'] = pd.to_datetime(options_df['expiration']).dt.date
+        else:
+            options_df = pd.DataFrame()
+        
+        del lean_history, all_options_data
         gc.collect()
         
         if progress_widgets:
-            status_label.value = f"<b style='color:#00cc00'>✓ Data preloaded: {len(lean_df)} days, {len(options_cache)} cached dates</b>"
+            status_label.value = f"<b style='color:#00cc00'>✓ Data preloaded: {len(lean_df)} days, {len(options_df)} options records</b>"
             progress_bar.value = 35
         
-        print(f"✓ Data preloaded: {len(lean_df)} days, {len(options_cache)} cached dates")
+        print(f"✓ Data preloaded: {len(lean_df)} days, {len(options_df)} options records")
         
-        return lean_df, options_cache
+        return lean_df, options_df
         
     except Exception as e:
         print(f"Error preloading data: {e}")
         return pd.DataFrame(), {}
 
 
+# ============================================================
+# UNIVERSAL DATA PRELOADER V2 (NEW!)
+# ============================================================
+def preload_data_universal(config, data_requests=None):
+    """
+    🚀 TRULY UNIVERSAL DATA PRELOADER - Works with ANY API endpoint!
+    
+    Supports:
+    - EOD data: options-rawiv, stock-prices, ivs-by-delta, ivx, etc.
+    - Intraday data: OPTIONS_INTRADAY, stock intraday, etc.
+    - Any custom endpoint with any parameters
+    - Automatic chunking for date ranges
+    - Manual single-date requests
+    
+    Args:
+        config: Strategy configuration (start_date, end_date, symbol)
+        data_requests: List of data requests to load. If None, tries auto-detection.
+                      
+                      Format:
+                      [
+                          {
+                              'name': 'options_data',          # Your name for this dataset
+                              'endpoint': '/equities/eod/options-rawiv',
+                              'params': {...},                 # Base params (symbol, etc.)
+                              'chunking': {                    # Optional: for date-range data
+                                  'enabled': True,
+                                  'date_param': 'from_',       # Param name for start date
+                                  'date_param_to': 'to',       # Param name for end date
+                                  'chunk_days': 90             # Chunk size in days
+                              },
+                              'post_process': lambda df: df,   # Optional: process DataFrame
+                          },
+                          {
+                              'name': 'ivx_data',
+                              'endpoint': '/equities/eod/ivx',
+                              'params': {
+                                  'symbol': config['symbol'],
+                                  'from_': config['start_date'],
+                                  'to': config['end_date']
+                              },
+                              'chunking': {'enabled': False}   # Single request
+                          },
+                          {
+                              'name': 'options_intraday',
+                              'endpoint': '/equities/intraday/options-rawiv',
+                              'params': {
+                                  'symbol': config['symbol']
+                              },
+                              'date_list': True,               # Load for each date separately
+                              'date_param': 'date'
+                          }
+                      ]
+    
+    Returns:
+        dict: Preloaded data with keys like:
+              {
+                  '_preloaded_options_data': DataFrame,
+                  '_preloaded_ivx_data': DataFrame,
+                  '_preloaded_options_intraday': DataFrame,
+                  '_stats': {...}
+              }
+    
+    Usage in strategy:
+        # Check for ANY preloaded data
+        if any(k.startswith('_preloaded_') for k in config):
+            options_df = config.get('_preloaded_options_data', pd.DataFrame()).copy()
+            ivx_df = config.get('_preloaded_ivx_data', pd.DataFrame()).copy()
+        else:
+            # Load fresh
+            ...
+    """
+    
+    print("\n" + "="*80)
+    print("🚀 UNIVERSAL PRELOADER V2 - Supports ANY endpoint (EOD/Intraday/IVX/etc.)")
+    print("="*80)
+    start_time = time.time()
+    
+    # Extract common config
+    start_date = datetime.strptime(config['start_date'], '%Y-%m-%d').date()
+    end_date = datetime.strptime(config['end_date'], '%Y-%m-%d').date()
+    symbol = config['symbol']
+    cache_config = config.get('cache_config', get_cache_config())
+    
+    # Auto-detection if not specified
+    if data_requests is None:
+        data_requests = _auto_detect_requests(config)
+        print(f"\n🔍 Auto-detected {len(data_requests)} data requests from config")
+    
+    preloaded = {}
+    total_rows = 0
+    
+    # Process each data request
+    for req_idx, request in enumerate(data_requests, 1):
+        req_name = request['name']
+        endpoint = request['endpoint']
+        base_params = request.get('params', {})
+        chunking = request.get('chunking', {'enabled': False})
+        post_process = request.get('post_process', None)
+        date_list = request.get('date_list', False)
+        
+        print(f"\n[{req_idx}/{len(data_requests)}] 📊 Loading: {req_name}")
+        print(f"           Endpoint: {endpoint}")
+        
+        all_data = []
+        
+        # ========================================================
+        # MODE 1: DATE LIST (one request per date, e.g., intraday)
+        # ========================================================
+        if date_list:
+            date_param = request.get('date_param', 'date')
+            trading_days = pd.bdate_range(start_date, end_date).date
+            
+            print(f"           Mode: Date list ({len(trading_days)} dates)")
+            
+            for day_idx, date in enumerate(trading_days):
+                params = base_params.copy()
+                params[date_param] = date.strftime('%Y-%m-%d')
+                
+                if day_idx % max(1, len(trading_days) // 10) == 0:
+                    print(f"           Progress: {day_idx}/{len(trading_days)} dates...")
+                
+                response = api_call(endpoint, cache_config, **params)
+                if response and 'data' in response:
+                    df = pd.DataFrame(response['data'])
+                    if len(df) > 0:
+                        all_data.append(df)
+        
+        # ========================================================
+        # MODE 2: CHUNKED LOADING (date ranges in chunks)
+        # ========================================================
+        elif chunking.get('enabled', False):
+            date_param_from = chunking.get('date_param', 'from_')
+            date_param_to = chunking.get('date_param_to', 'to')
+            chunk_days = chunking.get('chunk_days', 90)
+            chunk_size = timedelta(days=chunk_days)
+            
+            current = start_date
+            chunks = []
+            while current <= end_date:
+                chunk_end = min(current + chunk_size, end_date)
+                chunks.append((current, chunk_end))
+                current = chunk_end + timedelta(days=1)
+            
+            print(f"           Mode: Chunked ({len(chunks)} chunks of {chunk_days} days)")
+            
+            for chunk_idx, (chunk_start, chunk_end) in enumerate(chunks):
+                params = base_params.copy()
+                params[date_param_from] = chunk_start.strftime('%Y-%m-%d')
+                params[date_param_to] = chunk_end.strftime('%Y-%m-%d')
+                
+                if chunk_idx % max(1, len(chunks) // 5) == 0:
+                    print(f"           Progress: {chunk_idx+1}/{len(chunks)} chunks...")
+                
+                response = api_call(endpoint, cache_config, **params)
+                if response and 'data' in response:
+                    df = pd.DataFrame(response['data'])
+                    if len(df) > 0:
+                        all_data.append(df)
+        
+        # ========================================================
+        # MODE 3: SINGLE REQUEST (no chunking/date list)
+        # ========================================================
+        else:
+            print(f"           Mode: Single request")
+            
+            params = base_params.copy()
+            response = api_call(endpoint, cache_config, **params)
+            if response and 'data' in response:
+                df = pd.DataFrame(response['data'])
+                if len(df) > 0:
+                    all_data.append(df)
+        
+        # ========================================================
+        # COMBINE AND STORE
+        # ========================================================
+        if len(all_data) > 0:
+            combined_df = pd.concat(all_data, ignore_index=True)
+            
+            # Apply post-processing if provided
+            if post_process is not None:
+                try:
+                    combined_df = post_process(combined_df)
+                except Exception as e:
+                    print(f"           ⚠️  Post-processing failed: {e}")
+            
+            # Auto-process common date columns
+            combined_df = _auto_process_dates(combined_df)
+            
+            # Store with standardized key
+            key = f"_preloaded_{req_name}"
+            preloaded[key] = combined_df
+            total_rows += len(combined_df)
+            
+            print(f"           ✓ Loaded: {len(combined_df):,} rows → {key}")
+        else:
+            print(f"           ⚠️  No data returned")
+    
+    # ========================================================
+    # SUMMARY
+    # ========================================================
+    elapsed = time.time() - start_time
+    
+    # Collect detailed stats for each dataset
+    dataset_details = {}
+    for k in preloaded.keys():
+        if k.startswith('_preloaded_'):
+            dataset_name = k.replace('_preloaded_', '')
+            df = preloaded[k]
+            dataset_details[dataset_name] = {
+                'rows': len(df),
+                'endpoint': None
+            }
+    
+    # Map dataset names to endpoints from data_requests
+    if data_requests:
+        for req in data_requests:
+            req_name = req.get('name', 'unknown')
+            if req_name in dataset_details:
+                dataset_details[req_name]['endpoint'] = req.get('endpoint', 'unknown')
+    
+    preloaded['_stats'] = {
+        'load_time_seconds': int(elapsed),
+        'total_rows': total_rows,
+        'data_count': len([k for k in preloaded.keys() if k.startswith('_preloaded_')]),
+        'datasets': [k.replace('_preloaded_', '') for k in preloaded.keys() if k.startswith('_preloaded_')],
+        'dataset_details': dataset_details
+    }
+    
+    print(f"\n{'='*80}")
+    print(f"✅ PRELOAD COMPLETE:")
+    print(f"   • Time: {int(elapsed)}s")
+    print(f"   • Total rows: {total_rows:,}")
+    print(f"   • Datasets: {preloaded['_stats']['data_count']}")
+    for ds in preloaded['_stats']['datasets']:
+        print(f"     - {ds}")
+    print(f"   • Cached in RAM for 4-5x speedup! 🚀")
+    print(f"{'='*80}\n")
+    
+    return preloaded
+
+
+def _auto_detect_requests(config):
+    """Auto-detect what data to load based on config keys"""
+    requests = []
+    
+    # Always load options data for options strategies
+    requests.append({
+        'name': 'options',
+        'endpoint': '/equities/eod/options-rawiv',
+        'params': {
+            'symbol': config['symbol']
+        },
+        'chunking': {
+            'enabled': True,
+            'date_param': 'from_',
+            'date_param_to': 'to',
+            'chunk_days': 90
+        },
+        'post_process': lambda df: _process_options_df(df)
+    })
+    
+    # Load IV surface if strategy uses term structure
+    if any(k in config for k in ['short_tenor', 'long_tenor', 'delta_target']):
+        requests.append({
+            'name': 'ivs_surface',
+            'endpoint': '/equities/eod/ivs-by-delta',
+            'params': {
+                'symbol': config['symbol'],
+                'deltaFrom': config.get('delta_target', 0.5) - 0.05,
+                'deltaTo': config.get('delta_target', 0.5) + 0.05,
+                'periodFrom': config.get('short_tenor', 30) - 7,
+                'periodTo': config.get('long_tenor', 90) + 7
+            },
+            'chunking': {
+                'enabled': True,
+                'date_param': 'from_',
+                'date_param_to': 'to',
+                'chunk_days': 90
+            }
+        })
+    
+    # Load stock prices
+    requests.append({
+        'name': 'stock',
+        'endpoint': '/equities/eod/stock-prices',
+        'params': {
+            'symbol': config['symbol']
+        },
+        'chunking': {
+            'enabled': True,
+            'date_param': 'from_',
+            'date_param_to': 'to',
+            'chunk_days': 365  # Stock data is lightweight
+        }
+    })
+    
+    return requests
+
+
+def _process_options_df(df):
+    """Process options DataFrame: dates + DTE + OPTIMIZATIONS (5-10x faster!)"""
+    # Basic date processing
+    if 'date' in df.columns:
+        df['date'] = pd.to_datetime(df['date']).dt.date
+    if 'expiration' in df.columns:
+        df['expiration'] = pd.to_datetime(df['expiration']).dt.date
+    
+    if 'date' in df.columns and 'expiration' in df.columns:
+        df = df.copy()
+        df['dte'] = (pd.to_datetime(df['expiration']) - 
+                     pd.to_datetime(df['date'])).dt.days
+    
+    # ========================================================
+    # CRITICAL: SORT BY DATE FIRST! (Required for time-series)
+    # ========================================================
+    if 'date' in df.columns:
+        # Check if already sorted (skip if yes, fast!)
+        if not df['date'].is_monotonic_increasing:
+            df = df.sort_values('date')  # ✅ Sort only if needed
+    
+    # ========================================================
+    # AUTOMATIC OPTIMIZATIONS (applied by library)
+    # ========================================================
+    
+    # These optimizations are SAFE to apply automatically:
+    # - Categorical types for low-cardinality columns
+    # - Optimized numeric types (float32/int16 instead of float64/int64)
+    #
+    # NOTE: We do NOT set index on 'date' in library functions because:
+    # - It breaks existing code that uses .loc with non-date indices
+    # - Requires all strategies to handle Series vs scalar results
+    
+    # Convert Call/Put to categorical (60% less RAM, 2x faster filtering)
+    if 'Call/Put' in df.columns:
+        df['Call/Put'] = df['Call/Put'].astype('category')
+    
+    # Optimize data types (50% less RAM)
+    # float32 for prices (4 bytes instead of 8, enough precision)
+    float32_cols = ['strike', 'bid', 'ask', 'iv', 'price', 'mid', 'delta', 'gamma', 'vega', 'theta']
+    for col in float32_cols:
+        if col in df.columns:
+            df[col] = pd.to_numeric(df[col], errors='coerce').astype('float32')
+    
+    # int16 for DTE (2 bytes instead of 8, max 32767 days)
+    if 'dte' in df.columns:
+        df['dte'] = df['dte'].astype('int16')
+    
+    return df
+
+
+def _auto_process_dates(df):
+    """Auto-process common date columns + SORT BY DATE"""
+    date_columns = ['date', 'expiration', 'trade_date', 'time']
+    
+    for col in date_columns:
+        if col in df.columns:
+            try:
+                if col == 'time':
+                    # Keep time as string or datetime
+                    pass
+                else:
+                    df[col] = pd.to_datetime(df[col]).dt.date
+            except:
+                pass  # Already in correct format or not a date
+    
+    # ========================================================
+    # CRITICAL: SORT BY DATE! (Required for time-series)
+    # ========================================================
+    if 'date' in df.columns:
+        # Check if already sorted (O(1) check vs O(N log N) sort)
+        if not df['date'].is_monotonic_increasing:
+            df = df.sort_values('date')  # ✅ Sort only if needed
+    elif 'trade_date' in df.columns:
+        if not df['trade_date'].is_monotonic_increasing:
+            df = df.sort_values('trade_date')  # Alternative date column
+    
+    return df
+
+
 # ============================================================
 # NEW: OPTIMIZATION FRAMEWORK
 # ============================================================
 def optimize_parameters(base_config, param_grid, strategy_function,
                        optimization_metric='sharpe', min_trades=5,
                        max_drawdown_limit=None, parallel=False,
-                       export_each_combo=True # ← НОВЫЙ ПАРАМЕТР
+                       export_each_combo=True, # ← NEW PARAMETER
+                       optimization_config=None,  # ← NEW PARAMETER FOR PRESETS
+                       results_folder=None  # ← NEW: Use existing folder or create new
                        ):  
     """
     Optimize strategy parameters across multiple combinations
@@ -2419,7 +2923,7 @@ def optimize_parameters(base_config, param_grid, strategy_function,
         param_grid: Dict of parameters to optimize
             Example: {'z_score_entry': [1.0, 1.5, 2.0], 'z_score_exit': [0.1, 0.3, 0.5]}
         strategy_function: Strategy function to run
-        optimization_metric: Metric to optimize ('sharpe', 'total_return', 'profit_factor', 'calmar')
+        optimization_metric: Metric to optimize ('sharpe', 'total_return', 'total_pnl', 'profit_factor', 'calmar')
         min_trades: Minimum number of trades required
         max_drawdown_limit: Maximum acceptable drawdown (e.g., 0.10 for 10%)
         parallel: Use parallel processing (not implemented yet)
@@ -2429,10 +2933,35 @@ def optimize_parameters(base_config, param_grid, strategy_function,
         tuple: (results_df, best_params, results_folder)
     """
     
-    # ═══ ДОБАВИТЬ В НАЧАЛО ФУНКЦИИ ═══
-    # Create results folder
-    results_folder = create_optimization_folder()
-    print(f"📊 Results will be saved to: {results_folder}\n")
+    # Check if optimization_config has preset and apply it automatically
+    if optimization_config and isinstance(optimization_config, dict) and 'preset' in optimization_config:
+        preset = optimization_config['preset']
+        print(f"🔄 Auto-applying preset: {preset}")
+        apply_optimization_preset(optimization_config, preset)
+        print_preset_info(optimization_config)
+        
+        # Use preset parameters for grid and validation criteria
+        param_grid = optimization_config['param_grid']
+        min_trades = optimization_config['min_trades']
+        max_drawdown_limit = optimization_config['max_drawdown_limit']
+        
+        # Use optimization_config for optimization_metric if available
+        if 'optimization_metric' in optimization_config:
+            optimization_metric = optimization_config['optimization_metric']
+        
+        # Use optimization_config for execution settings if available
+        if 'parallel' in optimization_config:
+            parallel = optimization_config['parallel']
+        if 'export_each_combo' in optimization_config:
+            export_each_combo = optimization_config['export_each_combo']
+    
+    # ═══ ADD AT THE BEGINNING OF FUNCTION ═══
+    # Create results folder (or use provided one)
+    if results_folder is None:
+        results_folder = create_optimization_folder()
+        print(f"📊 Results will be saved to: {results_folder}\n")
+    else:
+        print(f"📊 Using existing results folder: {results_folder}\n")
     
     # Record start time
     optimization_start_time = datetime.now()
@@ -2496,7 +3025,7 @@ def optimize_parameters(base_config, param_grid, strategy_function,
     print("📥 PRELOADING OPTIONS DATA (loads ONCE, reused for all combinations)")
     print("="*80)
     
-    preloaded_lean_df, preloaded_options_cache = preload_options_data(
+    preloaded_lean_df, preloaded_options_df = preload_options_data(
         base_config, 
         progress_widgets=shared_progress['progress_widgets'] if shared_progress else None
     )
@@ -2534,7 +3063,7 @@ def optimize_parameters(base_config, param_grid, strategy_function,
         
         # ═══ ADD PRELOADED DATA TO CONFIG ═══
         test_config['_preloaded_lean_df'] = preloaded_lean_df
-        test_config['_preloaded_options_cache'] = preloaded_options_cache
+        test_config['_preloaded_options_cache'] = preloaded_options_df
         
         # Update progress
         if has_widgets:
@@ -2550,7 +3079,7 @@ def optimize_parameters(base_config, param_grid, strategy_function,
             if idx % max(1, total_combinations // 10) == 0:
                 print(f"[{idx}/{total_combinations}] {param_str}")
         
-        # ═══ ИЗМЕНИТЬ ВЫЗОВ run_backtest (строки ~2240-2248) ═══
+        # ═══ MODIFY run_backtest CALL (lines ~2240-2248) ═══
         try:
             # Create compact parameter string (e.g., Z1.0_E0.1_PT20)
             param_parts = []
@@ -2585,11 +3114,11 @@ def optimize_parameters(base_config, param_grid, strategy_function,
                 strategy_function,
                 test_config,
                 print_report=False,
-                create_charts=export_each_combo,  # ← СОЗДАЕМ ГРАФИКИ (сохраняются, но не показываются)
-                export_results=export_each_combo,  # ← ИЗМЕНЕНО
+                create_charts=export_each_combo,  # ← CREATE CHARTS (saved but not displayed)
+                export_results=export_each_combo,  # ← MODIFIED
                 progress_context=shared_progress,
-                chart_filename=os.path.join(combo_folder, 'equity_curve.png') if export_each_combo else None,  # ← ГРАФИКИ СОХРАНЯЮТСЯ
-                export_prefix=os.path.join(combo_folder, combo_prefix) if export_each_combo else None  # ← ДОБАВЛЕНО
+                chart_filename=os.path.join(combo_folder, 'equity_curve.png') if export_each_combo else None,  # ← CHARTS SAVED
+                export_prefix=os.path.join(combo_folder, combo_prefix) if export_each_combo else None  # ← ADDED
             )
             
             # Check validity
@@ -2670,6 +3199,8 @@ def optimize_parameters(base_config, param_grid, strategy_function,
                         valid_so_far.sort(key=lambda x: x['sharpe'], reverse=True)
                     elif optimization_metric == 'total_return':
                         valid_so_far.sort(key=lambda x: x['total_return'], reverse=True)
+                    elif optimization_metric == 'total_pnl':
+                        valid_so_far.sort(key=lambda x: x['total_pnl'], reverse=True)
                     elif optimization_metric == 'profit_factor':
                         valid_so_far.sort(key=lambda x: x['profit_factor'], reverse=True)
                     elif optimization_metric == 'calmar':
@@ -2693,7 +3224,10 @@ def optimize_parameters(base_config, param_grid, strategy_function,
         except Exception as e:
             print(f"\n[{idx}/{total_combinations}] {param_str}")
             print("-" * 80)
-            print(f"  ✗ ERROR: {str(e)[:100]}")
+            print(f"  ✗ ERROR: {str(e)}")
+            import traceback
+            print("  Full traceback:")
+            traceback.print_exc()
             
             result = {
                 'combination_id': idx,
@@ -2721,7 +3255,7 @@ def optimize_parameters(base_config, param_grid, strategy_function,
     for col in numeric_columns:
         results_df[col] = results_df[col].round(5)
 
-    # ═══ ДОБАВИТЬ СОХРАНЕНИЕ SUMMARY В ПАПКУ ═══
+    # ═══ ADD SUMMARY SAVE TO FOLDER ═══
     summary_path = os.path.join(results_folder, 'optimization_summary.csv')
     results_df.to_csv(summary_path, index=False)
     print(f"\n✓ Summary saved: {summary_path}")
@@ -2741,6 +3275,8 @@ def optimize_parameters(base_config, param_grid, strategy_function,
         best_idx = valid_results['sharpe'].idxmax()
     elif optimization_metric == 'total_return':
         best_idx = valid_results['total_return'].idxmax()
+    elif optimization_metric == 'total_pnl':
+        best_idx = valid_results['total_pnl'].idxmax()
     elif optimization_metric == 'profit_factor':
         best_idx = valid_results['profit_factor'].idxmax()
     elif optimization_metric == 'calmar':
@@ -2753,6 +3289,12 @@ def optimize_parameters(base_config, param_grid, strategy_function,
     # Extract best parameters
     best_params = {name: best_result[name] for name in param_names}
     
+    # Add stop_loss_pct if it exists in config (it's handled separately in notebook)
+    if 'stop_loss_config' in base_config and base_config['stop_loss_config']:
+        stop_loss_value = base_config['stop_loss_config'].get('value')
+        if stop_loss_value is not None:
+            best_params['stop_loss_pct'] = stop_loss_value
+    
     # Calculate total time
     optimization_end_time = datetime.now()
     total_duration = optimization_end_time - optimization_start_time
@@ -2760,39 +3302,81 @@ def optimize_parameters(base_config, param_grid, strategy_function,
     duration_str = format_time(total_duration.total_seconds())
     
     # Print summary
-    print("="*80)
-    print(" "*20 + "OPTIMIZATION COMPLETE")
-    print("="*80)
-    print(f"\nTotal Combinations Tested: {total_combinations}")
-    print(f"Valid Combinations: {len(valid_results)}")
-    print(f"Invalid Combinations: {len(results_df) - len(valid_results)}")
-    print(f"🕐 Started: {start_time_str}")
-    print(f"🕐 Finished: {end_time_str}")
-    print(f"⏱️ Total Duration: {duration_str}")
-    
-    print(f"\n{'='*80}")
-    print(" "*20 + "BEST PARAMETERS")
-    print("="*80)
-    for param_name, param_value in best_params.items():
-        print(f"{param_name:25s}: {param_value}")
+    print("\n" + "="*120)
+    print(" "*31 + "🏆 OPTIMIZATION COMPLETE 🏆")
+    print(" "*31 + "=========================")
+    print(f"  • Started              : {start_time_str}")
+    print(f"  • Finished             : {end_time_str}")
+    print(f"  • Total Duration       : {duration_str} ({int(total_duration.total_seconds())} seconds)")
+    print(f"  • Average per run      : {total_duration.total_seconds() / total_combinations:.1f} seconds")
+    print(f"  • Total combinations   : {total_combinations}")
+    print(f"  • Valid combinations   : {len(valid_results)}")
+    print(f"  • Invalid combinations : {len(results_df) - len(valid_results)}")
+    
+    print(f"\n📈 OPTIMIZATION METRIC:")
+    print(f"  • Metric optimized     : {optimization_metric.upper()}")
+    
+    # Format best parameters in one line (with special formatting for stop_loss_pct)
+    param_parts = []
+    for name, value in best_params.items():
+        if name == 'stop_loss_pct':
+            param_parts.append(f"stop_loss={value*100:.0f}%")
+        else:
+            param_parts.append(f"{name}={value}")
+    param_str = ", ".join(param_parts)
+    print(f"  • Best parameters      : {param_str}")
+    
+    # Add intraday stop-loss info if enabled
+    intraday_stops = base_config.get('intraday_stops', {})
+    if intraday_stops.get('enabled', False):
+        intraday_pct = intraday_stops.get('stop_pct', 0.03) * 100
+        intraday_days = intraday_stops.get('min_days_before_intraday', 3)
+        print(f"  • Intraday stop-loss   : Enabled ({intraday_pct:.0f}% after {intraday_days} days)")
+    
+    print(f"\n🏆 BEST PERFORMANCE:")
+    print(f"  • Total Return         : {best_result['total_return']:>10.2f}%")
+    print(f"  • Sharpe Ratio         : {best_result['sharpe']:>10.2f}")
+    print(f"  • Max Drawdown         : {best_result['max_drawdown']:>10.2f}%")
+    print(f"  • Win Rate             : {best_result['win_rate']:>10.1f}%")
+    print(f"  • Profit Factor        : {best_result['profit_factor']:>10.2f}")
+    print(f"  • Total Trades         : {best_result['total_trades']:>10.0f}")
+    
+    print(f"\n🔌 API ENDPOINTS:")
+    # Extract real endpoints from preloaded data stats
+    endpoints_info = []
+    
+    if '_stats' in base_config and 'dataset_details' in base_config['_stats']:
+        dataset_details = base_config['_stats']['dataset_details']
+        for dataset_name, info in dataset_details.items():
+            endpoint = info.get('endpoint')
+            rows = info.get('rows', 0)
+            if endpoint:
+                endpoints_info.append((endpoint, rows))
+    
+    # Check if intraday stops are enabled
+    intraday_stops = base_config.get('intraday_stops', {})
+    if intraday_stops.get('enabled', False):
+        intraday_endpoint = "/equities/intraday/stock-prices"
+        if not any(ep[0] == intraday_endpoint for ep in endpoints_info):
+            endpoints_info.append((intraday_endpoint, "on-demand"))
+    
+    if endpoints_info:
+        for idx, (endpoint, rows) in enumerate(endpoints_info, 1):
+            if isinstance(rows, int):
+                print(f"    {idx}. {endpoint:<45} ({rows:>10,} rows)")
+            else:
+                print(f"    {idx}. {endpoint:<45} ({rows})")
+    else:
+        # Fallback to static list if no stats available
+        print(f"    1. /equities/eod/options-rawiv")
+        print(f"    2. /equities/eod/stock-prices")
+        if intraday_stops.get('enabled', False):
+            print(f"    3. /equities/intraday/stock-prices")
     
-    print(f"\n{'='*80}")
-    print(" "*20 + "BEST PERFORMANCE")
-    print("="*80)
-    print(f"Total Return:        {best_result['total_return']:>10.2f}%")
-    print(f"Sharpe Ratio:        {best_result['sharpe']:>10.2f}")
-    print(f"Sortino Ratio:       {best_result['sortino']:>10.2f}")
-    print(f"Calmar Ratio:        {best_result['calmar']:>10.2f}")
-    print(f"Max Drawdown:        {best_result['max_drawdown']:>10.2f}%")
-    print(f"Win Rate:            {best_result['win_rate']:>10.1f}%")
-    print(f"Profit Factor:       {best_result['profit_factor']:>10.2f}")
-    print(f"Total Trades:        {best_result['total_trades']:>10.0f}")
-    print(f"Avg Win:             ${best_result['avg_win']:>10.2f}")
-    print(f"Avg Loss:            ${best_result['avg_loss']:>10.2f}")
-    print("="*80)
+    print("="*120)
     
     # ═══════════════════════════════════════════════════════════════════════════
-    # НОВОЕ! ПОЛНЫЙ БЭКТЕСТ ЛУЧШЕЙ КОМБИНАЦИИ СО ВСЕМИ ГРАФИКАМИ
+    # NEW! FULL BACKTEST OF BEST COMBINATION WITH ALL CHARTS
     # ═══════════════════════════════════════════════════════════════════════════
     print("\n" + "="*80)
     print(" "*15 + "RUNNING FULL BACKTEST FOR BEST COMBINATION")
@@ -2804,7 +3388,7 @@ def optimize_parameters(base_config, param_grid, strategy_function,
     best_config = base_config.copy()
     best_config.update(best_params)
     best_config['_preloaded_lean_df'] = preloaded_lean_df
-    best_config['_preloaded_options_cache'] = preloaded_options_cache
+    best_config['_preloaded_options_cache'] = preloaded_options_df
     
     # Create folder for best combination
     best_combo_folder = os.path.join(results_folder, 'best_combination')
@@ -2816,10 +3400,10 @@ def optimize_parameters(base_config, param_grid, strategy_function,
     best_analyzer = run_backtest(
         strategy_function,
         best_config,
-        print_report=True,  # ← ПОКАЗЫВАЕМ ПОЛНЫЙ ОТЧЕТ
-        create_charts=True,  # ← СОЗДАЕМ ВСЕ ГРАФИКИ
-        export_results=True,  # ← ЭКСПОРТИРУЕМ ВСЕ ФАЙЛЫ
-        progress_context=None,  # ← Обычный режим
+        print_report=True,  # ← SHOW FULL REPORT
+        create_charts=True,  # ← CREATE ALL CHARTS
+        export_results=True,  # ← EXPORT ALL FILES
+        progress_context=None,  # ← Normal mode
         chart_filename=os.path.join(best_combo_folder, 'equity_curve.png'),
         export_prefix=os.path.join(best_combo_folder, 'best')
     )
@@ -2837,7 +3421,7 @@ def optimize_parameters(base_config, param_grid, strategy_function,
     print(f"✓ Best combination results saved to: {best_combo_folder}/")
     
     # ═══════════════════════════════════════════════════════════════════════════
-    # ОТОБРАЖЕНИЕ ГРАФИКОВ ЛУЧШЕЙ КОМБИНАЦИИ В NOTEBOOK
+    # DISPLAY CHARTS FOR BEST COMBINATION IN NOTEBOOK
     # ═══════════════════════════════════════════════════════════════════════════
     try:
         # Charts are displayed in the notebook, not here
@@ -2847,12 +3431,34 @@ def optimize_parameters(base_config, param_grid, strategy_function,
     except Exception as e:
         print(f"\n⚠ Could not display charts (saved to {best_combo_folder}/): {e}")
     
+    # ═══════════════════════════════════════════════════════════════════════════
+    # CREATE OPTIMIZATION COMPARISON CHARTS (save only, display in notebook manually)
+    # ═══════════════════════════════════════════════════════════════════════════
+    print("\n" + "="*80)
+    print(" "*15 + "CREATING OPTIMIZATION COMPARISON CHARTS")
+    print("="*80)
+    try:
+        optimization_chart_path = os.path.join(results_folder, 'optimization_results.png')
+        # Save chart but don't display (show_plot=False) - display will be done in notebook for combined results
+        plot_optimization_results(
+            results_df,
+            param_names,
+            filename=optimization_chart_path,
+            show_plot=False  # Don't display here - will be shown in notebook for combined results
+        )
+        print(f"✓ Optimization comparison charts saved to: {optimization_chart_path}")
+        print("   (Chart will be displayed in notebook for combined results)")
+    except Exception as e:
+        print(f"⚠ Could not create optimization charts: {e}")
+        import traceback
+        traceback.print_exc()
+    
     print("="*80 + "\n")
     
     return results_df, best_params, results_folder
 
 
-def plot_optimization_results(results_df, param_names, filename='optimization_results.png'):
+def plot_optimization_results(results_df, param_names, filename='optimization_results.png', show_plot=True):
     """
     Create visualization of optimization results
     
@@ -2860,10 +3466,16 @@ def plot_optimization_results(results_df, param_names, filename='optimization_re
         results_df: Results DataFrame from optimize_parameters()
         param_names: List of parameter names
         filename: Output filename
+        show_plot: If True, display plot in Jupyter notebook (default: True)
     """
     import matplotlib.pyplot as plt
     import seaborn as sns
     
+    # Handle missing is_valid column (for combined results from multiple optimizations)
+    if 'is_valid' not in results_df.columns:
+        results_df = results_df.copy()
+        results_df['is_valid'] = True
+    
     valid_results = results_df[results_df['is_valid'] == True].copy()
     
     if valid_results.empty:
@@ -2944,18 +3556,393 @@ def plot_optimization_results(results_df, param_names, filename='optimization_re
     
     # 6. Top 10 combinations
     ax6 = plt.subplot(2, 3, 6)
-    top_10 = valid_results.nlargest(10, 'sharpe')[['combination_id', 'sharpe']].sort_values('sharpe')
-    ax6.barh(range(len(top_10)), top_10['sharpe'], color='green', alpha=0.7)
-    ax6.set_yticks(range(len(top_10)))
-    ax6.set_yticklabels([f"#{int(x)}" for x in top_10['combination_id']])
-    ax6.set_xlabel('Sharpe Ratio', fontsize=10)
-    ax6.set_title('Top 10 Combinations by Sharpe', fontsize=11, fontweight='bold')
+    if 'combination_id' in valid_results.columns:
+        top_10 = valid_results.nlargest(10, 'sharpe')[['combination_id', 'sharpe']].sort_values('sharpe')
+        ax6.barh(range(len(top_10)), top_10['sharpe'], color='green', alpha=0.7)
+        ax6.set_yticks(range(len(top_10)))
+        ax6.set_yticklabels([f"#{int(x)}" for x in top_10['combination_id']])
+        ax6.set_xlabel('Sharpe Ratio', fontsize=10)
+        ax6.set_title('Top 10 Combinations by Sharpe', fontsize=11, fontweight='bold')
+    else:
+        # Fallback: use index as combination ID
+        top_10 = valid_results.nlargest(10, 'sharpe')['sharpe'].sort_values()
+        ax6.barh(range(len(top_10)), top_10.values, color='green', alpha=0.7)
+        ax6.set_yticks(range(len(top_10)))
+        ax6.set_yticklabels([f"#{i+1}" for i in range(len(top_10))])
+        ax6.set_xlabel('Sharpe Ratio', fontsize=10)
+        ax6.set_title('Top 10 Combinations by Sharpe', fontsize=11, fontweight='bold')
     ax6.grid(True, alpha=0.3, axis='x')
     
     plt.tight_layout()
     plt.savefig(filename, dpi=150, bbox_inches='tight')
     print(f"\nVisualization saved: {filename}")
-    plt.close()  # Закрываем без показа, так как отображаем через display(Image)
+    
+    # Display plot if requested
+    if show_plot:
+        try:
+            # First try to use IPython.display.Image (most reliable in Jupyter)
+            from IPython.display import display, Image
+            import os
+            if os.path.exists(filename):
+                display(Image(filename))
+            else:
+                # If file doesn't exist yet, try plt.show()
+                plt.show()
+        except (ImportError, NameError):
+            # Not in Jupyter or IPython not available - try plt.show()
+            try:
+                plt.show()
+            except:
+                plt.close()
+        except Exception:
+            # Any other error - try plt.show() as fallback
+            try:
+                plt.show()
+            except:
+                plt.close()
+    else:
+        plt.close()  # Close without displaying
+
+
+# ============================================================
+# CACHE CONFIGURATION (integrated from universal_backend_system.py)
+# ============================================================
+def get_cache_config(disk_enabled: bool = True, memory_enabled: bool = True, 
+                    memory_percent: int = 10, max_age_days: int = 7, 
+                    debug: bool = False, cache_dir: str = 'cache',
+                    compression: bool = True, auto_cleanup: bool = True) -> Dict[str, Any]:
+    """
+    Get cache configuration
+    
+    Args:
+        disk_enabled: Enable disk cache
+        memory_enabled: Enable memory cache
+        memory_percent: RAM percentage for cache (default 10%)
+        max_age_days: Maximum cache age in days
+        debug: Debug mode
+        cache_dir: Cache directory
+        compression: Use compression (Parquet + Snappy)
+        auto_cleanup: Automatic cleanup of old cache
+    
+    Returns:
+        Dict with cache configuration
+    """
+    return {
+        'disk_enabled': disk_enabled,
+        'memory_enabled': memory_enabled,
+        'memory_percent': memory_percent,
+        'max_age_days': max_age_days,
+        'debug': debug,
+        'cache_dir': cache_dir,
+        'compression': compression,
+        'auto_cleanup': auto_cleanup
+    }
+
+
+# ============================================================
+# UNIVERSAL CACHE MANAGER (integrated from universal_backend_system.py)
+# ============================================================
+class UniversalCacheManager:
+    """Universal cache manager for any data types"""
+    
+    # Mapping data types to cache directories
+    DATA_TYPE_MAP = {
+        'stock_eod': 'STOCK_EOD',
+        'stock_intraday': 'STOCK_INTRADAY',
+        'options_eod': 'OPTIONS_EOD',
+        'options_intraday': 'OPTIONS_INTRADAY',
+        # Backward compatibility (old naming):
+        'stock': 'STOCK_EOD',
+        'options': 'OPTIONS_EOD',
+        'intraday': 'OPTIONS_INTRADAY',  # Default intraday = options
+    }
+    
+    def __init__(self, cache_config: Dict[str, Any]):
+        self.cache_config = cache_config
+        self.disk_enabled = cache_config.get('disk_enabled', True)
+        self.memory_enabled = cache_config.get('memory_enabled', True)
+        self.memory_percent = cache_config.get('memory_percent', 10)
+        self.max_age_days = cache_config.get('max_age_days', 7)
+        self.debug = cache_config.get('debug', False)
+        self.cache_dir = cache_config.get('cache_dir', 'cache')
+        self.compression = cache_config.get('compression', True)
+        self.auto_cleanup = cache_config.get('auto_cleanup', True)
+        
+        # Calculate cache size in RAM
+        if self.memory_enabled:
+            total_memory = psutil.virtual_memory().total
+            self.max_memory_bytes = int(total_memory * self.memory_percent / 100)
+            self.memory_cache = {}
+            self.cache_order = []
+        else:
+            self.max_memory_bytes = 0
+            self.memory_cache = {}
+            self.cache_order = []
+        
+        # Create cache directories
+        if self.disk_enabled:
+            os.makedirs(self.cache_dir, exist_ok=True)
+    
+    def get(self, key: str, data_type: str = 'default') -> Optional[Any]:
+        """Get data from cache"""
+        try:
+            # Check memory
+            if self.memory_enabled and key in self.memory_cache:
+                if self.debug:
+                    print(f"[CACHE] 🧠 Memory hit: {key}")
+                return self.memory_cache[key]
+            
+            # Check disk
+            if self.disk_enabled:
+                # Map data_type to proper directory structure using DATA_TYPE_MAP
+                dir_name = self.DATA_TYPE_MAP.get(data_type, data_type.upper())
+                data_dir = f"{self.cache_dir}/{dir_name}"
+                
+                cache_file = os.path.join(data_dir, f"{key}.parquet")
+                if os.path.exists(cache_file):
+                    if self._is_cache_valid(cache_file):
+                        data = self._load_from_disk(cache_file)
+                        if data is not None:
+                            # Save to memory
+                            if self.memory_enabled:
+                                self._save_to_memory(key, data)
+                            if self.debug:
+                                print(f"[CACHE] 💾 Disk hit: {key}")
+                            return data
+                
+                # NEW: If exact match not found, search for overlapping cache
+                # Only for date-range based cache types
+                if data_type in ['stock_eod', 'options_eod', 'stock_intraday', 'options_intraday']:
+                    overlapping_data = self._find_overlapping_cache(key, data_type, data_dir)
+                    if overlapping_data is not None:
+                        # Save to memory for fast access
+                        if self.memory_enabled:
+                            self._save_to_memory(key, overlapping_data)
+                        return overlapping_data
+            
+            if self.debug:
+                print(f"[CACHE] ❌ Cache miss: {key}")
+            return None
+            
+        except Exception as e:
+            if self.debug:
+                print(f"[CACHE] ❌ Error getting {key}: {e}")
+            return None
+    
+    def set(self, key: str, data: Any, data_type: str = 'default') -> bool:
+        """Save data to cache"""
+        try:
+            # Save to memory
+            if self.memory_enabled:
+                self._save_to_memory(key, data)
+            
+            # Save to disk
+            if self.disk_enabled:
+                # Map data_type to proper directory structure using DATA_TYPE_MAP
+                dir_name = self.DATA_TYPE_MAP.get(data_type, data_type.upper())
+                data_dir = f"{self.cache_dir}/{dir_name}"
+                
+                # Create directory if it doesn't exist
+                os.makedirs(data_dir, exist_ok=True)
+                
+                cache_file = os.path.join(data_dir, f"{key}.parquet")
+                self._save_to_disk(cache_file, data)
+            
+            if self.debug:
+                # Count records for reporting
+                record_count = len(data) if hasattr(data, '__len__') else '?'
+                print(f"[CACHE] 💾 Saved: {key}")
+                print(f"[CACHE] 💾 Saved to cache: {data_type.upper()} ({record_count} records)")
+            return True
+            
+        except Exception as e:
+            if self.debug:
+                print(f"[CACHE] ❌ Error saving {key}: {e}")
+            return False
+    
+    def _save_to_memory(self, key: str, data: Any):
+        """Save to memory with LRU logic"""
+        if key in self.memory_cache:
+            self.cache_order.remove(key)
+        else:
+            # Check cache size
+            while len(self.memory_cache) > 0 and self._get_memory_usage() > self.max_memory_bytes:
+                oldest_key = self.cache_order.pop(0)
+                del self.memory_cache[oldest_key]
+        
+        self.memory_cache[key] = data
+        self.cache_order.append(key)
+    
+    def _save_to_disk(self, file_path: str, data: Any):
+        """Save to disk"""
+        try:
+            # Ensure directory exists
+            file_dir = os.path.dirname(file_path)
+            if file_dir and not os.path.exists(file_dir):
+                os.makedirs(file_dir, exist_ok=True)
+            
+            if isinstance(data, pd.DataFrame):
+                if self.compression:
+                    data.to_parquet(file_path, compression='snappy')
+                else:
+                    data.to_parquet(file_path)
+            elif isinstance(data, dict):
+                # Convert dict to DataFrame
+                df = pd.DataFrame([data])
+                if self.compression:
+                    df.to_parquet(file_path, compression='snappy')
+                else:
+                    df.to_parquet(file_path)
+            else:
+                # Try to convert to DataFrame
+                df = pd.DataFrame(data)
+                if self.compression:
+                    df.to_parquet(file_path, compression='snappy')
+                else:
+                    df.to_parquet(file_path)
+        except Exception as e:
+            if self.debug:
+                print(f"[CACHE] ❌ Error saving to disk: {e}")
+    
+    def _load_from_disk(self, file_path: str) -> Optional[Any]:
+        """Load from disk"""
+        try:
+            return pd.read_parquet(file_path)
+        except Exception as e:
+            if self.debug:
+                print(f"[CACHE] ❌ Error loading from disk: {e}")
+            return None
+    
+    def _is_cache_valid(self, file_path: str) -> bool:
+        """Check cache validity"""
+        if not os.path.exists(file_path):
+            return False
+        
+        file_age = time.time() - os.path.getmtime(file_path)
+        max_age_seconds = self.max_age_days * 24 * 3600
+        
+        return file_age < max_age_seconds
+    
+    def _get_memory_usage(self) -> int:
+        """Get memory usage"""
+        total_size = 0
+        for key, value in self.memory_cache.items():
+            try:
+                if hasattr(value, 'memory_usage'):
+                    total_size += value.memory_usage(deep=True).sum()
+                else:
+                    total_size += sys.getsizeof(value)
+            except:
+                total_size += sys.getsizeof(value)
+        return total_size
+    
+    def _find_overlapping_cache(self, key: str, data_type: str, data_dir: str) -> Optional[Any]:
+        """
+        Find cache files with overlapping date ranges
+        
+        Args:
+            key: Cache key (format: SYMBOL_START_END or SYMBOL_DATE)
+            data_type: Data type (stock_eod, options_eod, etc.)
+            data_dir: Cache directory
+            
+        Returns:
+            Filtered data if overlapping cache found, None otherwise
+        """
+        try:
+            import re
+            import glob
+            from datetime import datetime
+            
+            # Parse symbol and dates from key
+            # Format: "SPY_2024-07-01_2025-10-29" or "SPY_2024-07-01"
+            match = re.search(r'^([A-Z]+)_(\d{4}-\d{2}-\d{2})(?:_(\d{4}-\d{2}-\d{2}))?$', key)
+            if not match:
+                return None
+            
+            symbol = match.group(1)
+            start_date_str = match.group(2)
+            end_date_str = match.group(3) if match.group(3) else start_date_str
+            
+            # Parse dates
+            start_date = datetime.strptime(start_date_str, '%Y-%m-%d').date()
+            end_date = datetime.strptime(end_date_str, '%Y-%m-%d').date()
+            
+            # Find all cache files for this symbol
+            if not os.path.exists(data_dir):
+                return None
+            
+            pattern = os.path.join(data_dir, f"{symbol}_*.parquet")
+            cache_files = glob.glob(pattern)
+            
+            if not cache_files:
+                return None
+            
+            # Search for best overlapping cache
+            best_match = None
+            best_size = float('inf')  # Prefer smallest file that covers range
+            
+            for cache_file in cache_files:
+                # Skip if cache is not valid
+                if not self._is_cache_valid(cache_file):
+                    continue
+                
+                # Parse dates from filename
+                filename = os.path.basename(cache_file)
+                file_match = re.search(r'(\d{4}-\d{2}-\d{2})(?:_(\d{4}-\d{2}-\d{2}))?', filename)
+                
+                if not file_match:
+                    continue
+                
+                cached_start_str = file_match.group(1)
+                cached_end_str = file_match.group(2) if file_match.group(2) else cached_start_str
+                
+                cached_start = datetime.strptime(cached_start_str, '%Y-%m-%d').date()
+                cached_end = datetime.strptime(cached_end_str, '%Y-%m-%d').date()
+                
+                # Check if cached range CONTAINS requested range
+                if cached_start <= start_date and cached_end >= end_date:
+                    # Calculate file size (prefer smaller files)
+                    file_size = os.path.getsize(cache_file)
+                    
+                    if file_size < best_size:
+                        best_match = cache_file
+                        best_size = file_size
+            
+            if best_match:
+                if self.debug:
+                    print(f"[CACHE] 🔍 Found overlapping cache: {os.path.basename(best_match)}")
+                    print(f"[CACHE]    Requested: {start_date_str} → {end_date_str}")
+                    print(f"[CACHE]    Filtering and loading...")
+                
+                # Load and filter data
+                df = pd.read_parquet(best_match)
+                
+                # Ensure date column is in correct format
+                if 'date' in df.columns:
+                    if df['date'].dtype == 'object':
+                        df['date'] = pd.to_datetime(df['date']).dt.date
+                    elif pd.api.types.is_datetime64_any_dtype(df['date']):
+                        df['date'] = df['date'].dt.date
+                    
+                    # Filter by date range
+                    filtered = df[(df['date'] >= start_date) & (df['date'] <= end_date)].copy()
+                    
+                    if self.debug:
+                        print(f"[CACHE] ✓ Overlapping cache hit: {len(filtered)} records (filtered from {len(df)})")
+                    
+                    return filtered
+                else:
+                    # No date column to filter - return as is
+                    if self.debug:
+                        print(f"[CACHE] ✓ Overlapping cache hit: {len(df)} records (no date filtering)")
+                    return df
+            
+            return None
+            
+        except Exception as e:
+            if self.debug:
+                print(f"[CACHE] ⚠️ Error searching for overlapping cache: {e}")
+            return None
 
 
 # Export all
@@ -2969,5 +3956,192 @@ __all__ = [
     'create_stoploss_comparison_chart',
     'optimize_parameters', 'plot_optimization_results',
     'create_optimization_folder',
-    'preload_options_data'  # ← ДОБАВЛЕНО
-]
+    'preload_options_data',
+    'preload_data_universal',  # NEW: Universal preloader V2
+    # New caching functions
+    # Optimization preset functions
+    'apply_optimization_preset', 'list_optimization_presets', 
+    'calculate_combinations_count', 'print_preset_info',
+    'get_cache_config', 'UniversalCacheManager'
+]
+
+
+# ============================================================
+# OPTIMIZATION PRESET FUNCTIONS
+# ============================================================
+
+def apply_optimization_preset(config, preset='default'):
+    """
+    Apply built-in optimization preset to config
+    
+    Args:
+        config: Configuration dictionary (will be updated)
+        preset: Preset name ('default', 'quick_test', 'aggressive', 'conservative')
+    
+    Returns:
+        dict: Updated configuration
+    """
+    presets = {
+        'default': {
+            'param_grid': {
+                'z_score_entry': [0.8, 1.0, 1.2, 1.5],
+                'z_score_exit': [0.05, 0.1, 0.15],
+                'lookback_period': [45, 60, 90],
+                'dte_target': [30, 45, 60]
+            },
+            'optimization_metric': 'sharpe',
+            'min_trades': 5,
+            'max_drawdown_limit': 0.50,
+            'parallel': False,
+            # 'export_each_combo': True,  # ← Убрано, будет использоваться из основного конфига
+            'results_folder_prefix': 'optimization',
+            'chart_filename': 'optimization_analysis.png',
+            'show_progress': True,
+            'verbose': True
+        },
+        'quick_test': {
+            'param_grid': {
+                'z_score_entry': [1.0, 1.5],
+                'z_score_exit': [0.1],
+                'lookback_period': [60],
+                'dte_target': [45]
+            },
+            'optimization_metric': 'sharpe',
+            'min_trades': 3,
+            'max_drawdown_limit': 0.40,
+            'parallel': False,
+            # 'export_each_combo': False,  # ← Убрано, будет использоваться из основного конфига
+            'results_folder_prefix': 'quick_test',
+            'chart_filename': 'quick_test_analysis.png',
+            'show_progress': True,
+            'verbose': False
+        },
+        'aggressive': {
+            'param_grid': {
+                'z_score_entry': [1.5, 2.0, 2.5],
+                'z_score_exit': [0.05, 0.1],
+                'lookback_period': [30, 45, 60],
+                'dte_target': [30, 45]
+            },
+            'optimization_metric': 'total_return',
+            'min_trades': 10,
+            'max_drawdown_limit': 0.60,
+            'parallel': False,
+            # 'export_each_combo': True,  # ← Убрано, будет использоваться из основного конфига
+            'results_folder_prefix': 'aggressive',
+            'chart_filename': 'aggressive_analysis.png',
+            'show_progress': True,
+            'verbose': True
+        },
+        'conservative': {
+            'param_grid': {
+                'z_score_entry': [0.8, 1.0],
+                'z_score_exit': [0.1, 0.15, 0.2],
+                'lookback_period': [60, 90, 120],
+                'dte_target': [45, 60, 90]
+            },
+            'optimization_metric': 'calmar',
+            'min_trades': 8,
+            'max_drawdown_limit': 0.25,
+            'parallel': False,
+            # 'export_each_combo': True,  # ← Убрано, будет использоваться из основного конфига
+            'results_folder_prefix': 'conservative',
+            'chart_filename': 'conservative_analysis.png',
+            'show_progress': True,
+            'verbose': True
+        }
+    }
+    
+    if preset not in presets:
+        available = list(presets.keys())
+        raise ValueError(f"Preset '{preset}' not found. Available: {available}")
+    
+    # Update only specific fields from preset
+    preset_data = presets[preset]
+    
+    # Save user-defined param_grid if it exists (user override has priority)
+    user_param_grid = config.get('param_grid')
+    
+    fields_to_update = [
+        'param_grid', 'min_trades', 'max_drawdown_limit',
+        'optimization_metric', 'parallel', 'export_each_combo',
+        'results_folder_prefix', 'chart_filename',
+        'show_progress', 'verbose'
+    ]
+    
+    for field in fields_to_update:
+        if field in preset_data:
+            # Special handling for param_grid: preserve user's param_grid if provided
+            if field == 'param_grid' and user_param_grid is not None:
+                # User defined param_grid - don't override with preset
+                continue
+            config[field] = preset_data[field]
+    
+    # Restore user's param_grid if it was saved (preserve user override)
+    if user_param_grid is not None:
+        config['param_grid'] = user_param_grid
+    
+    print(f"✓ Applied preset: {preset}")
+    if user_param_grid is not None:
+        print(f"  (Preserved user-defined param_grid)")
+    
+    return config
+
+
+def calculate_combinations_count(param_grid):
+    """
+    Calculate total number of parameter combinations
+    
+    Args:
+        param_grid: Dictionary with parameter lists
+        
+    Returns:
+        int: Total number of combinations
+    """
+    import math
+    return math.prod(len(values) for values in param_grid.values())
+
+
+def print_preset_info(config):
+    """
+    Print preset information and combination count
+    
+    Args:
+        config: Configuration dictionary with preset applied
+    """
+    preset = config.get('preset', 'unknown')
+    combinations = calculate_combinations_count(config['param_grid'])
+    
+    print(f"\n{'='*60}")
+    print(f"OPTIMIZATION PRESET: {preset.upper()}")
+    print(f"{'='*60}")
+    print(f"Total combinations: {combinations}")
+    print(f"Optimization metric: {config.get('optimization_metric', 'sharpe')}")
+    print(f"Min trades required: {config.get('min_trades', 10)}")
+    print(f"Max drawdown limit: {config.get('max_drawdown_limit', 0.50)}")
+    print(f"Parallel execution: {config.get('parallel', True)}")
+    print(f"Export each combo: {config.get('export_each_combo', False)}")
+    print(f"{'='*60}\n")
+
+
+def list_optimization_presets():
+    """Show available built-in presets"""
+    presets = {
+        'default': 'Standard configuration (4×3×3×3 = 108 combinations)',
+        'quick_test': 'Quick test (2×1×1×1 = 2 combinations)',
+        'aggressive': 'Aggressive strategy (3×2×3×2 = 36 combinations)',
+        'conservative': 'Conservative strategy (2×3×3×3 = 54 combinations)'
+    }
+    
+    print("\n📋 AVAILABLE OPTIMIZATION PRESETS:")
+    print("-" * 60)
+    for name, desc in presets.items():
+        print(f"  {name:<12} | {desc}")
+    print("-" * 60)
+
+
+
+
+
+
+