fa-quant 1.2.1__py3-none-any.whl

financial_agent/trading/preprocessing/preprocessors.py

@@ -0,0 +1,162 @@
+import numpy as np
+import pandas as pd
+from stockstats import StockDataFrame as Sdf
+from ..config import config
+
+
+class FeatureEngineer:
+    """Provides methods for preprocessing the stock price data
+
+    Attributes
+    ----------
+        use_technical_indicator : boolean
+            we technical indicator or not
+        tech_indicator_list : list
+            a list of technical indicator names (modified from config.py)
+        use_turbulence : boolean
+            use turbulence index or not
+        user_defined_feature:boolean
+            user user defined features or not
+
+    Methods
+    -------
+    preprocess_data()
+        main method to do the feature engineering
+
+    """
+
+    def __init__(
+        self,
+        use_technical_indicator=True,
+        tech_indicator_list=config.TECHNICAL_INDICATORS_LIST,
+        use_turbulence=False,
+        user_defined_feature=False,
+    ):
+        self.use_technical_indicator = use_technical_indicator
+        self.tech_indicator_list = tech_indicator_list
+        self.use_turbulence = use_turbulence
+        self.user_defined_feature = user_defined_feature
+
+    def preprocess_data(self, df):
+        """main method to do the feature engineering
+        @:param config: source dataframe
+        @:return: a DataMatrices object
+        """
+
+        if self.use_technical_indicator == True:
+            # add technical indicators using stockstats
+            df = self.add_technical_indicator(df)
+            print("Successfully added technical indicators")
+
+        # add turbulence index for multiple stock
+        if self.use_turbulence == True:
+            df = self.add_turbulence(df)
+            print("Successfully added turbulence index")
+
+        # add user defined feature
+        if self.user_defined_feature == True:
+            df = self.add_user_defined_feature(df)
+            print("Successfully added user defined features")
+
+        # fill the missing values at the beginning and the end
+        df = df.fillna(method="bfill").fillna(method="ffill")
+        return df
+
+    def add_technical_indicator(self, data):
+        """
+        calculate technical indicators
+        use stockstats package to add technical inidactors
+        :param data: (df) pandas dataframe
+        :return: (df) pandas dataframe
+        """
+        df = data.copy()
+        df = df.sort_values(by=['tic','date'])
+        stock = Sdf.retype(df.copy())
+        unique_ticker = stock.tic.unique()
+
+        for indicator in self.tech_indicator_list:
+            indicator_df = pd.DataFrame()
+            for i in range(len(unique_ticker)):
+                try:
+                    temp_indicator = stock[stock.tic == unique_ticker[i]][indicator]
+                    temp_indicator = pd.DataFrame(temp_indicator)
+                    temp_indicator['tic'] = unique_ticker[i]
+                    temp_indicator['date'] = df[df.tic == unique_ticker[i]]['date'].to_list()
+                    indicator_df = pd.concat([indicator_df, temp_indicator])
+                except Exception as e:
+                    print(e)
+            indicator_df.reset_index(drop=True, inplace=True)
+            df = df.merge(indicator_df[['tic','date',indicator]],on=['tic','date'],how='left')
+        df = df.sort_values(by=['date','tic'])
+        return df
+
+    def add_user_defined_feature(self, data):
+        """
+         add user defined features
+        :param data: (df) pandas dataframe
+        :return: (df) pandas dataframe
+        """
+        df = data.copy()
+        df["daily_return"] = df.close.pct_change(1)
+        # df['return_lag_1']=df.close.pct_change(2)
+        # df['return_lag_2']=df.close.pct_change(3)
+        # df['return_lag_3']=df.close.pct_change(4)
+        # df['return_lag_4']=df.close.pct_change(5)
+        return df
+
+    def add_turbulence(self, data):
+        """
+        add turbulence index from a precalcualted dataframe
+        :param data: (df) pandas dataframe
+        :return: (df) pandas dataframe
+        """
+        df = data.copy()
+        turbulence_index = self.calculate_turbulence(df)
+        df = df.merge(turbulence_index, on="date")
+        df = df.sort_values(["date", "tic"]).reset_index(drop=True)
+        return df
+
+    def calculate_turbulence(self, data):
+        """calculate turbulence index based on dow 30"""
+        # can add other market assets
+        df = data.copy()
+        df_price_pivot = df.pivot(index="date", columns="tic", values="close")
+        # use returns to calculate turbulence
+        df_price_pivot = df_price_pivot.pct_change()
+
+        unique_date = df.date.unique()
+        # start after a year
+        start = 252
+        turbulence_index = [0] * start
+        # turbulence_index = [0]
+        count = 0
+        for i in range(start, len(unique_date)):
+            current_price = df_price_pivot[df_price_pivot.index == unique_date[i]]
+            # use one year rolling window to calcualte covariance
+            hist_price = df_price_pivot[
+                (df_price_pivot.index < unique_date[i])
+                & (df_price_pivot.index >= unique_date[i - 252])
+            ]
+            # Drop tickers which has number missing values more than the "oldest" ticker
+            filtered_hist_price = hist_price.iloc[hist_price.isna().sum().min():].dropna(axis=1)
+
+            cov_temp = filtered_hist_price.cov()
+            current_temp = current_price[[x for x in filtered_hist_price]] - np.mean(filtered_hist_price, axis=0)
+            temp = current_temp.values.dot(np.linalg.pinv(cov_temp)).dot(
+                current_temp.values.T
+            )
+            if temp > 0:
+                count += 1
+                if count > 2:
+                    turbulence_temp = temp[0][0]
+                else:
+                    # avoid large outlier because of the calculation just begins
+                    turbulence_temp = 0
+            else:
+                turbulence_temp = 0
+            turbulence_index.append(turbulence_temp)
+
+        turbulence_index = pd.DataFrame(
+            {"date": df_price_pivot.index, "turbulence": turbulence_index}
+        )
+        return turbulence_index

financial_agent/trading/utils/gpu_config.py

@@ -0,0 +1,213 @@
+"""
+GPU Configuration Module for Financial Agent
+============================================
+
+This module provides centralized GPU configuration for all TensorFlow operations
+in the financial agent project. It handles device detection, memory management,
+and provides utility functions for GPU-accelerated training.
+
+Usage:
+    from gpu_config import setup_gpu, get_device_string, pick_device, DEVICE
+    
+    # Setup GPU at the beginning of your script
+    setup_gpu()
+    
+    # Or pick a device manually
+    device = pick_device(prefer_gpu=True)  # Returns '/device:GPU:0' or '/CPU:0'
+    
+    # Use DEVICE for all TensorFlow operations
+    with tf.device(DEVICE):
+        model = build_model()
+"""
+
+import os
+import tensorflow as tf
+
+# Global device variable
+DEVICE = None
+
+def pick_device(prefer_gpu=True) -> str:
+    """
+    Pick the optimal device for TensorFlow operations.
+    
+    Args:
+        prefer_gpu (bool): Whether to prefer GPU over CPU if available
+        
+    Returns:
+        str: Device name (e.g., '/device:GPU:0' or '/CPU:0')
+    """
+    if prefer_gpu:
+        gpus = tf.config.list_logical_devices('GPU')
+        if gpus:
+            return gpus[0].name  # e.g., '/device:GPU:0'
+    return '/CPU:0'
+
+def setup_gpu():
+    """
+    Configure GPU settings for optimal performance across all notebooks and scripts.
+    
+    This function:
+    1. Disables XLA compilation to avoid platform detection issues
+    2. Sets up memory growth for GPU devices
+    3. Configures TensorFlow for better compatibility
+    4. Sets the global DEVICE variable
+    
+    Returns:
+        str: The device string to use for TensorFlow operations
+    """
+    global DEVICE
+    
+    # Disable XLA compilation to avoid platform detection issues
+    os.environ['TF_XLA_FLAGS'] = '--tf_xla_enable_xla_devices=false'
+    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
+    
+    # Configure TensorFlow for better compatibility
+    tf.config.run_functions_eagerly(False)
+    tf.config.optimizer.set_jit(False)  # Disable XLA JIT compilation
+    
+    # Set memory growth for GPU if available
+    gpus = tf.config.experimental.list_physical_devices('GPU')
+    if gpus:
+        try:
+            for gpu in gpus:
+                tf.config.experimental.set_memory_growth(gpu, True)
+            print(f"✓ GPU available: {len(gpus)} device(s)")
+            print("✓ Memory growth enabled for all GPU devices")
+        except RuntimeError as e:
+            print(f"GPU memory growth setting failed: {e}")
+    else:
+        print("✓ No GPU detected, using CPU")
+    
+    # Pick the optimal device using actual TensorFlow device names
+    DEVICE = pick_device(prefer_gpu=True)
+    print(f"✓ TensorFlow configured for device: {DEVICE}")
+    return DEVICE
+
+def get_device_string():
+    """
+    Get the current device string for TensorFlow operations.
+    
+    Returns:
+        str: Device string (e.g., '/device:GPU:0' or '/CPU:0')
+    """
+    if DEVICE is None:
+        return setup_gpu()
+    return DEVICE
+
+def get_device_info():
+    """
+    Get detailed information about available devices.
+    
+    Returns:
+        dict: Device information including GPU details
+    """
+    info = {
+        'tensorflow_version': tf.__version__,
+        'gpu_available': len(tf.config.experimental.list_physical_devices('GPU')) > 0,
+        'gpu_count': len(tf.config.experimental.list_physical_devices('GPU')),
+        'current_device': get_device_string(),
+        'gpu_devices': tf.config.experimental.list_physical_devices('GPU'),
+        'cpu_devices': tf.config.experimental.list_physical_devices('CPU')
+    }
+    return info
+
+def print_device_info():
+    """Print comprehensive device information."""
+    info = get_device_info()
+    print("=" * 50)
+    print("GPU CONFIGURATION INFO")
+    print("=" * 50)
+    print(f"TensorFlow Version: {info['tensorflow_version']}")
+    print(f"GPU Available: {info['gpu_available']}")
+    print(f"GPU Count: {info['gpu_count']}")
+    print(f"Current Device: {info['current_device']}")
+    
+    if info['gpu_devices']:
+        print("\nGPU Devices:")
+        for i, gpu in enumerate(info['gpu_devices']):
+            print(f"  {i}: {gpu}")
+    
+    print(f"\nCPU Devices: {len(info['cpu_devices'])}")
+    print("=" * 50)
+
+def configure_mixed_precision():
+    """
+    Configure mixed precision training for better GPU performance.
+    
+    Returns:
+        bool: True if mixed precision is enabled, False otherwise
+    """
+    if 'GPU' in get_device_string():
+        try:
+            policy = tf.keras.mixed_precision.Policy('mixed_float16')
+            tf.keras.mixed_precision.set_global_policy(policy)
+            print("✓ Mixed precision training enabled for GPU")
+            return True
+        except Exception as e:
+            print(f"Failed to enable mixed precision: {e}")
+            return False
+    else:
+        print("Mixed precision only available on GPU")
+        return False
+
+def get_optimal_batch_size(base_batch_size=32):
+    """
+    Get optimal batch size based on available device.
+    
+    Args:
+        base_batch_size (int): Base batch size for CPU
+        
+    Returns:
+        int: Optimized batch size for the current device
+    """
+    if 'GPU' in get_device_string():
+        # Increase batch size for GPU
+        return base_batch_size * 4
+    else:
+        return base_batch_size
+
+def create_device_aware_model(model_builder, *args, **kwargs):
+    """
+    Create a model with proper device placement.
+    
+    Args:
+        model_builder: Function that builds the model
+        *args: Arguments for model builder
+        **kwargs: Keyword arguments for model builder
+        
+    Returns:
+        tf.keras.Model: Model created on the appropriate device
+    """
+    with tf.device(get_device_string()):
+        model = model_builder(*args, **kwargs)
+    return model
+
+def train_on_device(model, x_train, y_train, **kwargs):
+    """
+    Train a model with proper device placement.
+    
+    Args:
+        model: Keras model to train
+        x_train: Training features
+        y_train: Training labels
+        **kwargs: Additional arguments for model.fit()
+        
+    Returns:
+        tf.keras.callbacks.History: Training history
+    """
+    # Convert to tensors with proper device placement
+    with tf.device(get_device_string()):
+        x_train_tensor = tf.convert_to_tensor(x_train, dtype=tf.float32)
+        y_train_tensor = tf.convert_to_tensor(y_train, dtype=tf.float32)
+    
+    # Train the model
+    history = model.fit(x_train_tensor, y_train_tensor, **kwargs)
+    return history
+
+# Initialize GPU configuration when module is imported
+if __name__ == "__main__":
+    setup_gpu()
+    print_device_info()
+else:
+    # Auto-setup when imported
+    setup_gpu()

financial_agent/trading/utils/main.py

@@ -0,0 +1,52 @@
+import json
+import logging
+import os
+import time
+from argparse import ArgumentParser
+import datetime
+
+from ..config import config
+
+
+def build_parser():
+    parser = ArgumentParser()
+    parser.add_argument(
+        "--mode",
+        dest="mode",
+        help="start mode, train, download_data" " backtest",
+        metavar="MODE",
+        default="train",
+    )
+    return parser
+
+
+def main():
+    parser = build_parser()
+    options = parser.parse_args()
+    if not os.path.exists("./" + config.DATA_SAVE_DIR):
+        os.makedirs("./" + config.DATA_SAVE_DIR)
+    if not os.path.exists("./" + config.TRAINED_MODEL_DIR):
+        os.makedirs("./" + config.TRAINED_MODEL_DIR)
+    if not os.path.exists("./" + config.TENSORBOARD_LOG_DIR):
+        os.makedirs("./" + config.TENSORBOARD_LOG_DIR)
+    if not os.path.exists("./" + config.RESULTS_DIR):
+        os.makedirs("./" + config.RESULTS_DIR)
+
+    if options.mode == "train":
+        from ..autotrain import training
+
+        training.train_one()
+
+    elif options.mode == "download_data":
+        from ..marketdata.yahoodownloader import YahooDownloader
+
+        df = YahooDownloader(start_date=config.START_DATE,
+                             end_date=config.END_DATE,
+                             ticker_list=config.DOW_30_TICKER).fetch_data()
+        now = datetime.datetime.now().strftime("%Y%m%d-%Hh%M")
+        df.to_csv("./" + config.DATA_SAVE_DIR + "/" + now + ".csv")
+
+        
+        
+if __name__ == "__main__":
+    main()

financial_agent/trading/utils/setup_gpu_for_notebooks.py

@@ -0,0 +1,130 @@
+"""
+GPU Setup Script for Jupyter Notebooks
+======================================
+
+This script sets up GPU configuration for all Jupyter notebooks in the core_trading directory.
+Run this script at the beginning of any notebook to ensure proper GPU configuration.
+
+Usage in Jupyter Notebook:
+    %run setup_gpu_for_notebooks.py
+    
+Or import and use:
+    from setup_gpu_for_notebooks import setup_gpu_for_notebook
+    setup_gpu_for_notebook()
+"""
+
+import os
+import sys
+import warnings
+
+# Add the current directory to Python path
+current_dir = os.path.dirname(os.path.abspath(__file__))
+if current_dir not in sys.path:
+    sys.path.append(current_dir)
+
+# Import GPU configuration
+from gpu_config import setup_gpu, print_device_info, DEVICE, get_device_string
+
+def setup_gpu_for_notebook():
+    """
+    Setup GPU configuration for Jupyter notebooks.
+    This function should be called at the beginning of any notebook.
+    """
+    print("=" * 60)
+    print("🚀 SETTING UP GPU FOR FINANCIAL AGENT NOTEBOOKS")
+    print("=" * 60)
+    
+    # Setup GPU configuration
+    device = setup_gpu()
+    
+    # Print device information
+    print_device_info()
+    
+    # Import TensorFlow and other ML libraries
+    try:
+        import tensorflow as tf
+        print(f"✅ TensorFlow {tf.__version__} loaded successfully")
+        
+        # Test GPU availability
+        if device == '/GPU:0':
+            print("✅ GPU acceleration is ENABLED")
+            print("✅ All TensorFlow operations will use GPU")
+        else:
+            print("⚠️  GPU not available, using CPU")
+            
+    except ImportError as e:
+        print(f"❌ Error importing TensorFlow: {e}")
+        return False
+    
+    # Import other required libraries
+    try:
+        import numpy as np
+        import pandas as pd
+        print(f"✅ NumPy {np.__version__} loaded successfully")
+        print(f"✅ Pandas {pd.__version__} loaded successfully")
+    except ImportError as e:
+        print(f"❌ Error importing required libraries: {e}")
+        return False
+    
+    print("=" * 60)
+    print("✅ GPU SETUP COMPLETE - READY FOR TRAINING!")
+    print("=" * 60)
+    
+    return True
+
+def get_notebook_device():
+    """
+    Get the current device for use in notebook cells.
+    
+    Returns:
+        str: Device string ('/GPU:0' or '/CPU:0')
+    """
+    return get_device_string()
+
+def print_training_info():
+    """
+    Print information about GPU-accelerated training.
+    """
+    print("\n" + "=" * 60)
+    print("📊 GPU-ACCELERATED TRAINING INFORMATION")
+    print("=" * 60)
+    
+    device = get_device_string()
+    
+    if device == '/GPU:0':
+        print("🚀 GPU ACCELERATION ENABLED")
+        print("   • All TensorFlow operations will use GPU")
+        print("   • Training will be significantly faster")
+        print("   • Memory growth is enabled for optimal performance")
+        print("   • XLA compilation is disabled to prevent errors")
+    else:
+        print("💻 CPU TRAINING MODE")
+        print("   • All operations will use CPU")
+        print("   • Training will be slower but more stable")
+        print("   • No GPU memory management needed")
+    
+    print("\n📝 USAGE IN YOUR NOTEBOOK:")
+    print("   • Use DEVICE variable for all tf.device() calls")
+    print("   • Models will automatically be created on the correct device")
+    print("   • Training functions handle device placement automatically")
+    
+    print("=" * 60)
+
+if __name__ == "__main__":
+    # Run setup when script is executed directly
+    setup_gpu_for_notebook()
+    print_training_info()
+else:
+    # Auto-setup when imported
+    setup_gpu_for_notebook()
+
+
+
+
+
+
+
+
+
+
+