sovai 0.2.64__tar.gz → 0.2.65__tar.gz

{sovai-0.2.64 → sovai-0.2.65}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sovai
-Version: 0.2.64
+Version: 0.2.65
 Summary: python for asset management
 License: Proprietary License
 Author: Derek Snow

{sovai-0.2.64 → sovai-0.2.65}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "sovai"
-version = "0.2.64"
+version = "0.2.65"
 description = "python for asset management"
 authors = [
     {name = "Derek Snow", email = "d.snow@outlook.com"}

{sovai-0.2.64 → sovai-0.2.65}/sovai/extensions/pandas_extensions.py

@@ -679,7 +679,7 @@ class CustomDataFrame(pd.DataFrame):
 
         from sovai import data
 
-        df_closeadj = data("market/closeadj").rename({"closeadj": "price"}, axis=1)
+        df_closeadj = data("market/closeadj", full_history=True).rename({"closeadj": "price"}, axis=1)
 
         df_merge = self.merge(
             df_closeadj, left_index=True, right_index=True, how="left"
@@ -1005,7 +1005,7 @@ class CustomDataFrame(pd.DataFrame):
             print("2024-07-22  0.022  -0.061   0.002  ...")
             return None
 
-    def signal_evaluator(self):
+    def signal_evaluator(self, verbose=False):
         """
         Perform weight optimization on the input multi-index DataFrame.
 
@@ -1034,7 +1034,7 @@ class CustomDataFrame(pd.DataFrame):
 
         # If all checks pass, proceed with creating the SignalEvaluator object
         try:
-            optimized_evaluator = SignalEvaluator(self)
+            optimized_evaluator = SignalEvaluator(self, verbose)
             print("SignalEvaluator object created successfully with optimized weights.")
             return optimized_evaluator
         except Exception as e:

{sovai-0.2.64 → sovai-0.2.65}/sovai/extensions/signal_evaluation.py

@@ -1673,9 +1673,21 @@ from joblib import Parallel, delayed
 
 
 class SignalEvaluator:
-    def __init__(self, df_factor):
+    # --- Initialization with verbose parameter ---
+    def __init__(self, df_factor, verbose=False): # Added verbose parameter
+        self.verbose = verbose # Store verbose state
+        if self.verbose:
+            print("--- SignalEvaluator Initialization ---")
+            if isinstance(df_factor, (pd.DataFrame, pd.Series)):
+                print(f"Input df_factor shape: {df_factor.shape}")
+                # Use the new _log_df_preview for the initial df_factor print
+                # Note: Using head() here as requested in previous refinement
+                self._log_df_preview(df_factor, "Input df_factor head:")
+            else:
+                print(f"Input df_factor type: {type(df_factor)}")
 
         self.df_factor = df_factor
+        # The following attributes will be logged within their respective methods
         self.df_signal, self.df_prices = self._signal_frame()
         self.positions = self._create_portfolio_holdings()
         self.rebalance_mask = self._calc_rebalanced_mask()
@@ -1686,102 +1698,296 @@ class SignalEvaluator:
         self.portfolio_returns = self._portfolio_returns()
         self.cumulative_returns = self._cummulative_returns()
 
-        # self.statistics = self._calculate_statistics()
-
+        # self.statistics = self._calculate_statistics() # Keep commented if not used
+
+        if self.verbose:
+            print("--- SignalEvaluator Initialization Complete ---")
+
+    # --- Logging Helper Methods ---
+    def _log_df_preview(self, df_to_log, header_message="Preview:", max_cols_to_show=5, show_nan_summary=True):
+        """
+        Prints a preview of a DataFrame or Series, intelligently slicing wide DataFrames.
+        Uses .head() for the preview.
+        """
+        if df_to_log is None:
+            print(f"{header_message} (None)")
+            return
+
+        print(f"{header_message} (Shape: {df_to_log.shape})")
+        if isinstance(df_to_log, pd.DataFrame):
+            if not df_to_log.empty:
+                num_cols = df_to_log.shape[1]
+                cols_to_display = min(num_cols, max_cols_to_show)
+                # Display head of selected columns
+                print(df_to_log.iloc[:, :cols_to_display].head())
+                if show_nan_summary:
+                    print("NaNs count for displayed columns:")
+                    # Show NaN sum for the displayed columns
+                    print(df_to_log.iloc[:, :cols_to_display].isnull().sum())
+            else:
+                print("(DataFrame is empty)")
+        elif isinstance(df_to_log, pd.Series):
+            if not df_to_log.empty:
+                print(df_to_log.head()) # Show head for Series
+                if show_nan_summary:
+                    print(f"NaNs count: {df_to_log.isnull().sum()}")
+            else:
+                print("(Series is empty)")
+        else:
+            print(f"(Object is not a DataFrame or Series: {type(df_to_log)})")
+
+    def _log(self, message, df_to_preview=None, max_cols=5):
+        """Helper function for conditional logging. Uses _log_df_preview for DataFrames/Series."""
+        if self.verbose:
+            print(message)
+            if df_to_preview is not None:
+                # Pass an empty header as the main message is already printed
+                self._log_df_preview(df_to_preview, header_message="Preview:", max_cols_to_show=max_cols)
+
+
+    # --- Core Calculation Methods with Logging ---
     def _signal_frame(self):
+        self._log("Running _signal_frame...")
         factor = self.df_factor
+        # Ensure this column name matches your actual factor data
+        if factor.columns.empty:
+             self._log("Error: Input df_factor has no columns.")
+             return pd.DataFrame(), pd.DataFrame() # Return empty to avoid index error
         column_name = factor.columns[0]
-        df_factor = factor.add_price()
-
-        df_signal = df_factor[column_name].unstack(level="ticker").shift(1)
-
-        # Create wide DataFrame for prices
-        df_prices = df_factor["price"].unstack(level="ticker")
 
+        # --- Crucial Area 1: Getting df_with_price ---
+        # Verify that df_with_price correctly contains both the signal column
+        # AND a valid numeric 'price' column, indexed correctly by date and ticker.
+        if hasattr(factor, 'add_price') and callable(getattr(factor, 'add_price')):
+            # Make sure factor.add_price() actually returns a DataFrame with 'price'
+            df_with_price = factor.add_price()
+            self._log("Preview of df_with_price AFTER add_price():", df_with_price)
+        else:
+            # Assumes 'price' column already exists in factor if add_price not used
+            df_with_price = factor
+            if 'price' not in df_with_price.columns:
+                 self._log("Warning: 'price' column not found in df_factor and add_price() was not used.")
+                 # Handle error or return empty DFs
+                 return pd.DataFrame(), pd.DataFrame()
+            self._log("Preview of df_with_price (assuming price exists):", df_with_price)
+
+
+        # --- Crucial Area 2: Indexing Check ---
+        # Ensure 'ticker' and 'date' are index levels for unstacking
+        required_index_levels = ['date', 'ticker']
+        if not all(level in df_with_price.index.names for level in required_index_levels):
+             self._log(f"Warning: Index levels {required_index_levels} not found for unstacking. Current index: {df_with_price.index.names}", df_with_price.head())
+             # Add logic here to set the index correctly if needed, e.g.:
+             # if 'date' in df_with_price.columns and 'ticker' in df_with_price.columns:
+             #     self._log("Attempting to set index ['date', 'ticker']...")
+             #     try:
+             #         df_with_price = df_with_price.set_index(['date', 'ticker'])
+             #     except KeyError as e:
+             #         self._log(f"Failed to set index: {e}")
+             #         return pd.DataFrame(), pd.DataFrame() # Handle error
+             # else:
+             #     self._log("Cannot set index - 'date' or 'ticker' columns missing.")
+             #     return pd.DataFrame(), pd.DataFrame() # Handle error
+
+
+        # --- Crucial Area 3: The Unstack Operations ---
+        try:
+            # Check df_with_price[column_name] before unstacking
+            self._log(f"Preview of signal data '{column_name}' before unstack:", df_with_price[column_name])
+            df_signal = df_with_price[column_name].unstack(level="ticker").shift(1)
+
+            # **** FOCUS HERE: Check df_with_price['price'] ****
+            self._log("Preview of 'price' data before unstack:", df_with_price['price'])
+            df_prices = df_with_price["price"].unstack(level="ticker")
+
+            # **** Check if df_prices is all NaN ****
+            if not df_prices.empty and df_prices.isnull().all().all():
+                self._log("CRITICAL: df_prices is ALL NaN after unstacking! Check price data source and indexing.")
+
+        except Exception as e:
+            self._log(f"Error during unstacking in _signal_frame: {e}")
+            self._log("df_with_price index at time of error:", df_with_price.index.names)
+            self._log("df_with_price columns at time of error:", df_with_price.columns)
+            self._log("df_with_price head at time of error:", df_with_price.head())
+            # Return empty DataFrames or re-raise to prevent further issues
+            return pd.DataFrame(), pd.DataFrame()
+
+
+        self._log("Result of _signal_frame - df_signal:", df_signal)
+        self._log("Result of _signal_frame - df_prices:", df_prices)
         return df_signal, df_prices
 
     def _create_portfolio_holdings(self):
+        self._log("Running _create_portfolio_holdings...")
+        if self.df_signal.empty:
+            self._log("df_signal is empty in _create_portfolio_holdings. Returning empty DataFrame.")
+            return pd.DataFrame(index=self.df_signal.index, columns=self.df_signal.columns)
+
         signal_array = self.df_signal.values
         holdings_array = np.zeros_like(signal_array)
-        holdings_array[signal_array <= 10] = -1
-        holdings_array[signal_array >= 90] = 1
-        holdings_array[np.isnan(signal_array)] = np.nan
-        return pd.DataFrame(
+        try:
+            with np.errstate(invalid='ignore'): # Suppress warnings for NaN comparisons
+                 holdings_array[signal_array <= 10] = -1
+                 holdings_array[signal_array >= 90] = 1
+            holdings_array[np.isnan(signal_array)] = np.nan # Set NaNs explicitly
+        except TypeError as e:
+            self._log(f"TypeError during holdings_array assignment: {e}. signal_array sample: {signal_array[:2, :2] if signal_array.ndim > 1 else signal_array[:2]}")
+            # Fallback or re-raise
+            pass
+
+        df_holdings = pd.DataFrame(
             holdings_array, index=self.df_signal.index, columns=self.df_signal.columns
         )
+        self._log("Result of _create_portfolio_holdings - positions:", df_holdings)
+        return df_holdings
 
     def _calc_rebalanced_mask(self):
-        # Create the portfolio holdings DataFrame
-        df_holdings = self.positions
-
-        # Replace NaNs with 2 (our temporary placeholder)
-        df_temp = df_holdings.fillna(2)
+        self._log("Running _calc_rebalanced_mask...")
+        # Use the created positions index to create the mask
+        if self.positions.empty:
+            self._log("Positions DataFrame is empty in _calc_rebalanced_mask. Returning empty Series.")
+            return pd.Series(dtype=bool)
 
-        # Create a 4-weekly rebalancing mask
-        rebalance_mask = pd.Series(False, index=df_temp.index)
-        rebalance_mask.iloc[0] = True  # Always include the first row
-        rebalance_mask.iloc[4::4] = True  # Then every 4th row
+        # No need for fillna(2) here, just use the index
+        rebalance_mask = pd.Series(False, index=self.positions.index)
+        if not rebalance_mask.empty:
+            rebalance_mask.iloc[0] = True
+            rebalance_mask.iloc[4::4] = True # Every 4th row after the first
+        self._log("Result of _calc_rebalanced_mask - rebalance_mask (Series):", rebalance_mask)
         return rebalance_mask
 
     def _real_holdings(self):
-        # Create the portfolio holdings DataFrame
-        df_holdings = self.positions
+        self._log("Running _real_holdings...")
+        df_initial_holdings = self.positions
 
-        # Replace NaNs with 2 (our temporary placeholder)
-        df_temp = df_holdings.fillna(2)
+        if df_initial_holdings.empty or self.rebalance_mask.empty:
+            self._log("Initial positions or rebalance_mask is empty in _real_holdings. Returning empty DataFrame.")
+            return pd.DataFrame(index=df_initial_holdings.index, columns=df_initial_holdings.columns)
 
-        rebalance_mask = self.rebalance_mask
-
-        # Apply the rebalancing mask
-        df_balance = df_temp.where(rebalance_mask, np.nan)
-
-        # Forward fill
-        df_balance = df_balance.ffill()
-
-        # Replace 2 with NaN to restore original NaN structure
-        df_balance = df_balance.replace(2, np.nan)
+        # Apply the rebalancing mask and forward fill
+        # No need for the intermediate fillna(2) step
+        df_balance = df_initial_holdings.where(self.rebalance_mask).ffill()
 
+        # Replace 0 with NaN *after* forward filling
         df_balance = df_balance.replace(0, np.nan)
 
+        self._log("Result of _real_holdings - final holdings (df_balance):", df_balance)
         return df_balance
 
     def _portfolio_returns(self):
-        # Calculate portfolio returns (sum across all assets for each date)
+        self._log("Running _portfolio_returns...")
+        if self.position_returns.empty:
+            self._log("position_returns is empty. Returning empty Series for portfolio_returns.")
+            return pd.Series(dtype=float)
+
+        # Calculate mean returns, ignoring NaNs by default
         portfolio_returns = self.position_returns.mean(axis=1)
 
+        self._log("Result of _portfolio_returns - portfolio_returns (Series):", portfolio_returns)
         return portfolio_returns
 
     def _cummulative_returns(self):
-        # Calculate portfolio returns (sum across all assets for each date)
+        self._log("Running _cummulative_returns...")
+        if self.portfolio_returns.empty or self.portfolio_returns.isnull().all():
+            self._log("portfolio_returns is empty or all NaN. Returning empty/NaN Series for cumulative_returns.")
+            return pd.Series(dtype=float, index=self.portfolio_returns.index) # Keep index if possible
 
         # Calculate cumulative returns
         cumulative_returns = (1 + self.portfolio_returns).cumprod()
 
+        self._log("Result of _cummulative_returns - cumulative_returns (Series):", cumulative_returns)
         return cumulative_returns
 
     def _calculate_returns(self):
-        return self.df_prices.pct_change(fill_method=None).clip(upper=3)
+        self._log("Running _calculate_returns (price percentage changes)...")
+        if self.df_prices.empty:
+            self._log("df_prices is empty. Returning empty DataFrame for asset_price_returns.")
+            return pd.DataFrame(index=self.df_prices.index, columns=self.df_prices.columns)
+        if self.df_prices.isnull().all().all():
+             self._log("df_prices is ALL NaN. Returns calculation will result in NaNs.")
+
+        # fill_method=None is deprecated, default is None (no filling)
+        asset_price_returns = self.df_prices.pct_change()
+        asset_price_returns = asset_price_returns.clip(upper=3)
+
+        self._log("Result of _calculate_returns - asset_price_returns:", asset_price_returns)
+        return asset_price_returns
 
     def _calculate_position_returns(self):
-        return self.holdings * self.returns
+        self._log("Running _calculate_position_returns...")
+        if self.holdings.empty or self.returns.empty:
+            self._log("Holdings or returns are empty. Returning empty DataFrame for position_returns.")
+            idx = self.holdings.index if not self.holdings.empty else self.returns.index
+            cols = self.holdings.columns if not self.holdings.empty else self.returns.columns
+            return pd.DataFrame(index=idx, columns=cols)
+        if self.returns.isnull().all().all():
+             self._log("Returns DataFrame is ALL NaN. Position returns will be NaN.")
+
+        # Ensure alignment (pandas usually handles this, but good to be aware)
+        # Check if indices/columns match if issues arise
+        if not self.holdings.index.equals(self.returns.index):
+             self._log("Warning: Index mismatch between holdings and returns in _calculate_position_returns.")
+        if not self.holdings.columns.equals(self.returns.columns):
+             self._log("Warning: Column mismatch between holdings and returns in _calculate_position_returns.")
+
+
+        position_returns = self.holdings * self.returns
+        self._log("Result of _calculate_position_returns - position_returns:", position_returns)
+        return position_returns
 
     def _calculate_resampled_returns(self):
-        rebalance_mask = pd.Series(False, index=self.position_returns.index)
-        rebalance_mask.iloc[0] = True
-        rebalance_mask.iloc[4::4] = True
+        self._log("Running _calculate_resampled_returns...")
+        if self.position_returns.empty:
+            self._log("position_returns is empty in _calculate_resampled_returns. Returning empty DataFrame.")
+            return pd.DataFrame(columns=self.position_returns.columns)
+
+        resample_group_mask = pd.Series(False, index=self.position_returns.index)
+        if not resample_group_mask.empty:
+            resample_group_mask.iloc[0] = True
+            resample_group_mask.iloc[4::4] = True
 
         def custom_resampler(x):
+            # Sum ignores NaNs by default, so sum of all NaNs is 0.
             return x.sum()
 
-        group_numbers = (rebalance_mask.cumsum() - 1).ffill()
-        resampled_returns = self.position_returns.groupby(group_numbers).apply(
-            custom_resampler
-        )
-        resampled_returns.index = rebalance_mask[rebalance_mask == True].index
+        group_numbers = (resample_group_mask.cumsum() - 1).ffill()
+
+        if group_numbers.isnull().all():
+            self._log("No valid groups for resampling. Returning empty DataFrame.")
+            return pd.DataFrame(columns=self.position_returns.columns)
+
+        resampled_returns = self.position_returns.groupby(group_numbers).apply(custom_resampler)
+
+        # Align index with rebalance dates
+        true_mask_indices = resample_group_mask[resample_group_mask].index
+        if len(true_mask_indices) == len(resampled_returns):
+            resampled_returns.index = true_mask_indices
+        else:
+            # This case might happen if the last period doesn't align perfectly
+            # Or if groupby produces unexpected number of groups
+            self._log(f"Warning: Mismatch in length for resampled_returns index. Resampled length: {len(resampled_returns)}, True mask indices: {len(true_mask_indices)}. Attempting partial index assignment.")
+            common_len = min(len(true_mask_indices), len(resampled_returns))
+            resampled_returns = resampled_returns.iloc[:common_len]
+            resampled_returns.index = true_mask_indices[:common_len]
+
+
+        self._log("Result of _calculate_resampled_returns - resampled_returns:", resampled_returns)
         return resampled_returns
 
     def _calculate_statistics(self):
-        return draw_down_statistics(self.position_returns.mean(axis=1))
+        # This method was previously commented out or missing implementation details
+        self._log("Running _calculate_statistics...")
+        if self.portfolio_returns.empty or self.portfolio_returns.isnull().all():
+             self._log("Cannot calculate statistics: portfolio_returns is empty or all NaN.")
+             return None # Or return an empty structure / specific message
+
+        # Assuming draw_down_statistics exists and takes a Series
+        # stats = draw_down_statistics(self.portfolio_returns)
+        # self._log("Statistics calculation complete.") # Add preview if needed
+        # return stats
+        self._log("Placeholder for statistics calculation.")
+        return "Statistics Placeholder"
+
 
     @property
     def performance_plot(self):