sovai 0.2.63__tar.gz → 0.2.64__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sovai-0.2.63 → sovai-0.2.64}/PKG-INFO +1 -1
- {sovai-0.2.63 → sovai-0.2.64}/pyproject.toml +1 -1
- sovai-0.2.64/sovai/assets/tickers.parq +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/anomalies.py +1 -0
- sovai-0.2.64/sovai/extensions/change_point_generator.py +590 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/clustering.py +4 -1
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/dimensionality_reduction.py +2 -1
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/pandas_extensions.py +33 -9
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/time_decomposition.py +8 -4
- sovai-0.2.63/sovai/get_data.py → sovai-0.2.64/sovai/get_data copy 2.py +201 -45
- sovai-0.2.64/sovai/get_data copy.py +1732 -0
- sovai-0.2.64/sovai/get_data.py +1744 -0
- sovai-0.2.64/sovai/get_plots.py +311 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/utils/client_side_s3_part_high.py +32 -0
- sovai-0.2.64/sovai/utils/port_manager.py +292 -0
- sovai-0.2.63/sovai/get_data copy.py +0 -829
- sovai-0.2.63/sovai/get_plots.py +0 -226
- sovai-0.2.63/sovai/utils/client_side_s3_part_high copy 2.py +0 -1028
- sovai-0.2.63/sovai/utils/client_side_s3_part_high copy.py +0 -920
- {sovai-0.2.63 → sovai-0.2.64}/LICENSE +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/README.md +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/data/codes.parq +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/data/tickers.parq +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/.DS_Store +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/api_config.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/assets/dropdown.css +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/assets/features_mapping.parq +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/basic_auth.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/computations/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/computations/functions.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/errors/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/errors/sovai_errors.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/ask_df_llm.py +0 -0
- /sovai-0.2.63/sovai/extensions/change_point_generator.py → /sovai-0.2.64/sovai/extensions/change_point_generator copy.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/core_kshape.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/cum_sum_torch.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/cusum.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/feature_extraction.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/feature_importance.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/feature_neutralizer.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/filter_df.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/fractional_differencing.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/nowcasting.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/pairwise.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/parallel_functions.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/pfa_feature_selector.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/regime_change.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/regime_change_pca.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/shapley_global_importance.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/shapley_importance.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/signal_evaluation.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/technical_indicators.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/extensions/weight_optimization.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/get_compute.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/get_reports.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/get_tools.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/accounting/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/accounting/accounting_plots.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/allocation/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/allocation/allocation_plots.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/bankruptcy/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/bankruptcy/bankruptcy_plots.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/breakout/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/breakout/breakout_plots.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/corp_risk/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/corp_risk/corp_risk_plots.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/earnings_surprise/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/earnings_surprise/earnings_surprise_plots.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/insider/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/insider/insider_plots.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/institutional/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/institutional/institutional_plots.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/news/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/news/news_plots.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/ratios/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/plots/ratios/ratios_plots.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/reports/accounting/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/reports/accounting/accounting_balance_sheet.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/reports/bankruptcy/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/reports/bankruptcy/bankruptcy_monthly_top.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/reports/general/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/reports/general/general_plots.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/reports/news/__inti__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/reports/news/news_econometric_analysis.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/studies/nowcasting.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/token_auth.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/tools/authentication/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/tools/authentication/authentication.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/tools/sec/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/tools/sec/graphs.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/tools/sec/llm_code_generator.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/tools/sec/sec_10_k_8_k_filings.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/tools/sec/sec_edgar_search.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/utils/__init__.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/utils/client_side.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/utils/client_side_s3.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/utils/converter.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/utils/datetime_formats.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/utils/file_management.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/utils/get_tickers.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/utils/helpers.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/utils/pandas_extensions copy.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/utils/plot.py +0 -0
- /sovai-0.2.63/sovai/utils/port_manager.py → /sovai-0.2.64/sovai/utils/port_manager copy.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/utils/stream.py +0 -0
- {sovai-0.2.63 → sovai-0.2.64}/sovai/utils/verbose_utils.py +0 -0
|
Binary file
|
|
@@ -61,6 +61,7 @@ def prepare_and_select_neighbors(df_accounting, ticker_select, neighbours):
|
|
|
61
61
|
df_calc = df_accounting.loc[
|
|
62
62
|
df_accounting.index.get_level_values("ticker").isin(neighs)
|
|
63
63
|
]
|
|
64
|
+
df_calc = df_calc[~df_calc.index.duplicated(keep="first")] # ← NEW LINE
|
|
64
65
|
return df_calc
|
|
65
66
|
|
|
66
67
|
else:
|
|
@@ -0,0 +1,590 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pandas as pd
|
|
3
|
+
from typing import Tuple, List, Optional # Modified to include List and Optional for new functions
|
|
4
|
+
import plotly.graph_objects as go
|
|
5
|
+
from plotly.subplots import make_subplots
|
|
6
|
+
import dash
|
|
7
|
+
from dash import dcc, html
|
|
8
|
+
from dash.dependencies import Input, Output
|
|
9
|
+
import random
|
|
10
|
+
import socket # Added for port management
|
|
11
|
+
import time # Added for port management
|
|
12
|
+
import errno # Added for port management
|
|
13
|
+
|
|
14
|
+
# from port_manager_utility import get_unique_port, kill_process_on_port, release_port, get_port_manager_instance
|
|
15
|
+
# from utils.port_manager import get_unique_port, kill_process_on_port
|
|
16
|
+
from sovai.utils.port_manager import get_unique_port, kill_process_on_port
|
|
17
|
+
|
|
18
|
+
random.seed(42)
|
|
19
|
+
np.random.seed(42)
|
|
20
|
+
|
|
21
|
+
# Global dictionary to keep track of apps and their ports launched by this script
|
|
22
|
+
app_ports = {}
|
|
23
|
+
|
|
24
|
+
class ImprovedCusumDetector: # Exactly as in the first script
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
window_size_ratio: float = 0.05,
|
|
28
|
+
threshold_factor: float = 2.5,
|
|
29
|
+
drift_factor: float = 0.05,
|
|
30
|
+
min_distance: int = 30,
|
|
31
|
+
sideways_threshold: float = 0.03,
|
|
32
|
+
warm_up_period_ratio: float = 0.1,
|
|
33
|
+
):
|
|
34
|
+
self.window_size_ratio = window_size_ratio
|
|
35
|
+
self.threshold_factor = threshold_factor
|
|
36
|
+
self.drift_factor = drift_factor
|
|
37
|
+
self.min_distance = min_distance
|
|
38
|
+
self.sideways_threshold = sideways_threshold
|
|
39
|
+
self.warm_up_period_ratio = warm_up_period_ratio
|
|
40
|
+
|
|
41
|
+
def detect_changepoints(
|
|
42
|
+
self, signal: np.ndarray
|
|
43
|
+
) -> Tuple[np.ndarray, np.ndarray, np.ndarray, int]:
|
|
44
|
+
n = len(signal)
|
|
45
|
+
window_size = max(int(n * self.window_size_ratio), 8)
|
|
46
|
+
warm_up_period = int(n * self.warm_up_period_ratio)
|
|
47
|
+
|
|
48
|
+
ewma = self._calculate_ewma(signal, window_size)
|
|
49
|
+
z_scores = self._calculate_rolling_z_scores(signal, ewma, window_size)
|
|
50
|
+
scores, changepoints, trend_codes = self._calculate_two_sided_cusum(
|
|
51
|
+
signal,
|
|
52
|
+
z_scores,
|
|
53
|
+
self.drift_factor,
|
|
54
|
+
self.threshold_factor,
|
|
55
|
+
self.min_distance,
|
|
56
|
+
self.sideways_threshold,
|
|
57
|
+
warm_up_period,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
trends = np.where(
|
|
61
|
+
trend_codes == 1,
|
|
62
|
+
"Increasing",
|
|
63
|
+
np.where(trend_codes == -1, "Decreasing", "Sideways"),
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
return changepoints, trends, scores, window_size
|
|
67
|
+
|
|
68
|
+
import numba
|
|
69
|
+
import numpy as np
|
|
70
|
+
|
|
71
|
+
@staticmethod
|
|
72
|
+
@numba.njit(cache=True)
|
|
73
|
+
def _calculate_ewma(signal, window_size):
|
|
74
|
+
alpha = 2 / (window_size + 1)
|
|
75
|
+
ewma = np.zeros_like(signal)
|
|
76
|
+
ewma[0] = signal[0]
|
|
77
|
+
for i in range(1, len(signal)):
|
|
78
|
+
ewma[i] = alpha * signal[i] + (1 - alpha) * ewma[i - 1]
|
|
79
|
+
return ewma
|
|
80
|
+
|
|
81
|
+
@staticmethod
|
|
82
|
+
@numba.njit(cache=True)
|
|
83
|
+
def _calculate_rolling_z_scores(signal, ewma, window_size):
|
|
84
|
+
rolling_var = np.zeros_like(signal)
|
|
85
|
+
for i in range(window_size, len(signal)):
|
|
86
|
+
rolling_var[i] = np.var(signal[i - window_size + 1 : i + 1])
|
|
87
|
+
rolling_std = np.sqrt(rolling_var)
|
|
88
|
+
z_scores = (signal - ewma) / (rolling_std + 1e-8)
|
|
89
|
+
return z_scores
|
|
90
|
+
|
|
91
|
+
@staticmethod
|
|
92
|
+
@numba.njit(cache=True)
|
|
93
|
+
def _calculate_two_sided_cusum(
|
|
94
|
+
signal,
|
|
95
|
+
z,
|
|
96
|
+
drift,
|
|
97
|
+
threshold_factor,
|
|
98
|
+
min_distance,
|
|
99
|
+
sideways_threshold,
|
|
100
|
+
warm_up_period,
|
|
101
|
+
):
|
|
102
|
+
cs_upper = np.zeros_like(z)
|
|
103
|
+
cs_lower = np.zeros_like(z)
|
|
104
|
+
changepoints = []
|
|
105
|
+
trend_codes = []
|
|
106
|
+
last_cp = -min_distance
|
|
107
|
+
for i in range(1, len(z)):
|
|
108
|
+
if i < warm_up_period:
|
|
109
|
+
continue
|
|
110
|
+
cs_upper[i] = max(0, cs_upper[i - 1] + z[i] - drift)
|
|
111
|
+
cs_lower[i] = min(0, cs_lower[i - 1] + z[i] + drift)
|
|
112
|
+
if i - last_cp >= min_distance:
|
|
113
|
+
if cs_upper[i] > threshold_factor * np.sqrt(i):
|
|
114
|
+
changepoints.append(i)
|
|
115
|
+
trend_codes.append(1)
|
|
116
|
+
cs_upper[i] = 0
|
|
117
|
+
cs_lower[i] = 0
|
|
118
|
+
last_cp = i
|
|
119
|
+
elif cs_lower[i] < -threshold_factor * np.sqrt(i):
|
|
120
|
+
changepoints.append(i)
|
|
121
|
+
trend_codes.append(-1)
|
|
122
|
+
cs_upper[i] = 0
|
|
123
|
+
cs_lower[i] = 0
|
|
124
|
+
last_cp = i
|
|
125
|
+
elif ( # This condition for sideways might lead to ZeroDivisionError if signal[last_cp] is 0. Preserved as per original.
|
|
126
|
+
i > 0
|
|
127
|
+
and abs((signal[i] - signal[last_cp]) / signal[last_cp])
|
|
128
|
+
< sideways_threshold
|
|
129
|
+
):
|
|
130
|
+
changepoints.append(i)
|
|
131
|
+
trend_codes.append(0)
|
|
132
|
+
cs_upper[i] = 0
|
|
133
|
+
cs_lower[i] = 0
|
|
134
|
+
last_cp = i
|
|
135
|
+
return (
|
|
136
|
+
np.maximum(cs_upper, -cs_lower),
|
|
137
|
+
np.array(changepoints),
|
|
138
|
+
np.array(trend_codes),
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
def plot_cusum_results( # Exactly as in the first script
|
|
142
|
+
df_signal, signal_array, changepoints, trends, scores, ticker, feature
|
|
143
|
+
):
|
|
144
|
+
fig = make_subplots(
|
|
145
|
+
rows=2,
|
|
146
|
+
cols=1,
|
|
147
|
+
shared_xaxes=True,
|
|
148
|
+
vertical_spacing=0.1,
|
|
149
|
+
subplot_titles=(f"{feature} Over Time for {ticker}", "CUSUM Score"),
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
fig.add_trace(
|
|
153
|
+
go.Scatter(
|
|
154
|
+
x=df_signal.index,
|
|
155
|
+
y=signal_array,
|
|
156
|
+
mode="lines",
|
|
157
|
+
line=dict(color="gold", width=2),
|
|
158
|
+
name=feature,
|
|
159
|
+
),
|
|
160
|
+
row=1,
|
|
161
|
+
col=1,
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
fig.add_trace(
|
|
165
|
+
go.Scatter(
|
|
166
|
+
x=df_signal.index,
|
|
167
|
+
y=scores,
|
|
168
|
+
mode="lines",
|
|
169
|
+
line=dict(color="rgba(30, 144, 255, 0.7)", width=1.5),
|
|
170
|
+
name="CUSUM Score",
|
|
171
|
+
),
|
|
172
|
+
row=2,
|
|
173
|
+
col=1,
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
colors = {"Increasing": "lime", "Decreasing": "red", "Sideways": "yellow"}
|
|
177
|
+
symbols = {
|
|
178
|
+
"Increasing": "triangle-up",
|
|
179
|
+
"Decreasing": "triangle-down",
|
|
180
|
+
"Sideways": "circle",
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
for trend in set(trends):
|
|
184
|
+
mask = trends == trend
|
|
185
|
+
# Original script might have index out of bounds if changepoints[mask] is empty or values are too large. Preserved.
|
|
186
|
+
fig.add_trace(
|
|
187
|
+
go.Scatter(
|
|
188
|
+
x=df_signal.index[changepoints[mask]],
|
|
189
|
+
y=signal_array[changepoints[mask]],
|
|
190
|
+
mode="markers",
|
|
191
|
+
marker=dict(color=colors[trend], size=10, symbol=symbols[trend]),
|
|
192
|
+
name=f"{trend}",
|
|
193
|
+
),
|
|
194
|
+
row=1,
|
|
195
|
+
col=1,
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
fig.update_layout(
|
|
199
|
+
title=f"{feature} Change Point Detection for {ticker}",
|
|
200
|
+
plot_bgcolor="rgba(0,0,0,0.95)",
|
|
201
|
+
paper_bgcolor="rgba(0,0,0,0.95)",
|
|
202
|
+
font=dict(color="white", size=12),
|
|
203
|
+
autosize=True,
|
|
204
|
+
legend=dict(
|
|
205
|
+
orientation="h",
|
|
206
|
+
yanchor="bottom",
|
|
207
|
+
y=1.1,
|
|
208
|
+
xanchor="right",
|
|
209
|
+
x=1,
|
|
210
|
+
bgcolor="rgba(0,0,0,0.5)",
|
|
211
|
+
bordercolor="rgba(255,255,255,0.5)",
|
|
212
|
+
),
|
|
213
|
+
showlegend=True,
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
fig.update_xaxes(title_text="Date", row=2, col=1, gridcolor="rgba(255,255,255,0.2)")
|
|
217
|
+
fig.update_yaxes(
|
|
218
|
+
title_text=feature, type="log", row=1, col=1, gridcolor="rgba(255,255,255,0.2)"
|
|
219
|
+
)
|
|
220
|
+
fig.update_yaxes(
|
|
221
|
+
title_text="CUSUM Score", row=2, col=1, gridcolor="rgba(255,255,255,0.2)"
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
for i, (cp, trend) in enumerate(zip(changepoints, trends)):
|
|
225
|
+
# Original script might have index out of bounds if cp is too large. Preserved.
|
|
226
|
+
fig.add_annotation(
|
|
227
|
+
x=df_signal.index[cp],
|
|
228
|
+
y=signal_array[cp],
|
|
229
|
+
text=f"{trend[0]} {i+1}",
|
|
230
|
+
showarrow=False,
|
|
231
|
+
yshift=10 if trend == "Increasing" else -10,
|
|
232
|
+
font=dict(color=colors[trend]),
|
|
233
|
+
row=1,
|
|
234
|
+
col=1,
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
return fig
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def run_cusum_dashboard(df_accounting, ticker=None, feature=None): # Modified for new port logic
|
|
241
|
+
if ticker is None:
|
|
242
|
+
ticker_options = df_accounting.index.get_level_values("ticker").unique()
|
|
243
|
+
if not ticker_options.empty:
|
|
244
|
+
ticker = ticker_options[0]
|
|
245
|
+
else:
|
|
246
|
+
print("Error: No tickers found in DataFrame for dashboard.")
|
|
247
|
+
return
|
|
248
|
+
if feature is None:
|
|
249
|
+
if not df_accounting.columns.empty:
|
|
250
|
+
feature = df_accounting.columns[0]
|
|
251
|
+
else:
|
|
252
|
+
print("Error: No features found in DataFrame for dashboard.")
|
|
253
|
+
return
|
|
254
|
+
|
|
255
|
+
app = dash.Dash(__name__)
|
|
256
|
+
|
|
257
|
+
dark_mode_colors = {
|
|
258
|
+
"background": "#1E1E1E",
|
|
259
|
+
"text": "#FFFFFF",
|
|
260
|
+
"plot_bg": "#2B2B2B",
|
|
261
|
+
"plot_gridlines": "#3A3A3A",
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
app.layout = html.Div(
|
|
265
|
+
[
|
|
266
|
+
html.H3(
|
|
267
|
+
"Financial Metric Change Point Detection",
|
|
268
|
+
style={
|
|
269
|
+
"textAlign": "center",
|
|
270
|
+
"marginBottom": "20px",
|
|
271
|
+
"paddingTop": "10px",
|
|
272
|
+
"color": dark_mode_colors["text"],
|
|
273
|
+
},
|
|
274
|
+
),
|
|
275
|
+
html.Div(
|
|
276
|
+
[
|
|
277
|
+
html.Div(
|
|
278
|
+
[
|
|
279
|
+
html.Label(
|
|
280
|
+
"Select Ticker:",
|
|
281
|
+
style={"color": dark_mode_colors["text"]},
|
|
282
|
+
),
|
|
283
|
+
dcc.Dropdown(
|
|
284
|
+
id="ticker-dropdown",
|
|
285
|
+
options=[
|
|
286
|
+
{"label": t, "value": t}
|
|
287
|
+
for t in df_accounting.index.get_level_values(
|
|
288
|
+
"ticker"
|
|
289
|
+
).unique()
|
|
290
|
+
],
|
|
291
|
+
value=ticker,
|
|
292
|
+
),
|
|
293
|
+
],
|
|
294
|
+
style={"width": "48%", "display": "inline-block"},
|
|
295
|
+
),
|
|
296
|
+
html.Div(
|
|
297
|
+
[
|
|
298
|
+
html.Label(
|
|
299
|
+
"Select Feature:",
|
|
300
|
+
style={"color": dark_mode_colors["text"]},
|
|
301
|
+
),
|
|
302
|
+
dcc.Dropdown(
|
|
303
|
+
id="feature-dropdown",
|
|
304
|
+
options=[
|
|
305
|
+
{"label": col, "value": col}
|
|
306
|
+
for col in df_accounting.columns
|
|
307
|
+
],
|
|
308
|
+
value=feature,
|
|
309
|
+
),
|
|
310
|
+
],
|
|
311
|
+
style={
|
|
312
|
+
"width": "48%",
|
|
313
|
+
"float": "right",
|
|
314
|
+
"display": "inline-block",
|
|
315
|
+
},
|
|
316
|
+
),
|
|
317
|
+
],
|
|
318
|
+
style={"padding": "20px 0px"},
|
|
319
|
+
),
|
|
320
|
+
dcc.Graph(id="cusum-plot", style={"height": "80vh"}),
|
|
321
|
+
],
|
|
322
|
+
style={
|
|
323
|
+
"padding": "0 20px",
|
|
324
|
+
"margin": "0 auto",
|
|
325
|
+
"backgroundColor": dark_mode_colors["background"],
|
|
326
|
+
},
|
|
327
|
+
)
|
|
328
|
+
|
|
329
|
+
@app.callback(
|
|
330
|
+
Output("cusum-plot", "figure"),
|
|
331
|
+
[Input("ticker-dropdown", "value"), Input("feature-dropdown", "value")],
|
|
332
|
+
)
|
|
333
|
+
def update_graph(selected_ticker, selected_feature): # Exactly as in the first script's dashboard
|
|
334
|
+
df_signal = (
|
|
335
|
+
df_accounting.query(f"ticker == '{selected_ticker}'")
|
|
336
|
+
.reset_index()
|
|
337
|
+
.set_index("date")[selected_feature]
|
|
338
|
+
)
|
|
339
|
+
signal_array = df_signal.values
|
|
340
|
+
# Add a check for empty signal_array to prevent errors downstream
|
|
341
|
+
if len(signal_array) == 0:
|
|
342
|
+
fig = go.Figure()
|
|
343
|
+
fig.update_layout(
|
|
344
|
+
title_text=f"No data available for {selected_ticker} - {selected_feature}",
|
|
345
|
+
plot_bgcolor=dark_mode_colors["plot_bg"],
|
|
346
|
+
paper_bgcolor=dark_mode_colors["plot_bg"],
|
|
347
|
+
font_color=dark_mode_colors["text"],
|
|
348
|
+
)
|
|
349
|
+
return fig
|
|
350
|
+
|
|
351
|
+
detector = ImprovedCusumDetector()
|
|
352
|
+
changepoints, trends, scores, window_size = detector.detect_changepoints(
|
|
353
|
+
signal_array
|
|
354
|
+
)
|
|
355
|
+
fig = plot_cusum_results(
|
|
356
|
+
df_signal,
|
|
357
|
+
signal_array,
|
|
358
|
+
changepoints,
|
|
359
|
+
trends,
|
|
360
|
+
scores,
|
|
361
|
+
selected_ticker,
|
|
362
|
+
selected_feature,
|
|
363
|
+
)
|
|
364
|
+
fig.update_layout(
|
|
365
|
+
plot_bgcolor=dark_mode_colors["plot_bg"],
|
|
366
|
+
paper_bgcolor=dark_mode_colors["plot_bg"],
|
|
367
|
+
font_color=dark_mode_colors["text"],
|
|
368
|
+
)
|
|
369
|
+
fig.update_xaxes(gridcolor=dark_mode_colors["plot_gridlines"])
|
|
370
|
+
fig.update_yaxes(gridcolor=dark_mode_colors["plot_gridlines"])
|
|
371
|
+
return fig
|
|
372
|
+
|
|
373
|
+
app_name = f"cumsum-app-{ticker}-{feature}".replace(" ","_") # Using more specific app name
|
|
374
|
+
|
|
375
|
+
# --- Robust Server Launch (from the second script) ---
|
|
376
|
+
max_retries = 3
|
|
377
|
+
for attempt in range(max_retries):
|
|
378
|
+
port = get_unique_port(app_name)
|
|
379
|
+
try:
|
|
380
|
+
print(f"Attempting to launch Dash app '{app_name}' on port {port} (Attempt {attempt + 1}/{max_retries})")
|
|
381
|
+
# Using host='0.0.0.0' to make it accessible on the network
|
|
382
|
+
app.run_server(debug=False, port=port, host='0.0.0.0')
|
|
383
|
+
print(f"Dash app '{app_name}' running on http://127.0.0.1:{port}/ or http://<your-ip>:{port}/")
|
|
384
|
+
break # Success
|
|
385
|
+
except socket.error as e:
|
|
386
|
+
if e.errno == errno.EADDRINUSE: # Address already in use
|
|
387
|
+
print(f"Port {port} is in use. Attempting to kill blocking process(es).")
|
|
388
|
+
if kill_process_on_port(port):
|
|
389
|
+
print(f"Successfully killed process(es) on port {port}. Retrying Dash launch...")
|
|
390
|
+
time.sleep(2) # Give OS a moment to free the port
|
|
391
|
+
else:
|
|
392
|
+
print(f"Could not kill process(es) on port {port} or no process found. Trying a different port.")
|
|
393
|
+
# Invalidate this port for this app_name so get_unique_port tries a new one
|
|
394
|
+
if app_name in app_ports and app_ports[app_name] == port:
|
|
395
|
+
del app_ports[app_name]
|
|
396
|
+
|
|
397
|
+
if attempt == max_retries - 1:
|
|
398
|
+
print(f"Failed to launch Dash app '{app_name}' after {max_retries} attempts. Port {port} remains occupied.")
|
|
399
|
+
# Not returning here, as the original run_server call was the return
|
|
400
|
+
# This will effectively mean the function finishes if all retries fail.
|
|
401
|
+
# To match original behavior of returning the server object, this structure would need adjustment
|
|
402
|
+
# However, run_server is blocking, so returning it means the rest of the script doesn't run until server stops.
|
|
403
|
+
# This current structure runs the server and blocks, or prints failure and continues if it can't.
|
|
404
|
+
# The original `return app.run_server(...)` would also block.
|
|
405
|
+
# If it fails to launch due to port, original would raise error immediately.
|
|
406
|
+
# This structure tries to recover or fails after retries.
|
|
407
|
+
# To make it fully equivalent on failure, we should re-raise here.
|
|
408
|
+
raise RuntimeError(f"Failed to launch Dash app '{app_name}' after {max_retries} attempts. Port {port} remains occupied.")
|
|
409
|
+
else:
|
|
410
|
+
print(f"An unexpected socket error occurred: {e}")
|
|
411
|
+
raise # Re-raise other socket errors
|
|
412
|
+
except Exception as e:
|
|
413
|
+
print(f"An unexpected error occurred during app launch: {e}")
|
|
414
|
+
raise
|
|
415
|
+
else: # Executed if the loop completes without break (i.e., all retries failed before exception)
|
|
416
|
+
print(f"CRITICAL: Dash app '{app_name}' could not be started after {max_retries} attempts due to port issues without raising an exception in the loop (should not happen).")
|
|
417
|
+
# This else block for the for loop is unlikely to be hit if EADDRINUSE always occurs and leads to a raise on final attempt.
|
|
418
|
+
# If get_unique_port exhausted options and raised an error, that would be caught by the generic Exception.
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def perform_cusum_analysis(df_accounting, ticker=None, feature=None): # Exactly as in the first script
|
|
422
|
+
# Get the first ticker and feature if not provided
|
|
423
|
+
if ticker is None:
|
|
424
|
+
ticker = df_accounting.index.get_level_values("ticker").unique()[0]
|
|
425
|
+
if feature is None:
|
|
426
|
+
feature = df_accounting.columns[0]
|
|
427
|
+
|
|
428
|
+
# Extract the signal for the specified ticker and feature
|
|
429
|
+
df_signal = (
|
|
430
|
+
df_accounting.query(f"ticker == '{ticker}'")
|
|
431
|
+
.reset_index()
|
|
432
|
+
.set_index("date")[feature]
|
|
433
|
+
)
|
|
434
|
+
|
|
435
|
+
detector = ImprovedCusumDetector()
|
|
436
|
+
signal_array = df_signal.values
|
|
437
|
+
|
|
438
|
+
# Added basic check for empty signal to prevent error in detect_changepoints
|
|
439
|
+
if len(signal_array) == 0:
|
|
440
|
+
print(f"Warning: No data for {ticker} - {feature} in perform_cusum_analysis. Returning empty results.")
|
|
441
|
+
empty_analysis_df = pd.DataFrame(columns=["Date", "Trend", "Signal_Value", "CUSUM_Score", "Duration", "Duration_Days", "Percent_Change"])
|
|
442
|
+
empty_stats_df = pd.DataFrame({
|
|
443
|
+
"Statistic": [
|
|
444
|
+
"Ticker", "Feature", "Total_Change_Points", "Average_Duration_Days",
|
|
445
|
+
"Average_Percent_Change", "Increasing_Trends", "Decreasing_Trends",
|
|
446
|
+
"Sideways_Trends", "Max_CUSUM_Score", "Min_CUSUM_Score", "Window_Size"],
|
|
447
|
+
"Value": [ticker, feature, 0, 0, 0, 0, 0, 0, 0, 0, 0]
|
|
448
|
+
})
|
|
449
|
+
empty_analysis_df.attrs["stats"] = empty_stats_df
|
|
450
|
+
return empty_analysis_df
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
changepoints, trends, scores, window_size = detector.detect_changepoints(
|
|
454
|
+
signal_array
|
|
455
|
+
)
|
|
456
|
+
|
|
457
|
+
# Handle case with no changepoints to avoid errors
|
|
458
|
+
if len(changepoints) == 0:
|
|
459
|
+
analysis_df = pd.DataFrame(columns=["Date", "Trend", "Signal_Value", "CUSUM_Score"])
|
|
460
|
+
else:
|
|
461
|
+
# Original script might have index out of bounds or issues if changepoints are empty or too large. Preserved.
|
|
462
|
+
# CUSUM_Score indexing `changepoints - 1` could be problematic if changepoints contains 0.
|
|
463
|
+
# `_calculate_two_sided_cusum` starts `i` from 1, so changepoints should be >= 1.
|
|
464
|
+
# However, if warm_up_period is very large, changepoints could be empty.
|
|
465
|
+
analysis_df = pd.DataFrame(
|
|
466
|
+
{
|
|
467
|
+
"Date": df_signal.index[changepoints],
|
|
468
|
+
"Trend": trends,
|
|
469
|
+
"Signal_Value": signal_array[changepoints],
|
|
470
|
+
"CUSUM_Score": scores[changepoints - 1],
|
|
471
|
+
}
|
|
472
|
+
)
|
|
473
|
+
|
|
474
|
+
# Calculate duration between change points
|
|
475
|
+
if not analysis_df.empty:
|
|
476
|
+
analysis_df["Duration"] = (
|
|
477
|
+
analysis_df["Date"].diff().shift(-1).fillna(pd.Timedelta(days=0))
|
|
478
|
+
)
|
|
479
|
+
analysis_df["Duration_Days"] = analysis_df["Duration"].dt.days
|
|
480
|
+
|
|
481
|
+
# Calculate percentage change between change points
|
|
482
|
+
analysis_df["Percent_Change"] = (
|
|
483
|
+
analysis_df["Signal_Value"].pct_change().shift(-1) * 100
|
|
484
|
+
).fillna(0)
|
|
485
|
+
else: # Ensure columns exist if analysis_df is empty
|
|
486
|
+
analysis_df["Duration"] = pd.Series(dtype='timedelta64[ns]')
|
|
487
|
+
analysis_df["Duration_Days"] = pd.Series(dtype='int')
|
|
488
|
+
analysis_df["Percent_Change"] = pd.Series(dtype='float')
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
# Overall statistics
|
|
492
|
+
# Original `scores[scores > 0].min()` could fail if scores[scores > 0] is empty.
|
|
493
|
+
min_cusum_score_val = 0
|
|
494
|
+
if any(scores > 0):
|
|
495
|
+
min_positive_scores = scores[scores > 0]
|
|
496
|
+
if len(min_positive_scores) > 0:
|
|
497
|
+
min_cusum_score_val = min_positive_scores.min()
|
|
498
|
+
|
|
499
|
+
overall_stats = pd.DataFrame(
|
|
500
|
+
{
|
|
501
|
+
"Statistic": [
|
|
502
|
+
"Ticker",
|
|
503
|
+
"Feature",
|
|
504
|
+
"Total_Change_Points",
|
|
505
|
+
"Average_Duration_Days",
|
|
506
|
+
"Average_Percent_Change",
|
|
507
|
+
"Increasing_Trends",
|
|
508
|
+
"Decreasing_Trends",
|
|
509
|
+
"Sideways_Trends",
|
|
510
|
+
"Max_CUSUM_Score",
|
|
511
|
+
"Min_CUSUM_Score", # Original name was Min_CUSUM_Score
|
|
512
|
+
"Window_Size",
|
|
513
|
+
],
|
|
514
|
+
"Value": [
|
|
515
|
+
ticker,
|
|
516
|
+
feature,
|
|
517
|
+
len(changepoints),
|
|
518
|
+
analysis_df["Duration_Days"].mean() if not analysis_df.empty else 0,
|
|
519
|
+
analysis_df["Percent_Change"].mean() if not analysis_df.empty else 0,
|
|
520
|
+
sum(trends == "Increasing") if len(trends) > 0 else 0,
|
|
521
|
+
sum(trends == "Decreasing") if len(trends) > 0 else 0,
|
|
522
|
+
sum(trends == "Sideways") if len(trends) > 0 else 0,
|
|
523
|
+
scores.max() if len(scores) > 0 else 0,
|
|
524
|
+
min_cusum_score_val,
|
|
525
|
+
window_size,
|
|
526
|
+
],
|
|
527
|
+
}
|
|
528
|
+
)
|
|
529
|
+
# Fill NaN for mean values if analysis_df was empty or had single row for stats
|
|
530
|
+
overall_stats['Value'] = overall_stats['Value'].fillna(0)
|
|
531
|
+
|
|
532
|
+
analysis_df.attrs["stats"] = overall_stats
|
|
533
|
+
|
|
534
|
+
return analysis_df
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
if __name__ == "__main__":
|
|
538
|
+
# Create a dummy df_accounting for demonstration as in the second script
|
|
539
|
+
# This helps in testing the updated script functionality.
|
|
540
|
+
dates = pd.to_datetime(pd.date_range(start="2020-01-01", periods=300, freq="B"))
|
|
541
|
+
data_aapl = np.random.lognormal(mean=0.001, sigma=0.02, size=300).cumprod() * 100
|
|
542
|
+
data_msft = np.random.lognormal(mean=0.0005, sigma=0.025, size=300).cumprod() * 150
|
|
543
|
+
|
|
544
|
+
data_aapl[100:150] *= 1.3
|
|
545
|
+
data_aapl[200:250] *= 0.7
|
|
546
|
+
|
|
547
|
+
data_msft[50:80] *= 0.8
|
|
548
|
+
data_msft[150:200] *= 1.5
|
|
549
|
+
data_msft[250:] += 20
|
|
550
|
+
|
|
551
|
+
df_aapl = pd.DataFrame({"Close": data_aapl, "Volume": np.random.randint(1000, 5000, 300)}, index=dates)
|
|
552
|
+
df_aapl["ticker"] = "AAPL"
|
|
553
|
+
|
|
554
|
+
df_msft = pd.DataFrame({"Close": data_msft, "Volume": np.random.randint(2000, 7000, 300)}, index=dates)
|
|
555
|
+
df_msft["ticker"] = "MSFT"
|
|
556
|
+
|
|
557
|
+
# Ensure df_accounting structure matches what functions expect (MultiIndex with 'ticker', 'date')
|
|
558
|
+
df_accounting_demo = pd.concat([
|
|
559
|
+
df_aapl.reset_index().rename(columns={'index': 'date'}).set_index(['ticker', 'date']),
|
|
560
|
+
df_msft.reset_index().rename(columns={'index': 'date'}).set_index(['ticker', 'date'])
|
|
561
|
+
])
|
|
562
|
+
|
|
563
|
+
print("---- Running CUSUM Analysis for AAPL Close ----")
|
|
564
|
+
analysis_results_aapl = perform_cusum_analysis(df_accounting_demo, ticker="AAPL", feature="Close")
|
|
565
|
+
if not analysis_results_aapl.empty: # Check if DataFrame is not empty
|
|
566
|
+
print(analysis_results_aapl)
|
|
567
|
+
print("\nOverall Stats (AAPL Close):")
|
|
568
|
+
print(analysis_results_aapl.attrs["stats"])
|
|
569
|
+
else:
|
|
570
|
+
print("No analysis results for AAPL Close.")
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
print("\n---- Running CUSUM Analysis for MSFT Volume ----")
|
|
574
|
+
analysis_results_msft_vol = perform_cusum_analysis(df_accounting_demo, ticker="MSFT", feature="Volume")
|
|
575
|
+
if not analysis_results_msft_vol.empty: # Check if DataFrame is not empty
|
|
576
|
+
print(analysis_results_msft_vol)
|
|
577
|
+
print("\nOverall Stats (MSFT Volume):")
|
|
578
|
+
print(analysis_results_msft_vol.attrs["stats"])
|
|
579
|
+
else:
|
|
580
|
+
print("No analysis results for MSFT Volume.")
|
|
581
|
+
|
|
582
|
+
print("\n---- Launching Dashboard ----")
|
|
583
|
+
print("Note: If a port is busy, the script will attempt to kill the process.")
|
|
584
|
+
print("Ensure you are comfortable with this, especially if not in a dev environment.")
|
|
585
|
+
|
|
586
|
+
try:
|
|
587
|
+
# df_accounting_demo needs to be passed to the dashboard
|
|
588
|
+
run_cusum_dashboard(df_accounting_demo, ticker="AAPL", feature="Close")
|
|
589
|
+
except Exception as e:
|
|
590
|
+
print(f"Dashboard launch failed: {e}")
|
|
@@ -151,6 +151,9 @@ def segment_series(df_accounting, features_select=None):
|
|
|
151
151
|
|
|
152
152
|
|
|
153
153
|
def cluster(df_mega, features_select=None):
|
|
154
|
+
|
|
155
|
+
df_mega = df_mega[~df_mega.index.duplicated(keep="first")] # ← NEW LINE
|
|
156
|
+
|
|
154
157
|
days = df_mega.index.get_level_values("date").nunique()
|
|
155
158
|
|
|
156
159
|
(
|
|
@@ -205,7 +208,7 @@ def cluster(df_mega, features_select=None):
|
|
|
205
208
|
output_df["labels"] = output_df["labels"].apply(lambda x: f"Centroid {int(x)}")
|
|
206
209
|
|
|
207
210
|
# Group by 'ticker' and sum across rows
|
|
208
|
-
summed_series = output_df.groupby("ticker").sum().sum(axis=1)
|
|
211
|
+
summed_series = output_df.groupby("ticker").sum(numeric_only=True).sum(axis=1)
|
|
209
212
|
|
|
210
213
|
# Find the indices where the value is 0.00
|
|
211
214
|
zero_indices = summed_series[summed_series == 0.00].index.tolist()
|
|
@@ -4,7 +4,7 @@ from sklearn.decomposition import PCA, TruncatedSVD, FactorAnalysis
|
|
|
4
4
|
from sklearn.random_projection import GaussianRandomProjection
|
|
5
5
|
from sklearn.preprocessing import StandardScaler
|
|
6
6
|
from sklearn.impute import SimpleImputer
|
|
7
|
-
|
|
7
|
+
|
|
8
8
|
import warnings
|
|
9
9
|
|
|
10
10
|
|
|
@@ -124,6 +124,7 @@ def dimensionality_reduction(df, method, explained_variance=0.95, n_components=N
|
|
|
124
124
|
"truncated_svd": TruncatedSVD,
|
|
125
125
|
"factor_analysis": FactorAnalysis,
|
|
126
126
|
"gaussian_random_projection": GaussianRandomProjection,
|
|
127
|
+
|
|
127
128
|
}
|
|
128
129
|
|
|
129
130
|
if n_components is not None:
|