sovai 0.2.63__tar.gz → 0.2.65__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. {sovai-0.2.63 → sovai-0.2.65}/PKG-INFO +1 -1
  2. {sovai-0.2.63 → sovai-0.2.65}/pyproject.toml +1 -1
  3. sovai-0.2.65/sovai/assets/tickers.parq +0 -0
  4. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/anomalies.py +1 -0
  5. sovai-0.2.65/sovai/extensions/change_point_generator.py +590 -0
  6. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/clustering.py +4 -1
  7. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/dimensionality_reduction.py +2 -1
  8. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/pandas_extensions.py +36 -12
  9. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/signal_evaluation.py +255 -49
  10. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/time_decomposition.py +8 -4
  11. sovai-0.2.63/sovai/get_data.py → sovai-0.2.65/sovai/get_data copy 2.py +201 -45
  12. sovai-0.2.65/sovai/get_data copy.py +1732 -0
  13. sovai-0.2.65/sovai/get_data.py +1744 -0
  14. sovai-0.2.65/sovai/get_plots.py +311 -0
  15. {sovai-0.2.63 → sovai-0.2.65}/sovai/utils/client_side_s3_part_high.py +32 -0
  16. sovai-0.2.65/sovai/utils/port_manager.py +292 -0
  17. sovai-0.2.63/sovai/get_data copy.py +0 -829
  18. sovai-0.2.63/sovai/get_plots.py +0 -226
  19. sovai-0.2.63/sovai/utils/client_side_s3_part_high copy 2.py +0 -1028
  20. sovai-0.2.63/sovai/utils/client_side_s3_part_high copy.py +0 -920
  21. {sovai-0.2.63 → sovai-0.2.65}/LICENSE +0 -0
  22. {sovai-0.2.63 → sovai-0.2.65}/README.md +0 -0
  23. {sovai-0.2.63 → sovai-0.2.65}/data/codes.parq +0 -0
  24. {sovai-0.2.63 → sovai-0.2.65}/data/tickers.parq +0 -0
  25. {sovai-0.2.63 → sovai-0.2.65}/sovai/.DS_Store +0 -0
  26. {sovai-0.2.63 → sovai-0.2.65}/sovai/__init__.py +0 -0
  27. {sovai-0.2.63 → sovai-0.2.65}/sovai/api_config.py +0 -0
  28. {sovai-0.2.63 → sovai-0.2.65}/sovai/assets/dropdown.css +0 -0
  29. {sovai-0.2.63 → sovai-0.2.65}/sovai/assets/features_mapping.parq +0 -0
  30. {sovai-0.2.63 → sovai-0.2.65}/sovai/basic_auth.py +0 -0
  31. {sovai-0.2.63 → sovai-0.2.65}/sovai/computations/__init__.py +0 -0
  32. {sovai-0.2.63 → sovai-0.2.65}/sovai/computations/functions.py +0 -0
  33. {sovai-0.2.63 → sovai-0.2.65}/sovai/errors/__init__.py +0 -0
  34. {sovai-0.2.63 → sovai-0.2.65}/sovai/errors/sovai_errors.py +0 -0
  35. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/__init__.py +0 -0
  36. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/ask_df_llm.py +0 -0
  37. /sovai-0.2.63/sovai/extensions/change_point_generator.py → /sovai-0.2.65/sovai/extensions/change_point_generator copy.py +0 -0
  38. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/core_kshape.py +0 -0
  39. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/cum_sum_torch.py +0 -0
  40. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/cusum.py +0 -0
  41. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/feature_extraction.py +0 -0
  42. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/feature_importance.py +0 -0
  43. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/feature_neutralizer.py +0 -0
  44. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/filter_df.py +0 -0
  45. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/fractional_differencing.py +0 -0
  46. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/nowcasting.py +0 -0
  47. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/pairwise.py +0 -0
  48. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/parallel_functions.py +0 -0
  49. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/pfa_feature_selector.py +0 -0
  50. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/regime_change.py +0 -0
  51. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/regime_change_pca.py +0 -0
  52. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/shapley_global_importance.py +0 -0
  53. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/shapley_importance.py +0 -0
  54. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/technical_indicators.py +0 -0
  55. {sovai-0.2.63 → sovai-0.2.65}/sovai/extensions/weight_optimization.py +0 -0
  56. {sovai-0.2.63 → sovai-0.2.65}/sovai/get_compute.py +0 -0
  57. {sovai-0.2.63 → sovai-0.2.65}/sovai/get_reports.py +0 -0
  58. {sovai-0.2.63 → sovai-0.2.65}/sovai/get_tools.py +0 -0
  59. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/__init__.py +0 -0
  60. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/accounting/__init__.py +0 -0
  61. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/accounting/accounting_plots.py +0 -0
  62. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/allocation/__init__.py +0 -0
  63. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/allocation/allocation_plots.py +0 -0
  64. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/bankruptcy/__init__.py +0 -0
  65. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/bankruptcy/bankruptcy_plots.py +0 -0
  66. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/breakout/__init__.py +0 -0
  67. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/breakout/breakout_plots.py +0 -0
  68. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/corp_risk/__init__.py +0 -0
  69. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/corp_risk/corp_risk_plots.py +0 -0
  70. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/earnings_surprise/__init__.py +0 -0
  71. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/earnings_surprise/earnings_surprise_plots.py +0 -0
  72. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/insider/__init__.py +0 -0
  73. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/insider/insider_plots.py +0 -0
  74. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/institutional/__init__.py +0 -0
  75. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/institutional/institutional_plots.py +0 -0
  76. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/news/__init__.py +0 -0
  77. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/news/news_plots.py +0 -0
  78. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/ratios/__init__.py +0 -0
  79. {sovai-0.2.63 → sovai-0.2.65}/sovai/plots/ratios/ratios_plots.py +0 -0
  80. {sovai-0.2.63 → sovai-0.2.65}/sovai/reports/accounting/__init__.py +0 -0
  81. {sovai-0.2.63 → sovai-0.2.65}/sovai/reports/accounting/accounting_balance_sheet.py +0 -0
  82. {sovai-0.2.63 → sovai-0.2.65}/sovai/reports/bankruptcy/__init__.py +0 -0
  83. {sovai-0.2.63 → sovai-0.2.65}/sovai/reports/bankruptcy/bankruptcy_monthly_top.py +0 -0
  84. {sovai-0.2.63 → sovai-0.2.65}/sovai/reports/general/__init__.py +0 -0
  85. {sovai-0.2.63 → sovai-0.2.65}/sovai/reports/general/general_plots.py +0 -0
  86. {sovai-0.2.63 → sovai-0.2.65}/sovai/reports/news/__inti__.py +0 -0
  87. {sovai-0.2.63 → sovai-0.2.65}/sovai/reports/news/news_econometric_analysis.py +0 -0
  88. {sovai-0.2.63 → sovai-0.2.65}/sovai/studies/nowcasting.py +0 -0
  89. {sovai-0.2.63 → sovai-0.2.65}/sovai/token_auth.py +0 -0
  90. {sovai-0.2.63 → sovai-0.2.65}/sovai/tools/authentication/__init__.py +0 -0
  91. {sovai-0.2.63 → sovai-0.2.65}/sovai/tools/authentication/authentication.py +0 -0
  92. {sovai-0.2.63 → sovai-0.2.65}/sovai/tools/sec/__init__.py +0 -0
  93. {sovai-0.2.63 → sovai-0.2.65}/sovai/tools/sec/graphs.py +0 -0
  94. {sovai-0.2.63 → sovai-0.2.65}/sovai/tools/sec/llm_code_generator.py +0 -0
  95. {sovai-0.2.63 → sovai-0.2.65}/sovai/tools/sec/sec_10_k_8_k_filings.py +0 -0
  96. {sovai-0.2.63 → sovai-0.2.65}/sovai/tools/sec/sec_edgar_search.py +0 -0
  97. {sovai-0.2.63 → sovai-0.2.65}/sovai/utils/__init__.py +0 -0
  98. {sovai-0.2.63 → sovai-0.2.65}/sovai/utils/client_side.py +0 -0
  99. {sovai-0.2.63 → sovai-0.2.65}/sovai/utils/client_side_s3.py +0 -0
  100. {sovai-0.2.63 → sovai-0.2.65}/sovai/utils/converter.py +0 -0
  101. {sovai-0.2.63 → sovai-0.2.65}/sovai/utils/datetime_formats.py +0 -0
  102. {sovai-0.2.63 → sovai-0.2.65}/sovai/utils/file_management.py +0 -0
  103. {sovai-0.2.63 → sovai-0.2.65}/sovai/utils/get_tickers.py +0 -0
  104. {sovai-0.2.63 → sovai-0.2.65}/sovai/utils/helpers.py +0 -0
  105. {sovai-0.2.63 → sovai-0.2.65}/sovai/utils/pandas_extensions copy.py +0 -0
  106. {sovai-0.2.63 → sovai-0.2.65}/sovai/utils/plot.py +0 -0
  107. /sovai-0.2.63/sovai/utils/port_manager.py → /sovai-0.2.65/sovai/utils/port_manager copy.py +0 -0
  108. {sovai-0.2.63 → sovai-0.2.65}/sovai/utils/stream.py +0 -0
  109. {sovai-0.2.63 → sovai-0.2.65}/sovai/utils/verbose_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: sovai
3
- Version: 0.2.63
3
+ Version: 0.2.65
4
4
  Summary: python for asset management
5
5
  License: Proprietary License
6
6
  Author: Derek Snow
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
4
4
 
5
5
  [project]
6
6
  name = "sovai"
7
- version = "0.2.63"
7
+ version = "0.2.65"
8
8
  description = "python for asset management"
9
9
  authors = [
10
10
  {name = "Derek Snow", email = "d.snow@outlook.com"}
Binary file
@@ -61,6 +61,7 @@ def prepare_and_select_neighbors(df_accounting, ticker_select, neighbours):
61
61
  df_calc = df_accounting.loc[
62
62
  df_accounting.index.get_level_values("ticker").isin(neighs)
63
63
  ]
64
+ df_calc = df_calc[~df_calc.index.duplicated(keep="first")] # ← NEW LINE
64
65
  return df_calc
65
66
 
66
67
  else:
@@ -0,0 +1,590 @@
1
+ import numpy as np
2
+ import pandas as pd
3
+ from typing import Tuple, List, Optional # Modified to include List and Optional for new functions
4
+ import plotly.graph_objects as go
5
+ from plotly.subplots import make_subplots
6
+ import dash
7
+ from dash import dcc, html
8
+ from dash.dependencies import Input, Output
9
+ import random
10
+ import socket # Added for port management
11
+ import time # Added for port management
12
+ import errno # Added for port management
13
+
14
+ # from port_manager_utility import get_unique_port, kill_process_on_port, release_port, get_port_manager_instance
15
+ # from utils.port_manager import get_unique_port, kill_process_on_port
16
+ from sovai.utils.port_manager import get_unique_port, kill_process_on_port
17
+
18
+ random.seed(42)
19
+ np.random.seed(42)
20
+
21
+ # Global dictionary to keep track of apps and their ports launched by this script
22
+ app_ports = {}
23
+
24
+ class ImprovedCusumDetector: # Exactly as in the first script
25
+ def __init__(
26
+ self,
27
+ window_size_ratio: float = 0.05,
28
+ threshold_factor: float = 2.5,
29
+ drift_factor: float = 0.05,
30
+ min_distance: int = 30,
31
+ sideways_threshold: float = 0.03,
32
+ warm_up_period_ratio: float = 0.1,
33
+ ):
34
+ self.window_size_ratio = window_size_ratio
35
+ self.threshold_factor = threshold_factor
36
+ self.drift_factor = drift_factor
37
+ self.min_distance = min_distance
38
+ self.sideways_threshold = sideways_threshold
39
+ self.warm_up_period_ratio = warm_up_period_ratio
40
+
41
+ def detect_changepoints(
42
+ self, signal: np.ndarray
43
+ ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, int]:
44
+ n = len(signal)
45
+ window_size = max(int(n * self.window_size_ratio), 8)
46
+ warm_up_period = int(n * self.warm_up_period_ratio)
47
+
48
+ ewma = self._calculate_ewma(signal, window_size)
49
+ z_scores = self._calculate_rolling_z_scores(signal, ewma, window_size)
50
+ scores, changepoints, trend_codes = self._calculate_two_sided_cusum(
51
+ signal,
52
+ z_scores,
53
+ self.drift_factor,
54
+ self.threshold_factor,
55
+ self.min_distance,
56
+ self.sideways_threshold,
57
+ warm_up_period,
58
+ )
59
+
60
+ trends = np.where(
61
+ trend_codes == 1,
62
+ "Increasing",
63
+ np.where(trend_codes == -1, "Decreasing", "Sideways"),
64
+ )
65
+
66
+ return changepoints, trends, scores, window_size
67
+
68
+ import numba
69
+ import numpy as np
70
+
71
+ @staticmethod
72
+ @numba.njit(cache=True)
73
+ def _calculate_ewma(signal, window_size):
74
+ alpha = 2 / (window_size + 1)
75
+ ewma = np.zeros_like(signal)
76
+ ewma[0] = signal[0]
77
+ for i in range(1, len(signal)):
78
+ ewma[i] = alpha * signal[i] + (1 - alpha) * ewma[i - 1]
79
+ return ewma
80
+
81
+ @staticmethod
82
+ @numba.njit(cache=True)
83
+ def _calculate_rolling_z_scores(signal, ewma, window_size):
84
+ rolling_var = np.zeros_like(signal)
85
+ for i in range(window_size, len(signal)):
86
+ rolling_var[i] = np.var(signal[i - window_size + 1 : i + 1])
87
+ rolling_std = np.sqrt(rolling_var)
88
+ z_scores = (signal - ewma) / (rolling_std + 1e-8)
89
+ return z_scores
90
+
91
+ @staticmethod
92
+ @numba.njit(cache=True)
93
+ def _calculate_two_sided_cusum(
94
+ signal,
95
+ z,
96
+ drift,
97
+ threshold_factor,
98
+ min_distance,
99
+ sideways_threshold,
100
+ warm_up_period,
101
+ ):
102
+ cs_upper = np.zeros_like(z)
103
+ cs_lower = np.zeros_like(z)
104
+ changepoints = []
105
+ trend_codes = []
106
+ last_cp = -min_distance
107
+ for i in range(1, len(z)):
108
+ if i < warm_up_period:
109
+ continue
110
+ cs_upper[i] = max(0, cs_upper[i - 1] + z[i] - drift)
111
+ cs_lower[i] = min(0, cs_lower[i - 1] + z[i] + drift)
112
+ if i - last_cp >= min_distance:
113
+ if cs_upper[i] > threshold_factor * np.sqrt(i):
114
+ changepoints.append(i)
115
+ trend_codes.append(1)
116
+ cs_upper[i] = 0
117
+ cs_lower[i] = 0
118
+ last_cp = i
119
+ elif cs_lower[i] < -threshold_factor * np.sqrt(i):
120
+ changepoints.append(i)
121
+ trend_codes.append(-1)
122
+ cs_upper[i] = 0
123
+ cs_lower[i] = 0
124
+ last_cp = i
125
+ elif ( # This condition for sideways might lead to ZeroDivisionError if signal[last_cp] is 0. Preserved as per original.
126
+ i > 0
127
+ and abs((signal[i] - signal[last_cp]) / signal[last_cp])
128
+ < sideways_threshold
129
+ ):
130
+ changepoints.append(i)
131
+ trend_codes.append(0)
132
+ cs_upper[i] = 0
133
+ cs_lower[i] = 0
134
+ last_cp = i
135
+ return (
136
+ np.maximum(cs_upper, -cs_lower),
137
+ np.array(changepoints),
138
+ np.array(trend_codes),
139
+ )
140
+
141
+ def plot_cusum_results( # Exactly as in the first script
142
+ df_signal, signal_array, changepoints, trends, scores, ticker, feature
143
+ ):
144
+ fig = make_subplots(
145
+ rows=2,
146
+ cols=1,
147
+ shared_xaxes=True,
148
+ vertical_spacing=0.1,
149
+ subplot_titles=(f"{feature} Over Time for {ticker}", "CUSUM Score"),
150
+ )
151
+
152
+ fig.add_trace(
153
+ go.Scatter(
154
+ x=df_signal.index,
155
+ y=signal_array,
156
+ mode="lines",
157
+ line=dict(color="gold", width=2),
158
+ name=feature,
159
+ ),
160
+ row=1,
161
+ col=1,
162
+ )
163
+
164
+ fig.add_trace(
165
+ go.Scatter(
166
+ x=df_signal.index,
167
+ y=scores,
168
+ mode="lines",
169
+ line=dict(color="rgba(30, 144, 255, 0.7)", width=1.5),
170
+ name="CUSUM Score",
171
+ ),
172
+ row=2,
173
+ col=1,
174
+ )
175
+
176
+ colors = {"Increasing": "lime", "Decreasing": "red", "Sideways": "yellow"}
177
+ symbols = {
178
+ "Increasing": "triangle-up",
179
+ "Decreasing": "triangle-down",
180
+ "Sideways": "circle",
181
+ }
182
+
183
+ for trend in set(trends):
184
+ mask = trends == trend
185
+ # Original script might have index out of bounds if changepoints[mask] is empty or values are too large. Preserved.
186
+ fig.add_trace(
187
+ go.Scatter(
188
+ x=df_signal.index[changepoints[mask]],
189
+ y=signal_array[changepoints[mask]],
190
+ mode="markers",
191
+ marker=dict(color=colors[trend], size=10, symbol=symbols[trend]),
192
+ name=f"{trend}",
193
+ ),
194
+ row=1,
195
+ col=1,
196
+ )
197
+
198
+ fig.update_layout(
199
+ title=f"{feature} Change Point Detection for {ticker}",
200
+ plot_bgcolor="rgba(0,0,0,0.95)",
201
+ paper_bgcolor="rgba(0,0,0,0.95)",
202
+ font=dict(color="white", size=12),
203
+ autosize=True,
204
+ legend=dict(
205
+ orientation="h",
206
+ yanchor="bottom",
207
+ y=1.1,
208
+ xanchor="right",
209
+ x=1,
210
+ bgcolor="rgba(0,0,0,0.5)",
211
+ bordercolor="rgba(255,255,255,0.5)",
212
+ ),
213
+ showlegend=True,
214
+ )
215
+
216
+ fig.update_xaxes(title_text="Date", row=2, col=1, gridcolor="rgba(255,255,255,0.2)")
217
+ fig.update_yaxes(
218
+ title_text=feature, type="log", row=1, col=1, gridcolor="rgba(255,255,255,0.2)"
219
+ )
220
+ fig.update_yaxes(
221
+ title_text="CUSUM Score", row=2, col=1, gridcolor="rgba(255,255,255,0.2)"
222
+ )
223
+
224
+ for i, (cp, trend) in enumerate(zip(changepoints, trends)):
225
+ # Original script might have index out of bounds if cp is too large. Preserved.
226
+ fig.add_annotation(
227
+ x=df_signal.index[cp],
228
+ y=signal_array[cp],
229
+ text=f"{trend[0]} {i+1}",
230
+ showarrow=False,
231
+ yshift=10 if trend == "Increasing" else -10,
232
+ font=dict(color=colors[trend]),
233
+ row=1,
234
+ col=1,
235
+ )
236
+
237
+ return fig
238
+
239
+
240
+ def run_cusum_dashboard(df_accounting, ticker=None, feature=None): # Modified for new port logic
241
+ if ticker is None:
242
+ ticker_options = df_accounting.index.get_level_values("ticker").unique()
243
+ if not ticker_options.empty:
244
+ ticker = ticker_options[0]
245
+ else:
246
+ print("Error: No tickers found in DataFrame for dashboard.")
247
+ return
248
+ if feature is None:
249
+ if not df_accounting.columns.empty:
250
+ feature = df_accounting.columns[0]
251
+ else:
252
+ print("Error: No features found in DataFrame for dashboard.")
253
+ return
254
+
255
+ app = dash.Dash(__name__)
256
+
257
+ dark_mode_colors = {
258
+ "background": "#1E1E1E",
259
+ "text": "#FFFFFF",
260
+ "plot_bg": "#2B2B2B",
261
+ "plot_gridlines": "#3A3A3A",
262
+ }
263
+
264
+ app.layout = html.Div(
265
+ [
266
+ html.H3(
267
+ "Financial Metric Change Point Detection",
268
+ style={
269
+ "textAlign": "center",
270
+ "marginBottom": "20px",
271
+ "paddingTop": "10px",
272
+ "color": dark_mode_colors["text"],
273
+ },
274
+ ),
275
+ html.Div(
276
+ [
277
+ html.Div(
278
+ [
279
+ html.Label(
280
+ "Select Ticker:",
281
+ style={"color": dark_mode_colors["text"]},
282
+ ),
283
+ dcc.Dropdown(
284
+ id="ticker-dropdown",
285
+ options=[
286
+ {"label": t, "value": t}
287
+ for t in df_accounting.index.get_level_values(
288
+ "ticker"
289
+ ).unique()
290
+ ],
291
+ value=ticker,
292
+ ),
293
+ ],
294
+ style={"width": "48%", "display": "inline-block"},
295
+ ),
296
+ html.Div(
297
+ [
298
+ html.Label(
299
+ "Select Feature:",
300
+ style={"color": dark_mode_colors["text"]},
301
+ ),
302
+ dcc.Dropdown(
303
+ id="feature-dropdown",
304
+ options=[
305
+ {"label": col, "value": col}
306
+ for col in df_accounting.columns
307
+ ],
308
+ value=feature,
309
+ ),
310
+ ],
311
+ style={
312
+ "width": "48%",
313
+ "float": "right",
314
+ "display": "inline-block",
315
+ },
316
+ ),
317
+ ],
318
+ style={"padding": "20px 0px"},
319
+ ),
320
+ dcc.Graph(id="cusum-plot", style={"height": "80vh"}),
321
+ ],
322
+ style={
323
+ "padding": "0 20px",
324
+ "margin": "0 auto",
325
+ "backgroundColor": dark_mode_colors["background"],
326
+ },
327
+ )
328
+
329
+ @app.callback(
330
+ Output("cusum-plot", "figure"),
331
+ [Input("ticker-dropdown", "value"), Input("feature-dropdown", "value")],
332
+ )
333
+ def update_graph(selected_ticker, selected_feature): # Exactly as in the first script's dashboard
334
+ df_signal = (
335
+ df_accounting.query(f"ticker == '{selected_ticker}'")
336
+ .reset_index()
337
+ .set_index("date")[selected_feature]
338
+ )
339
+ signal_array = df_signal.values
340
+ # Add a check for empty signal_array to prevent errors downstream
341
+ if len(signal_array) == 0:
342
+ fig = go.Figure()
343
+ fig.update_layout(
344
+ title_text=f"No data available for {selected_ticker} - {selected_feature}",
345
+ plot_bgcolor=dark_mode_colors["plot_bg"],
346
+ paper_bgcolor=dark_mode_colors["plot_bg"],
347
+ font_color=dark_mode_colors["text"],
348
+ )
349
+ return fig
350
+
351
+ detector = ImprovedCusumDetector()
352
+ changepoints, trends, scores, window_size = detector.detect_changepoints(
353
+ signal_array
354
+ )
355
+ fig = plot_cusum_results(
356
+ df_signal,
357
+ signal_array,
358
+ changepoints,
359
+ trends,
360
+ scores,
361
+ selected_ticker,
362
+ selected_feature,
363
+ )
364
+ fig.update_layout(
365
+ plot_bgcolor=dark_mode_colors["plot_bg"],
366
+ paper_bgcolor=dark_mode_colors["plot_bg"],
367
+ font_color=dark_mode_colors["text"],
368
+ )
369
+ fig.update_xaxes(gridcolor=dark_mode_colors["plot_gridlines"])
370
+ fig.update_yaxes(gridcolor=dark_mode_colors["plot_gridlines"])
371
+ return fig
372
+
373
+ app_name = f"cumsum-app-{ticker}-{feature}".replace(" ","_") # Using more specific app name
374
+
375
+ # --- Robust Server Launch (from the second script) ---
376
+ max_retries = 3
377
+ for attempt in range(max_retries):
378
+ port = get_unique_port(app_name)
379
+ try:
380
+ print(f"Attempting to launch Dash app '{app_name}' on port {port} (Attempt {attempt + 1}/{max_retries})")
381
+ # Using host='0.0.0.0' to make it accessible on the network
382
+ app.run_server(debug=False, port=port, host='0.0.0.0')
383
+ print(f"Dash app '{app_name}' running on http://127.0.0.1:{port}/ or http://<your-ip>:{port}/")
384
+ break # Success
385
+ except socket.error as e:
386
+ if e.errno == errno.EADDRINUSE: # Address already in use
387
+ print(f"Port {port} is in use. Attempting to kill blocking process(es).")
388
+ if kill_process_on_port(port):
389
+ print(f"Successfully killed process(es) on port {port}. Retrying Dash launch...")
390
+ time.sleep(2) # Give OS a moment to free the port
391
+ else:
392
+ print(f"Could not kill process(es) on port {port} or no process found. Trying a different port.")
393
+ # Invalidate this port for this app_name so get_unique_port tries a new one
394
+ if app_name in app_ports and app_ports[app_name] == port:
395
+ del app_ports[app_name]
396
+
397
+ if attempt == max_retries - 1:
398
+ print(f"Failed to launch Dash app '{app_name}' after {max_retries} attempts. Port {port} remains occupied.")
399
+ # Not returning here, as the original run_server call was the return
400
+ # This will effectively mean the function finishes if all retries fail.
401
+ # To match original behavior of returning the server object, this structure would need adjustment
402
+ # However, run_server is blocking, so returning it means the rest of the script doesn't run until server stops.
403
+ # This current structure runs the server and blocks, or prints failure and continues if it can't.
404
+ # The original `return app.run_server(...)` would also block.
405
+ # If it fails to launch due to port, original would raise error immediately.
406
+ # This structure tries to recover or fails after retries.
407
+ # To make it fully equivalent on failure, we should re-raise here.
408
+ raise RuntimeError(f"Failed to launch Dash app '{app_name}' after {max_retries} attempts. Port {port} remains occupied.")
409
+ else:
410
+ print(f"An unexpected socket error occurred: {e}")
411
+ raise # Re-raise other socket errors
412
+ except Exception as e:
413
+ print(f"An unexpected error occurred during app launch: {e}")
414
+ raise
415
+ else: # Executed if the loop completes without break (i.e., all retries failed before exception)
416
+ print(f"CRITICAL: Dash app '{app_name}' could not be started after {max_retries} attempts due to port issues without raising an exception in the loop (should not happen).")
417
+ # This else block for the for loop is unlikely to be hit if EADDRINUSE always occurs and leads to a raise on final attempt.
418
+ # If get_unique_port exhausted options and raised an error, that would be caught by the generic Exception.
419
+
420
+
421
+ def perform_cusum_analysis(df_accounting, ticker=None, feature=None): # Exactly as in the first script
422
+ # Get the first ticker and feature if not provided
423
+ if ticker is None:
424
+ ticker = df_accounting.index.get_level_values("ticker").unique()[0]
425
+ if feature is None:
426
+ feature = df_accounting.columns[0]
427
+
428
+ # Extract the signal for the specified ticker and feature
429
+ df_signal = (
430
+ df_accounting.query(f"ticker == '{ticker}'")
431
+ .reset_index()
432
+ .set_index("date")[feature]
433
+ )
434
+
435
+ detector = ImprovedCusumDetector()
436
+ signal_array = df_signal.values
437
+
438
+ # Added basic check for empty signal to prevent error in detect_changepoints
439
+ if len(signal_array) == 0:
440
+ print(f"Warning: No data for {ticker} - {feature} in perform_cusum_analysis. Returning empty results.")
441
+ empty_analysis_df = pd.DataFrame(columns=["Date", "Trend", "Signal_Value", "CUSUM_Score", "Duration", "Duration_Days", "Percent_Change"])
442
+ empty_stats_df = pd.DataFrame({
443
+ "Statistic": [
444
+ "Ticker", "Feature", "Total_Change_Points", "Average_Duration_Days",
445
+ "Average_Percent_Change", "Increasing_Trends", "Decreasing_Trends",
446
+ "Sideways_Trends", "Max_CUSUM_Score", "Min_CUSUM_Score", "Window_Size"],
447
+ "Value": [ticker, feature, 0, 0, 0, 0, 0, 0, 0, 0, 0]
448
+ })
449
+ empty_analysis_df.attrs["stats"] = empty_stats_df
450
+ return empty_analysis_df
451
+
452
+
453
+ changepoints, trends, scores, window_size = detector.detect_changepoints(
454
+ signal_array
455
+ )
456
+
457
+ # Handle case with no changepoints to avoid errors
458
+ if len(changepoints) == 0:
459
+ analysis_df = pd.DataFrame(columns=["Date", "Trend", "Signal_Value", "CUSUM_Score"])
460
+ else:
461
+ # Original script might have index out of bounds or issues if changepoints are empty or too large. Preserved.
462
+ # CUSUM_Score indexing `changepoints - 1` could be problematic if changepoints contains 0.
463
+ # `_calculate_two_sided_cusum` starts `i` from 1, so changepoints should be >= 1.
464
+ # However, if warm_up_period is very large, changepoints could be empty.
465
+ analysis_df = pd.DataFrame(
466
+ {
467
+ "Date": df_signal.index[changepoints],
468
+ "Trend": trends,
469
+ "Signal_Value": signal_array[changepoints],
470
+ "CUSUM_Score": scores[changepoints - 1],
471
+ }
472
+ )
473
+
474
+ # Calculate duration between change points
475
+ if not analysis_df.empty:
476
+ analysis_df["Duration"] = (
477
+ analysis_df["Date"].diff().shift(-1).fillna(pd.Timedelta(days=0))
478
+ )
479
+ analysis_df["Duration_Days"] = analysis_df["Duration"].dt.days
480
+
481
+ # Calculate percentage change between change points
482
+ analysis_df["Percent_Change"] = (
483
+ analysis_df["Signal_Value"].pct_change().shift(-1) * 100
484
+ ).fillna(0)
485
+ else: # Ensure columns exist if analysis_df is empty
486
+ analysis_df["Duration"] = pd.Series(dtype='timedelta64[ns]')
487
+ analysis_df["Duration_Days"] = pd.Series(dtype='int')
488
+ analysis_df["Percent_Change"] = pd.Series(dtype='float')
489
+
490
+
491
+ # Overall statistics
492
+ # Original `scores[scores > 0].min()` could fail if scores[scores > 0] is empty.
493
+ min_cusum_score_val = 0
494
+ if any(scores > 0):
495
+ min_positive_scores = scores[scores > 0]
496
+ if len(min_positive_scores) > 0:
497
+ min_cusum_score_val = min_positive_scores.min()
498
+
499
+ overall_stats = pd.DataFrame(
500
+ {
501
+ "Statistic": [
502
+ "Ticker",
503
+ "Feature",
504
+ "Total_Change_Points",
505
+ "Average_Duration_Days",
506
+ "Average_Percent_Change",
507
+ "Increasing_Trends",
508
+ "Decreasing_Trends",
509
+ "Sideways_Trends",
510
+ "Max_CUSUM_Score",
511
+ "Min_CUSUM_Score", # Original name was Min_CUSUM_Score
512
+ "Window_Size",
513
+ ],
514
+ "Value": [
515
+ ticker,
516
+ feature,
517
+ len(changepoints),
518
+ analysis_df["Duration_Days"].mean() if not analysis_df.empty else 0,
519
+ analysis_df["Percent_Change"].mean() if not analysis_df.empty else 0,
520
+ sum(trends == "Increasing") if len(trends) > 0 else 0,
521
+ sum(trends == "Decreasing") if len(trends) > 0 else 0,
522
+ sum(trends == "Sideways") if len(trends) > 0 else 0,
523
+ scores.max() if len(scores) > 0 else 0,
524
+ min_cusum_score_val,
525
+ window_size,
526
+ ],
527
+ }
528
+ )
529
+ # Fill NaN for mean values if analysis_df was empty or had single row for stats
530
+ overall_stats['Value'] = overall_stats['Value'].fillna(0)
531
+
532
+ analysis_df.attrs["stats"] = overall_stats
533
+
534
+ return analysis_df
535
+
536
+
537
+ if __name__ == "__main__":
538
+ # Create a dummy df_accounting for demonstration as in the second script
539
+ # This helps in testing the updated script functionality.
540
+ dates = pd.to_datetime(pd.date_range(start="2020-01-01", periods=300, freq="B"))
541
+ data_aapl = np.random.lognormal(mean=0.001, sigma=0.02, size=300).cumprod() * 100
542
+ data_msft = np.random.lognormal(mean=0.0005, sigma=0.025, size=300).cumprod() * 150
543
+
544
+ data_aapl[100:150] *= 1.3
545
+ data_aapl[200:250] *= 0.7
546
+
547
+ data_msft[50:80] *= 0.8
548
+ data_msft[150:200] *= 1.5
549
+ data_msft[250:] += 20
550
+
551
+ df_aapl = pd.DataFrame({"Close": data_aapl, "Volume": np.random.randint(1000, 5000, 300)}, index=dates)
552
+ df_aapl["ticker"] = "AAPL"
553
+
554
+ df_msft = pd.DataFrame({"Close": data_msft, "Volume": np.random.randint(2000, 7000, 300)}, index=dates)
555
+ df_msft["ticker"] = "MSFT"
556
+
557
+ # Ensure df_accounting structure matches what functions expect (MultiIndex with 'ticker', 'date')
558
+ df_accounting_demo = pd.concat([
559
+ df_aapl.reset_index().rename(columns={'index': 'date'}).set_index(['ticker', 'date']),
560
+ df_msft.reset_index().rename(columns={'index': 'date'}).set_index(['ticker', 'date'])
561
+ ])
562
+
563
+ print("---- Running CUSUM Analysis for AAPL Close ----")
564
+ analysis_results_aapl = perform_cusum_analysis(df_accounting_demo, ticker="AAPL", feature="Close")
565
+ if not analysis_results_aapl.empty: # Check if DataFrame is not empty
566
+ print(analysis_results_aapl)
567
+ print("\nOverall Stats (AAPL Close):")
568
+ print(analysis_results_aapl.attrs["stats"])
569
+ else:
570
+ print("No analysis results for AAPL Close.")
571
+
572
+
573
+ print("\n---- Running CUSUM Analysis for MSFT Volume ----")
574
+ analysis_results_msft_vol = perform_cusum_analysis(df_accounting_demo, ticker="MSFT", feature="Volume")
575
+ if not analysis_results_msft_vol.empty: # Check if DataFrame is not empty
576
+ print(analysis_results_msft_vol)
577
+ print("\nOverall Stats (MSFT Volume):")
578
+ print(analysis_results_msft_vol.attrs["stats"])
579
+ else:
580
+ print("No analysis results for MSFT Volume.")
581
+
582
+ print("\n---- Launching Dashboard ----")
583
+ print("Note: If a port is busy, the script will attempt to kill the process.")
584
+ print("Ensure you are comfortable with this, especially if not in a dev environment.")
585
+
586
+ try:
587
+ # df_accounting_demo needs to be passed to the dashboard
588
+ run_cusum_dashboard(df_accounting_demo, ticker="AAPL", feature="Close")
589
+ except Exception as e:
590
+ print(f"Dashboard launch failed: {e}")
@@ -151,6 +151,9 @@ def segment_series(df_accounting, features_select=None):
151
151
 
152
152
 
153
153
  def cluster(df_mega, features_select=None):
154
+
155
+ df_mega = df_mega[~df_mega.index.duplicated(keep="first")] # ← NEW LINE
156
+
154
157
  days = df_mega.index.get_level_values("date").nunique()
155
158
 
156
159
  (
@@ -205,7 +208,7 @@ def cluster(df_mega, features_select=None):
205
208
  output_df["labels"] = output_df["labels"].apply(lambda x: f"Centroid {int(x)}")
206
209
 
207
210
  # Group by 'ticker' and sum across rows
208
- summed_series = output_df.groupby("ticker").sum().sum(axis=1)
211
+ summed_series = output_df.groupby("ticker").sum(numeric_only=True).sum(axis=1)
209
212
 
210
213
  # Find the indices where the value is 0.00
211
214
  zero_indices = summed_series[summed_series == 0.00].index.tolist()
@@ -4,7 +4,7 @@ from sklearn.decomposition import PCA, TruncatedSVD, FactorAnalysis
4
4
  from sklearn.random_projection import GaussianRandomProjection
5
5
  from sklearn.preprocessing import StandardScaler
6
6
  from sklearn.impute import SimpleImputer
7
- # import umap
7
+
8
8
  import warnings
9
9
 
10
10
 
@@ -124,6 +124,7 @@ def dimensionality_reduction(df, method, explained_variance=0.95, n_components=N
124
124
  "truncated_svd": TruncatedSVD,
125
125
  "factor_analysis": FactorAnalysis,
126
126
  "gaussian_random_projection": GaussianRandomProjection,
127
+
127
128
  }
128
129
 
129
130
  if n_components is not None: