chowclassifier 1.0.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- chowclassifier/__init__.py +6 -0
- chowclassifier/__main__.py +81 -0
- chowclassifier/chewdata.py +382 -0
- chowclassifier/chow.py +552 -0
- chowclassifier/utilities.py +121 -0
- chowclassifier-1.0.10.dist-info/METADATA +181 -0
- chowclassifier-1.0.10.dist-info/RECORD +11 -0
- chowclassifier-1.0.10.dist-info/WHEEL +5 -0
- chowclassifier-1.0.10.dist-info/entry_points.txt +2 -0
- chowclassifier-1.0.10.dist-info/licenses/LICENSE +117 -0
- chowclassifier-1.0.10.dist-info/top_level.txt +1 -0
chowclassifier/chow.py
ADDED
|
@@ -0,0 +1,552 @@
|
|
|
1
|
+
"""Implementation of a Chow Classifier"""
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pandas as pd
|
|
5
|
+
from scipy.stats import f as Fdist
|
|
6
|
+
import matplotlib.pyplot as plt
|
|
7
|
+
import seaborn as sns
|
|
8
|
+
import statsmodels.api as sm
|
|
9
|
+
from typing import Union
|
|
10
|
+
from .utilities import conf_int_to_trend, parse_cmap
|
|
11
|
+
from .utilities import SCATTERPARAMS, FILLPARAMS, FITPARAMS, CI_COLORS, TREND_COLORS
|
|
12
|
+
|
|
13
|
+
### Set plot style
|
|
14
|
+
sns.set_style('white')
|
|
15
|
+
plt.rcParams.update({'font.family':'serif',
|
|
16
|
+
'font.size':14,
|
|
17
|
+
"xtick.bottom" : True,
|
|
18
|
+
"ytick.left" : True})
|
|
19
|
+
plt.rcParams['font.serif'] = ['EB Garamond', 'Garamond','Times New Roman','Times']
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Chow(object):
|
|
23
|
+
"""
|
|
24
|
+
Class handling the chow analysis
|
|
25
|
+
Call `run()` to run the analysis (find breakpoint and classify)
|
|
26
|
+
Call afterwards `params()` to get the regressions (y=k * x + b) parameters as
|
|
27
|
+
[[model1_k, model1_b, model1_Rsquared],[model2_k, model2_b, model2_Rsquared]]
|
|
28
|
+
(if the breakpoint is not significant, model1 = model2)
|
|
29
|
+
The significance level of the breakpoint is adjusted with Bonferroni correction,
|
|
30
|
+
i.e. dividing alpha by the number of tests. The confidence interval of the slope
|
|
31
|
+
and intercept are at significance level alpha (not corrected).
|
|
32
|
+
|
|
33
|
+
:param df: data on which to perform the Chow categorisation
|
|
34
|
+
:param name: name of the dataset (optional)
|
|
35
|
+
:param initial_breakpoint: breakpoint (if None, midpoint of X series will be used)
|
|
36
|
+
:param timecol: name of time column (X)
|
|
37
|
+
:param ycol: name of the variable column (y)
|
|
38
|
+
:param groupcol: name of the group column (g) if not in df will be ignored
|
|
39
|
+
:param alpha: significance level (without Bonferroni correction), can be changed with ``Chow.set_alpha(0.01)``
|
|
40
|
+
:param margin: range of indexes around breakpoint (or midpoint) where the best breakpoint
|
|
41
|
+
will be searched. E.g. if X = [0,1,1.5,2,3,4,5], breakpoint = 2 and margin = 1,
|
|
42
|
+
the set of breakpoints [1.5,2,3] will be tested.
|
|
43
|
+
(set = 0 if you want to look only at the given breakpoint or the midpoint)
|
|
44
|
+
"""
|
|
45
|
+
def __init__(self, df,
|
|
46
|
+
name:str = "",
|
|
47
|
+
initial_breakpoint:float = None,
|
|
48
|
+
timecol:str = 'year',
|
|
49
|
+
ycol:str = 'value',
|
|
50
|
+
groupcol:str = 'g',
|
|
51
|
+
alpha = 0.01,
|
|
52
|
+
margin:int = 2):
|
|
53
|
+
"""
|
|
54
|
+
Initialize a Chow classifier object.
|
|
55
|
+
|
|
56
|
+
:param df: DataFrame containing the data to analyze
|
|
57
|
+
:param name: Name identifier for this dataset
|
|
58
|
+
:param initial_breakpoint: Initial breakpoint value (if None, uses midpoint)
|
|
59
|
+
:param timecol: Name of the time column in df
|
|
60
|
+
:param ycol: Name of the dependent variable column in df
|
|
61
|
+
:param groupcol: Name of the grouping column in df (optional)
|
|
62
|
+
:param alpha: Significance level for statistical tests
|
|
63
|
+
:param margin: Range around breakpoint to search for optimal break
|
|
64
|
+
"""
|
|
65
|
+
try:
|
|
66
|
+
self.df = df.loc[:,[timecol,groupcol, ycol]]
|
|
67
|
+
self.df.columns = ('X','g','y')
|
|
68
|
+
except KeyError:
|
|
69
|
+
self.df = df.loc[:,[timecol,ycol]]
|
|
70
|
+
self.df.columns = ('X','y')
|
|
71
|
+
self.df['g'] = 'a'
|
|
72
|
+
self.name = name
|
|
73
|
+
self.df['X'] = pd.to_numeric(self.df['X'])
|
|
74
|
+
self.df['y'] = pd.to_numeric(self.df['y'])
|
|
75
|
+
self.initial_breakpoint = initial_breakpoint
|
|
76
|
+
self.margin=margin
|
|
77
|
+
self.breakpoints = None
|
|
78
|
+
self.normalise=False
|
|
79
|
+
self.tt = -1
|
|
80
|
+
self.m = margin*2+1 # number of hypotheses tested, will be used for Bonferroni correction
|
|
81
|
+
self.F=None
|
|
82
|
+
self.alpha0 = alpha # original significance level
|
|
83
|
+
self.alpha = None # significance level with Bonferroni correction
|
|
84
|
+
self.model0=None
|
|
85
|
+
self.model1=None
|
|
86
|
+
self.model2=None
|
|
87
|
+
self.gCs = {}# sub analysis with grouping on 'g'
|
|
88
|
+
self.successful=None
|
|
89
|
+
self.color = "#000000"
|
|
90
|
+
def get_breakpoint_indices(self):
|
|
91
|
+
"""
|
|
92
|
+
Calculate and set the breakpoint indices to search.
|
|
93
|
+
|
|
94
|
+
:return: List of breakpoint indices to test
|
|
95
|
+
"""
|
|
96
|
+
X = sorted(set(self.df['X']))
|
|
97
|
+
if self.breakpoints is not None:
|
|
98
|
+
self.breakpoints = sorted(set(self.breakpoints).intersection(set(X)))
|
|
99
|
+
breakpoint_indices = range(0,len(self.breakpoints))
|
|
100
|
+
else:
|
|
101
|
+
breakpoint_indices = range(0,len(X))
|
|
102
|
+
self.breakpoints = X
|
|
103
|
+
if self.initial_breakpoint is None:
|
|
104
|
+
self.initial_breakpoint = self.breakpoints[int(len(self.breakpoints)/2)]
|
|
105
|
+
if self.initial_breakpoint not in self.breakpoints:
|
|
106
|
+
s = min(range(len(X)), key=lambda i: abs(X[i]-self.initial_breakpoint))
|
|
107
|
+
self.initial_breakpoint = X[s]
|
|
108
|
+
breakpoint_index = self.breakpoints.index(self.initial_breakpoint)
|
|
109
|
+
if breakpoint_index < 3:
|
|
110
|
+
breakpoint_index = 3
|
|
111
|
+
self.initial_breakpoint = self.breakpoints[breakpoint_index]
|
|
112
|
+
print(f"breakpoint index for {self.name} is below 3, will be replace by 3.\n The new breakpoint is {self.initial_breakpoint}")
|
|
113
|
+
elif breakpoint_index > len(X)-4:
|
|
114
|
+
breakpoint_index = len(X)-4
|
|
115
|
+
self.initial_breakpoint = self.breakpoints[breakpoint_index]
|
|
116
|
+
print(f"breakpoint index for {self.name} is above {len(X)-4}, will be replace by {len(X)-4}.\n The new breakpoint is {self.initial_breakpoint}")
|
|
117
|
+
breakpoint_indices = list(range(breakpoint_index - self.margin, breakpoint_index+self.margin))
|
|
118
|
+
return breakpoint_indices
|
|
119
|
+
def find_best_bkp(self, breakpoint_indices=None,**kwargs):
|
|
120
|
+
"""
|
|
121
|
+
Find the best breakpoint and run Chow test to find OLS parameters.
|
|
122
|
+
|
|
123
|
+
Tests multiple breakpoints and selects the one with the best score.
|
|
124
|
+
Updates self.initial_breakpoint with the optimal value.
|
|
125
|
+
|
|
126
|
+
:param breakpoint_indices: Optional list of indices to test. If None, uses get_breakpoint_indices()
|
|
127
|
+
:param kwargs: Additional keyword arguments (currently unused)
|
|
128
|
+
"""
|
|
129
|
+
if breakpoint_indices is not None:
|
|
130
|
+
if not isinstance(breakpoint_indices, list):
|
|
131
|
+
breakpoint_indices=[breakpoint_indices]
|
|
132
|
+
midpoint_index = int(np.median(breakpoint_indices))
|
|
133
|
+
else:
|
|
134
|
+
breakpoint_indices = self.get_breakpoint_indices()
|
|
135
|
+
midpoint_index = breakpoint_indices[int(len(breakpoint_indices)/2)]
|
|
136
|
+
results = {}
|
|
137
|
+
for bk in breakpoint_indices:
|
|
138
|
+
results[bk]=self.run_chow(bkp=self.breakpoints[bk])
|
|
139
|
+
self.results=pd.DataFrame(results)
|
|
140
|
+
self.results = self.results.transpose()
|
|
141
|
+
self.results['delay'] = np.abs(midpoint_index - self.results.index)
|
|
142
|
+
self.results = self.results.sort_values('score')
|
|
143
|
+
#find best breakpoint
|
|
144
|
+
self.initial_breakpoint = self.breakpoints[self.results.score.idxmin()]
|
|
145
|
+
self.m = len(breakpoint_indices)
|
|
146
|
+
self.set_alpha(self.alpha0)
|
|
147
|
+
def run_chow(self, bkp, normalise = False):
|
|
148
|
+
"""
|
|
149
|
+
Run Chow test at specified breakpoint.
|
|
150
|
+
|
|
151
|
+
Fits OLS models to the full dataset and two subsets (before and after breakpoint),
|
|
152
|
+
then calculates the F-statistic to test for structural break.
|
|
153
|
+
|
|
154
|
+
:param bkp: Breakpoint value at which to split the data
|
|
155
|
+
:param normalise: Whether to normalize the y values before fitting
|
|
156
|
+
:return: Dictionary containing score, F-statistic, and residual sum of squares
|
|
157
|
+
"""
|
|
158
|
+
if self.normalise:
|
|
159
|
+
self.df['y']=(self.df['y']-self.df['y'].mean())/self.df['y'].std()
|
|
160
|
+
X=self.df.loc[:,'X']
|
|
161
|
+
y=self.df.loc[:,'y']
|
|
162
|
+
X1=self.df.loc[self.df['X']<bkp,'X']
|
|
163
|
+
X2=self.df.loc[self.df['X']>=bkp,'X']
|
|
164
|
+
y1=self.df.loc[self.df['X']<bkp,'y']
|
|
165
|
+
y2=self.df.loc[self.df['X']>=bkp,'y']
|
|
166
|
+
self.successful = False
|
|
167
|
+
if np.any(X1) and np.any(X2):
|
|
168
|
+
X1=sm.add_constant(X1)
|
|
169
|
+
X2=sm.add_constant(X2)
|
|
170
|
+
X=sm.add_constant(X)
|
|
171
|
+
self.model0=sm.OLS(y,X).fit()
|
|
172
|
+
self.model1=sm.OLS(y1,X1).fit()
|
|
173
|
+
self.model2=sm.OLS(y2,X2).fit()
|
|
174
|
+
self.ss_res_0=np.sum((y-self.model0.predict(X))**2)
|
|
175
|
+
self.ss_res_1=np.sum((y1-self.model1.predict(X1))**2)
|
|
176
|
+
self.ss_res_2=np.sum((y2-self.model2.predict(X2))**2)
|
|
177
|
+
k = 2 # Total number of parameters
|
|
178
|
+
self.F=((self.ss_res_0-(self.ss_res_1+self.ss_res_2))/k) / ((self.ss_res_1+self.ss_res_2)/(X.shape[0]-2*k))
|
|
179
|
+
if X.shape[0]-2*k >0:
|
|
180
|
+
self.score=Fdist.sf(self.F, 2, X.shape[0]-2*k)
|
|
181
|
+
self.successful=True
|
|
182
|
+
if self.successful is False and np.any(X) and X.shape[0]>1:
|
|
183
|
+
X=sm.add_constant(X)
|
|
184
|
+
self.model0=sm.OLS(y,X).fit()
|
|
185
|
+
self.model1 = self.model0
|
|
186
|
+
self.model2 = self.model0
|
|
187
|
+
self.ss_res_0=np.sum((y-self.model0.predict(X))**2)
|
|
188
|
+
self.ss_res_1 = None
|
|
189
|
+
self.ss_res_2 = None
|
|
190
|
+
self.F = None
|
|
191
|
+
self.score = 1
|
|
192
|
+
#self.successful = False
|
|
193
|
+
elif self.successful is False:
|
|
194
|
+
raise ValueError("Data missing for at least one period for {}, with breakpoint {}".format(self.name,self.initial_breakpoint))
|
|
195
|
+
return {"score" : self.score,
|
|
196
|
+
"F" : self.F,
|
|
197
|
+
"ss_res_0" : self.ss_res_0,
|
|
198
|
+
"ss_res_1" : self.ss_res_1,
|
|
199
|
+
"ss_res_2" : self.ss_res_2,}
|
|
200
|
+
def classify(self,**kwargs):
|
|
201
|
+
"""
|
|
202
|
+
Classify the dataset based on results of Chow test at significance level alpha.
|
|
203
|
+
|
|
204
|
+
Possible classification values:
|
|
205
|
+
|
|
206
|
+
**No significant breakpoint:**
|
|
207
|
+
|
|
208
|
+
1. N: non-significant overall trend
|
|
209
|
+
2. I: significant increasing overall trend
|
|
210
|
+
3. D: significant decreasing overall trend
|
|
211
|
+
|
|
212
|
+
**Significant breakpoint** (set1 and set2 indicate points before/after breakpoint):
|
|
213
|
+
|
|
214
|
+
4. NN: non-significant trend on set1 and non-significant trend on set2
|
|
215
|
+
5. NI: non-significant trend on set1 and significant increasing trend on set2
|
|
216
|
+
6. ND: non-significant trend on set1 and significant decreasing trend on set2
|
|
217
|
+
7. IN: significant increasing trend on set1 and non-significant trend on set2
|
|
218
|
+
8. ID: significant increasing trend on set1 and significant decreasing trend on set2
|
|
219
|
+
9. iI: significant increasing trend on both set1 and set2 with greater increase in set2
|
|
220
|
+
10. Ii: significant increasing trend on both set1 and set2 with greater increase in set1
|
|
221
|
+
11. DN: significant decreasing trend on set1 and non-significant trend on set2
|
|
222
|
+
12. DI: significant decreasing trend on set1 and significant increasing trend on set2
|
|
223
|
+
13. dD: significant decreasing trend on both set1 and set2 with greater decrease in set2
|
|
224
|
+
14. Dd: significant decreasing trend on both set1 and set2 with greater decrease in set1
|
|
225
|
+
|
|
226
|
+
:param kwargs: Additional keyword arguments (currently unused)
|
|
227
|
+
:return: Dictionary with model parameters
|
|
228
|
+
"""
|
|
229
|
+
if self.alpha is None:
|
|
230
|
+
self.set_alpha(self.alpha0)
|
|
231
|
+
if self.score < self.alpha:#
|
|
232
|
+
tt1= conf_int_to_trend(self.model1.conf_int(alpha = self.alpha0))
|
|
233
|
+
tt2= conf_int_to_trend(self.model2.conf_int(alpha = self.alpha0))
|
|
234
|
+
# correction for insignificant trend on one period when actually
|
|
235
|
+
# the trend is significantly higher or lower than the next/previous period
|
|
236
|
+
# at the initial_breakpoint year (i.e. confidence intervals are disjoints at that point)
|
|
237
|
+
if (tt1==0 or tt2==0) and self.initial_breakpoint:
|
|
238
|
+
lci1, uci1 = self.model1.conf_int(alpha = self.alpha0).loc['X',:]
|
|
239
|
+
lci2, uci2 = self.model2.conf_int(alpha = self.alpha0).loc['X',:]
|
|
240
|
+
b1, a1 = self.model1.params
|
|
241
|
+
b2, a2 = self.model2.params
|
|
242
|
+
right_limit = b1 + self.initial_breakpoint * a1 #right limit of first period
|
|
243
|
+
left_limit = b2 + self.initial_breakpoint * a2#left limit of second period
|
|
244
|
+
if right_limit >= left_limit:
|
|
245
|
+
if b2 + lci2 * self.initial_breakpoint > b1 + uci1 * self.initial_breakpoint:
|
|
246
|
+
if tt1==0:
|
|
247
|
+
tt1=1 # the trend is in fact increasing
|
|
248
|
+
if tt2==0:
|
|
249
|
+
tt2=2 # the trend is in fact decreasing
|
|
250
|
+
else:
|
|
251
|
+
if b2 + uci2 * self.initial_breakpoint < b1 + lci1 * self.initial_breakpoint:
|
|
252
|
+
if tt1==0:
|
|
253
|
+
tt1=2 # the trend is in fact decreasing
|
|
254
|
+
if tt2==0:
|
|
255
|
+
tt2=1 # the trend is in fact increasing
|
|
256
|
+
if tt1==0:
|
|
257
|
+
tc=['NN','NI','ND'][tt2]
|
|
258
|
+
elif tt1==1:
|
|
259
|
+
if tt2==1:
|
|
260
|
+
tc = ['iI','Ii'][1*(self.model1.params['X']>self.model2.params['X'])]
|
|
261
|
+
else:
|
|
262
|
+
tc=['IN','','ID'][tt2]
|
|
263
|
+
elif tt1==2:
|
|
264
|
+
if tt2==2:
|
|
265
|
+
tc = ['dD','Dd'][1*(self.model1.params['X']<self.model2.params['X'])]
|
|
266
|
+
else:
|
|
267
|
+
tc=['DN','DI'][tt2]
|
|
268
|
+
else:
|
|
269
|
+
#N0, N1 or N2
|
|
270
|
+
self.tt= conf_int_to_trend(self.model0.conf_int(alpha = self.alpha0))
|
|
271
|
+
tc = ['N','I','D'][self.tt]
|
|
272
|
+
self.initial_breakpoint=None
|
|
273
|
+
self.treeclass = tc
|
|
274
|
+
self.set_color()
|
|
275
|
+
return self.params()
|
|
276
|
+
def set_color(self):
|
|
277
|
+
"""
|
|
278
|
+
Set the color attribute based on the trend type (tt).
|
|
279
|
+
|
|
280
|
+
Uses predefined color scheme from TREND_COLORS.
|
|
281
|
+
"""
|
|
282
|
+
colors = {'Ii':'#8c510a','iI':'#bf812d','IN':'#dfc27d','NI':'#f6e8c3',
|
|
283
|
+
'Dd':'#c7eae5','dD':'#80cdc1','DN':'#35978f','ND':'#01665e',
|
|
284
|
+
'NN':'#555555','NI':'#006837','ND':'#d73027','N':'#AAAAAA','I':'#1a9850','D':'#a50026',
|
|
285
|
+
'ID':'#9970ab','DI':'#762a83'}
|
|
286
|
+
self.color = TREND_COLORS[self.tt]
|
|
287
|
+
def params_names(self):
|
|
288
|
+
"""
|
|
289
|
+
Get list of parameter names for model output.
|
|
290
|
+
|
|
291
|
+
:return: List of parameter name strings
|
|
292
|
+
"""
|
|
293
|
+
return ['intercept','slope',
|
|
294
|
+
'R2','se_intercept','se_slope','n',
|
|
295
|
+
'intercept_pvalue', 'slope_pvalue',
|
|
296
|
+
'intercept_ci_lower','intercept_ci_upper',
|
|
297
|
+
'slope_ci_lower','slope_ci_upper',
|
|
298
|
+
'alpha','alpha_Bonferroni_corrected']
|
|
299
|
+
def params(self):
|
|
300
|
+
"""
|
|
301
|
+
Return parameters of the fitted model(s).
|
|
302
|
+
|
|
303
|
+
:return: Dictionary containing model0, model1, and model2 parameters.
|
|
304
|
+
If breakpoint is significant, model0 is empty and model1/model2 are populated.
|
|
305
|
+
Otherwise, model0 is populated and model1/model2 are empty.
|
|
306
|
+
"""
|
|
307
|
+
def get_params(model):
|
|
308
|
+
d = [*[i for i in model.params],\
|
|
309
|
+
model.rsquared,\
|
|
310
|
+
*[i for i in model.bse],\
|
|
311
|
+
self.df.shape[0],\
|
|
312
|
+
*[i for i in model.pvalues],\
|
|
313
|
+
*[i for i in model.conf_int(alpha = self.alpha0).loc['const',:]],\
|
|
314
|
+
*[i for i in model.conf_int(alpha = self.alpha0).loc['X',:]],\
|
|
315
|
+
self.alpha0,\
|
|
316
|
+
self.alpha]
|
|
317
|
+
return {k:v for v,k in zip(d,self.params_names())}
|
|
318
|
+
if self.score < self.alpha:
|
|
319
|
+
return {'model0':{},
|
|
320
|
+
'model1':get_params(self.model1),
|
|
321
|
+
'model2':get_params(self.model2)}
|
|
322
|
+
else:
|
|
323
|
+
return {'model0':get_params(self.model0),
|
|
324
|
+
'model1':{},
|
|
325
|
+
'model2':{}}
|
|
326
|
+
def run(self, **kwargs):
|
|
327
|
+
"""
|
|
328
|
+
Run the complete Chow classification analysis.
|
|
329
|
+
|
|
330
|
+
Finds best breakpoint, runs Chow test, and classifies the trend.
|
|
331
|
+
|
|
332
|
+
:param kwargs: Keyword arguments forwarded to find_best_bkp and classify.
|
|
333
|
+
Can include 'normalise' for run_chow, 'alpha' for classify.
|
|
334
|
+
:return: Classification code (string) or None if analysis fails
|
|
335
|
+
"""
|
|
336
|
+
if np.any(self.df):
|
|
337
|
+
try:
|
|
338
|
+
self.find_best_bkp(**kwargs)
|
|
339
|
+
except IndexError:
|
|
340
|
+
print(f"No suitable breakpoint found for {self.name}.")
|
|
341
|
+
self.run_chow(bkp = self.initial_breakpoint, normalise = kwargs.get('normalise',False))
|
|
342
|
+
self.classify(**kwargs)
|
|
343
|
+
return self.treeclass
|
|
344
|
+
else:
|
|
345
|
+
self.successful=False
|
|
346
|
+
return None
|
|
347
|
+
def set_alpha(self, alpha:float):
|
|
348
|
+
"""
|
|
349
|
+
Change the current statistical significance level.
|
|
350
|
+
|
|
351
|
+
The Bonferroni correction will be applied based on the number of tests.
|
|
352
|
+
|
|
353
|
+
:param alpha: Significance level, positive float in (0,1)
|
|
354
|
+
"""
|
|
355
|
+
self.alpha0 = alpha
|
|
356
|
+
self.alpha = self.alpha0 / max(self.m,1) # applying bonferroni correction
|
|
357
|
+
|
|
358
|
+
def summary(self):
|
|
359
|
+
"""
|
|
360
|
+
Print the statistical summary of the fitted model(s).
|
|
361
|
+
|
|
362
|
+
If breakpoint is significant, prints summaries for both model1 and model2.
|
|
363
|
+
Otherwise, prints summary for model0 (whole dataset).
|
|
364
|
+
"""
|
|
365
|
+
if self.score < self.alpha:
|
|
366
|
+
print(f"Summary model 1: start-{self.initial_breakpoint}")
|
|
367
|
+
self.model1.summary()
|
|
368
|
+
print(f"Summary model 2: {self.initial_breakpoint}-end")
|
|
369
|
+
self.model2.summary()
|
|
370
|
+
else:
|
|
371
|
+
print(f"Summary model 0: whole dataset")
|
|
372
|
+
self.model0.summary()
|
|
373
|
+
def plot(self, filename:str = None, ax = None, figsize = (16,8),
|
|
374
|
+
ylog:bool=False,
|
|
375
|
+
fill:bool = True,
|
|
376
|
+
scatter:bool = True,
|
|
377
|
+
scatterparams:dict = {},
|
|
378
|
+
fillparams:dict = {},
|
|
379
|
+
fitparams:dict = {},
|
|
380
|
+
linestyles:list = ['dashed','solid','solid'],
|
|
381
|
+
show_legend:bool = True,
|
|
382
|
+
**params):
|
|
383
|
+
"""
|
|
384
|
+
Plot the regression model(s).
|
|
385
|
+
|
|
386
|
+
:param filename: Filename (with extension format), if None, will call plt.show() instead
|
|
387
|
+
:param ax: Matplotlib axis on which to plot, if None will create new figure
|
|
388
|
+
:param figsize: Tuple with figure size
|
|
389
|
+
:param ylog: Make the y-axis log-scale
|
|
390
|
+
:param fill: Plot the confidence interval
|
|
391
|
+
:param scatter: Plot individual points
|
|
392
|
+
:param scatterparams: Parameters for the scatter plot
|
|
393
|
+
:param fillparams: Parameters for the confidence intervals
|
|
394
|
+
:param fitparams: Parameters for the fitted line
|
|
395
|
+
:param linestyles: Select the linestyle of each trend (list of size 3), set to None if you want to define it in fitparams
|
|
396
|
+
:param show_legend: Legend options passed to ax.legend()
|
|
397
|
+
:param params: Additional parameters including color, xlabel, ylabel, title, xlim, ylim
|
|
398
|
+
"""
|
|
399
|
+
### update plot parameters
|
|
400
|
+
SCATTERPARAMS.update(scatterparams)
|
|
401
|
+
FILLPARAMS.update(fillparams)
|
|
402
|
+
FITPARAMS.update(fitparams)
|
|
403
|
+
# record if it is plotting for parent process
|
|
404
|
+
main_p = ax is None
|
|
405
|
+
# create the plot:
|
|
406
|
+
self.set_color()
|
|
407
|
+
def add_reg_plot(sub_df, model):
|
|
408
|
+
# X values
|
|
409
|
+
X = sorted(set(sub_df['X']))
|
|
410
|
+
inc = (max(X)-min(X))/max(1,len(X)-1)
|
|
411
|
+
X = np.linspace(min(X)-0.5*inc, max(X)+0.5*inc, 2*(len(X)+1))
|
|
412
|
+
Xc = sm.add_constant(X)
|
|
413
|
+
# get confidence interval
|
|
414
|
+
ci_l,ci_u = zip(*model.get_prediction(Xc).conf_int(alpha = self.alpha))
|
|
415
|
+
tt = conf_int_to_trend(model.conf_int(alpha = self.alpha))
|
|
416
|
+
if fill is True:
|
|
417
|
+
# set color depending on time trend
|
|
418
|
+
FILLPARAMS['color'] = fillparams.get('color',CI_COLORS[tt])
|
|
419
|
+
ax.fill_between(x = X, y1=ci_l, y2 = ci_u, **FILLPARAMS)
|
|
420
|
+
FITPARAMS['color'] = fitparams.get('color',TREND_COLORS[tt])
|
|
421
|
+
try:
|
|
422
|
+
FITPARAMS['linestyle'] = linestyles[tt]
|
|
423
|
+
except:
|
|
424
|
+
FITPARAMS['linestyle'] = FITPARAMS.get('linestyle','solid')
|
|
425
|
+
intercept, slope = model.params
|
|
426
|
+
y0 = intercept + slope * min(X)
|
|
427
|
+
y1 = intercept + slope * max(X)
|
|
428
|
+
ax.plot((min(X),max(X)),(y0,y1), marker = None, **FITPARAMS)
|
|
429
|
+
if ax is None:
|
|
430
|
+
plt.clf()
|
|
431
|
+
f, (ax) = plt.subplots(nrows=1, ncols=1, figsize=figsize)
|
|
432
|
+
if self.score < self.alpha:
|
|
433
|
+
df1 = self.df.loc[self.df['X'] < self.initial_breakpoint,]
|
|
434
|
+
df2 = self.df.loc[self.df['X'] >= self.initial_breakpoint,]
|
|
435
|
+
add_reg_plot(df1,self.model1)
|
|
436
|
+
add_reg_plot(df2,self.model2)
|
|
437
|
+
else:
|
|
438
|
+
add_reg_plot(self.df,self.model0)
|
|
439
|
+
if scatter is True:
|
|
440
|
+
SCATTERPARAMS['color'] = scatterparams.get('color',self.color)
|
|
441
|
+
ax.scatter(self.df['X'],self.df['y'],
|
|
442
|
+
**SCATTERPARAMS)
|
|
443
|
+
# Tweak the visual presentation
|
|
444
|
+
ax.xaxis.grid(True)
|
|
445
|
+
ax.yaxis.grid(True)
|
|
446
|
+
ax.minorticks_on()
|
|
447
|
+
ax.set(ylabel=params.get("ylabel", 'y'),
|
|
448
|
+
xlabel=params.get("xlabel",'X'),
|
|
449
|
+
title=params.get("title",self.name))
|
|
450
|
+
ax.set_xlim(*params.get('xlim',(self.df['X'].min(), self.df['X'].max())))
|
|
451
|
+
if params.get('ylim') is not None:
|
|
452
|
+
ax.set_ylim(*params.get('ylim',(self.df['y'].min(), self.df['y'].max())))
|
|
453
|
+
if ylog is True:
|
|
454
|
+
ax.set_yscale('log')
|
|
455
|
+
if show_legend is False:
|
|
456
|
+
ax.get_legend().remove()
|
|
457
|
+
#sns.despine(trim=True, left=True)
|
|
458
|
+
#f.suptitle("")
|
|
459
|
+
if filename is not None and main_p is True:
|
|
460
|
+
plt.savefig(f'{filename}')
|
|
461
|
+
elif main_p is True:
|
|
462
|
+
plt.show()
|
|
463
|
+
def plot_by_group(self,
|
|
464
|
+
filename:str = None,
|
|
465
|
+
ax = None,
|
|
466
|
+
figsize = (16,8),
|
|
467
|
+
cmap:str = 'Set1',
|
|
468
|
+
show_legend:bool=True,
|
|
469
|
+
plot_overall:bool=True,
|
|
470
|
+
plot_individual_fill:bool = True,
|
|
471
|
+
scatterparams:dict = {},
|
|
472
|
+
fillparams:dict = {},
|
|
473
|
+
fitparams:dict = {},
|
|
474
|
+
groups_order:list = None,
|
|
475
|
+
**params):
|
|
476
|
+
"""
|
|
477
|
+
Plot the regression model(s) grouped by the group column.
|
|
478
|
+
|
|
479
|
+
:param filename: Filename (with extension format), if None, will call plt.show() instead
|
|
480
|
+
:param ax: Matplotlib axis on which to plot, if None will create new figure
|
|
481
|
+
:param figsize: Tuple with figure size
|
|
482
|
+
:param cmap: Colormap name (from matplotlib) or custom color mapping
|
|
483
|
+
:param show_legend: Include the legend
|
|
484
|
+
:param plot_overall: Plot overall trend and confidence interval
|
|
485
|
+
:param plot_individual_fill: Plot confidence interval for each individual group
|
|
486
|
+
:param scatterparams: Parameters for the scatter plot
|
|
487
|
+
:param fillparams: Parameters for the confidence intervals
|
|
488
|
+
:param fitparams: Parameters for the fitted line
|
|
489
|
+
:param groups_order: List showing the order in which the groups must be plotted
|
|
490
|
+
:param params: Additional parameters including xlabel, ylabel, title, ylog, scatter
|
|
491
|
+
"""
|
|
492
|
+
cmap = parse_cmap(cmap)
|
|
493
|
+
scatterparams = params.get('scatterparams',{})
|
|
494
|
+
fillparams = params.get('fillparams',{})
|
|
495
|
+
fitparams = params.get('fitparams',{})
|
|
496
|
+
params['xlabel'] = params.get('xlabel','X')
|
|
497
|
+
params['ylabel'] = params.get('ylabel','y')
|
|
498
|
+
# record if it is plotting for parent process
|
|
499
|
+
main_p = ax is None
|
|
500
|
+
if ax is None:
|
|
501
|
+
plt.clf()
|
|
502
|
+
f, (ax) = plt.subplots(nrows=1,
|
|
503
|
+
ncols=1,
|
|
504
|
+
sharey=params.get("sharey",True),
|
|
505
|
+
figsize=params.get('figsize',figsize))
|
|
506
|
+
if groups_order is None:
|
|
507
|
+
groups_order = []
|
|
508
|
+
params['title'] = self.name
|
|
509
|
+
groups_order = groups_order + [x for x in set(self.df['g']) if x not in groups_order]
|
|
510
|
+
self.df['go'] = [groups_order.index(x) for x in self.df['g']]
|
|
511
|
+
for i,(n,dfg) in enumerate(self.df.groupby(['go','g'],sort=True)):
|
|
512
|
+
n = n[1]
|
|
513
|
+
if n not in self.gCs.keys():
|
|
514
|
+
self.gCs[n] = Chow(df = dfg,
|
|
515
|
+
name = n,
|
|
516
|
+
initial_breakpoint = self.initial_breakpoint,
|
|
517
|
+
timecol = 'X',
|
|
518
|
+
ycol = 'y',
|
|
519
|
+
groupcol = 'g',
|
|
520
|
+
alpha = self.alpha,
|
|
521
|
+
margin = self.margin)
|
|
522
|
+
success = self.gCs[n].run()
|
|
523
|
+
if success is None:
|
|
524
|
+
print(f"Analysis failed for {self.name} {n}")
|
|
525
|
+
else:
|
|
526
|
+
scatterparams['color'] = cmap.get(n,cmap[i])
|
|
527
|
+
fitparams['color'] = cmap.get(n,cmap[i])
|
|
528
|
+
fillparams['color'] = cmap.get(n,cmap[i])
|
|
529
|
+
scatterparams['label'] = n
|
|
530
|
+
self.gCs[n].plot(ax = ax,
|
|
531
|
+
fill = plot_individual_fill,
|
|
532
|
+
scatterparams = scatterparams,
|
|
533
|
+
fillparams = fillparams,
|
|
534
|
+
fitparams = fitparams,
|
|
535
|
+
**params)
|
|
536
|
+
if plot_overall is True:
|
|
537
|
+
scatterparams['color'] = (0,0,0)
|
|
538
|
+
fitparams['color'] = (0,0,0)
|
|
539
|
+
fillparams['color'] = (0,0,0)
|
|
540
|
+
scatterparams['label'] = params.get('overall_label','Overall')
|
|
541
|
+
self.plot(ax = ax,
|
|
542
|
+
scatter = False,
|
|
543
|
+
scatterparams = scatterparams,
|
|
544
|
+
fillparams = fillparams,
|
|
545
|
+
fitparams = fitparams,
|
|
546
|
+
**params)
|
|
547
|
+
if show_legend is True:
|
|
548
|
+
ax.legend()
|
|
549
|
+
if filename is not None and main_p is True:
|
|
550
|
+
plt.savefig(f'{filename}')
|
|
551
|
+
elif main_p is True:
|
|
552
|
+
plt.show()
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
import matplotlib.pyplot as plt
|
|
3
|
+
from matplotlib import colormaps
|
|
4
|
+
import seaborn as sns
|
|
5
|
+
from typing import Union
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
|
|
8
|
+
TREND_COLORS = ["#4d4c49", "#d85e5e", "#3a668c", "#000000"]# constant, increasing,decreasing
|
|
9
|
+
CI_COLORS = ["#fef5da", "#ff988c", "#b4c5f4", "#474747"]
|
|
10
|
+
SCATTERPARAMS = {"marker": 'o', "linewidths":0.5, "alpha":1,"s":50,"label":'_nolegend_'}
|
|
11
|
+
FILLPARAMS = {"linestyle":"None", "alpha":0.3,"label":'_nolegend_'}
|
|
12
|
+
FITPARAMS = {"label":'_nolegend_'}
|
|
13
|
+
|
|
14
|
+
def parse_cmap(cmap):
|
|
15
|
+
"""
|
|
16
|
+
Parse and normalize colormap to dictionary format.
|
|
17
|
+
|
|
18
|
+
Converts string colormap names, lists, or dictionaries to a standardized
|
|
19
|
+
dictionary format with integer keys.
|
|
20
|
+
|
|
21
|
+
:param cmap: Colormap as string name, list of colors, or dictionary
|
|
22
|
+
:return: Dictionary mapping integers to color values
|
|
23
|
+
"""
|
|
24
|
+
if isinstance(cmap,str):
|
|
25
|
+
cmap = {i:x for i,x in enumerate(colormaps[cmap].colors)}
|
|
26
|
+
elif isinstance(cmap,list):
|
|
27
|
+
cmap = {i:x for i,x in enumerate(cmap)}
|
|
28
|
+
elif isinstance(cmap, dict):
|
|
29
|
+
if not all(isinstance(item, int) for item in cmap.keys()):
|
|
30
|
+
cmap.update({i:v for i,(k,v) in enumerate(cmap.items())})
|
|
31
|
+
else:
|
|
32
|
+
cmap = {i:x for i,x in enumerate(colormaps['Set1'].colors)}
|
|
33
|
+
return cmap
|
|
34
|
+
|
|
35
|
+
def conf_int_to_trend(ci:Union[pd.DataFrame,list], param:str='X'):
|
|
36
|
+
"""
|
|
37
|
+
Convert confidence interval to trend classification.
|
|
38
|
+
|
|
39
|
+
Returns trend code 0, 1, or 2 from the confidence interval:
|
|
40
|
+
|
|
41
|
+
- if ci crosses 0, then returns 0 (non-significant)
|
|
42
|
+
- if ci is completely below 0, then returns 2 (decreasing)
|
|
43
|
+
- else returns 1 (increasing)
|
|
44
|
+
|
|
45
|
+
:param ci: Confidence interval to process, either a tuple (length 2), a DataFrame, or result from conf_int given by statsmodels
|
|
46
|
+
:param param: Column name to use if a DataFrame was given as ci
|
|
47
|
+
:return: Integer trend code (0, 1, or 2)
|
|
48
|
+
"""
|
|
49
|
+
if isinstance(ci, list) and len(ci)==2:
|
|
50
|
+
l,u = ci
|
|
51
|
+
else:
|
|
52
|
+
l,u=ci.loc[param,:]
|
|
53
|
+
if u<0:
|
|
54
|
+
return 2 # significantly decreasing (at tested level)
|
|
55
|
+
elif l>0:
|
|
56
|
+
return 1 # significantly increasing (at tested level)
|
|
57
|
+
else:
|
|
58
|
+
return 0 # non significant (at tested level)
|
|
59
|
+
|
|
60
|
+
def infer_date_format(s_date:str):
|
|
61
|
+
"""
|
|
62
|
+
Infer date format from a sample date string.
|
|
63
|
+
|
|
64
|
+
:param s_date: Sample date string to infer format from
|
|
65
|
+
:return: Date format string if successfully inferred, None otherwise
|
|
66
|
+
"""
|
|
67
|
+
date_patterns = ["%d-%m-%Y","%d-%m-%Y %H:%M:%S",
|
|
68
|
+
"%Y-%m-%d","%Y-%m-%d %H:%M:%S",
|
|
69
|
+
"%y-%m-%d","%y-%m-%d %H:%M:%S",
|
|
70
|
+
"%d %b %y","%d %b %y %H:%M:%S",
|
|
71
|
+
"%d %b %Y","%d %b %Y %H:%M:%S",
|
|
72
|
+
]
|
|
73
|
+
|
|
74
|
+
for pattern in date_patterns:
|
|
75
|
+
try:
|
|
76
|
+
datetime.strptime(s_date, pattern).date()
|
|
77
|
+
except:
|
|
78
|
+
pass
|
|
79
|
+
else:
|
|
80
|
+
return pattern
|
|
81
|
+
|
|
82
|
+
def guess_timelapse_type(series:Union[list,pd.DataFrame,pd.Series], date_format:str = "%Y-%m-%d"):
|
|
83
|
+
"""
|
|
84
|
+
Guess the time lapse unit in a time series and re-index it.
|
|
85
|
+
|
|
86
|
+
Analyzes datetime differences to determine whether data is indexed by
|
|
87
|
+
seconds, minutes, hours, days, or years.
|
|
88
|
+
|
|
89
|
+
:param series: Time series data (list, DataFrame, or Series)
|
|
90
|
+
:param date_format: Format string for parsing datetime values
|
|
91
|
+
:return: Tuple of (indexed series, datetype string)
|
|
92
|
+
"""
|
|
93
|
+
try:
|
|
94
|
+
series = pd.to_datetime(series, format = date_format)
|
|
95
|
+
except:
|
|
96
|
+
pass
|
|
97
|
+
start = min(series)
|
|
98
|
+
series = series-start # reindex timecol from lowest one
|
|
99
|
+
if all([i.seconds==0 for i in series]):
|
|
100
|
+
#check if indexed by minutes
|
|
101
|
+
if all([i.seconds%60 == 0 for i in series]):
|
|
102
|
+
#check if indexed by hour
|
|
103
|
+
if all([i.seconds%3600 == 0 for i in series]):
|
|
104
|
+
#check if indexed by year
|
|
105
|
+
if all([i.days%365==0 or i.days%366==0 for i in series]):
|
|
106
|
+
#index by year
|
|
107
|
+
series = [int(i.days/365) for i in series]
|
|
108
|
+
datetype = 'year'
|
|
109
|
+
else:#index by day
|
|
110
|
+
series = [i.days for i in series]
|
|
111
|
+
datetype = 'day'
|
|
112
|
+
else:#index by hour
|
|
113
|
+
series = [i.days * 24 + i.hours for i in series]
|
|
114
|
+
datetype = 'hour'
|
|
115
|
+
else:#index by minute
|
|
116
|
+
series = [i.days * 60 * 24 + i.minutes for i in series]
|
|
117
|
+
datetype = 'minute'
|
|
118
|
+
else:# index on the seconds:
|
|
119
|
+
series = [i.days * 3600 * 24 + i.seconds for i in series]
|
|
120
|
+
datetype = 'second'
|
|
121
|
+
return series,datetype
|