chowclassifier 1.0.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- chowclassifier/__init__.py +6 -0
- chowclassifier/__main__.py +81 -0
- chowclassifier/chewdata.py +382 -0
- chowclassifier/chow.py +552 -0
- chowclassifier/utilities.py +121 -0
- chowclassifier-1.0.10.dist-info/METADATA +181 -0
- chowclassifier-1.0.10.dist-info/RECORD +11 -0
- chowclassifier-1.0.10.dist-info/WHEEL +5 -0
- chowclassifier-1.0.10.dist-info/entry_points.txt +2 -0
- chowclassifier-1.0.10.dist-info/licenses/LICENSE +117 -0
- chowclassifier-1.0.10.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# __main__.py
|
|
2
|
+
import argparse
|
|
3
|
+
from chowclassifier.chewdata import ChewData
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def main():
|
|
7
|
+
"""
|
|
8
|
+
run analysis on given data
|
|
9
|
+
|
|
10
|
+
:params -f, --filename: filename
|
|
11
|
+
:params -s, --sep: separator for parsing the filename (if type csv)
|
|
12
|
+
:params -X, --timecol: timecol
|
|
13
|
+
:params -y, --ycol: ycol
|
|
14
|
+
:params -n, --namecol: name of column for groupby
|
|
15
|
+
:params -b, --breakpoint: initial breakpoint (if None, midpoint will be taken)
|
|
16
|
+
:params -k, --breakpoints: candidates for breakpoints (list)
|
|
17
|
+
:params -m, --margin: margin around initial breakpoint
|
|
18
|
+
:params -o, --output: filename for the excel or csv file output
|
|
19
|
+
:params -p, --path: path where the figures will be saved
|
|
20
|
+
:params -a, --alpha: significance level in (0,1)
|
|
21
|
+
:params --no-figures: Do not export figures
|
|
22
|
+
:params --no-table: Do not export results table
|
|
23
|
+
:params --nrows: Numbers of rows in the grouped figure (leave ncols blank)
|
|
24
|
+
:params --ncols: Number of columns in the groupe figure (leave nrows blank)
|
|
25
|
+
:params --fig-title: title of the figure
|
|
26
|
+
:params --xlabel: label of the x axis
|
|
27
|
+
:params --ylabel: label of the y axis
|
|
28
|
+
:params --sharey: share y axis across subplots
|
|
29
|
+
:params --figsize: size of figures
|
|
30
|
+
"""
|
|
31
|
+
parser = argparse.ArgumentParser()
|
|
32
|
+
parser.add_argument('-f','--filename', type=str)
|
|
33
|
+
parser.add_argument('-X','--timecol', type=str, default = 'date')
|
|
34
|
+
parser.add_argument('-y','--ycol', type=str, default = 'value')
|
|
35
|
+
parser.add_argument('-n','--namecol', type=str, default = 'name')
|
|
36
|
+
parser.add_argument('-b','--breakpoint', type=float, default = None)
|
|
37
|
+
parser.add_argument('-k','--breakpoints', type=list, default = None)
|
|
38
|
+
parser.add_argument('-m','--margin', type=int, default = 5)
|
|
39
|
+
parser.add_argument('-o','--output', type=str, default = 'output.xlsx')
|
|
40
|
+
parser.add_argument('-p','--path', type=str, default = 'figs')
|
|
41
|
+
parser.add_argument('-s','--sep', type=str, default = ',')
|
|
42
|
+
parser.add_argument('-a','--alpha', type=float, default = 0.01)
|
|
43
|
+
parser.add_argument('--no-figures', type=bool, default = False)
|
|
44
|
+
parser.add_argument('--no-table', type=bool, default = False)
|
|
45
|
+
parser.add_argument('--nrows', type=int, default = None)
|
|
46
|
+
parser.add_argument('--ncols', type=int, default = 3)
|
|
47
|
+
parser.add_argument('--fig-title', type=str, default = '')
|
|
48
|
+
parser.add_argument('--fig-format', type=str, default = 'pdf')
|
|
49
|
+
parser.add_argument('--xlabel', type=str, default = 'X')
|
|
50
|
+
parser.add_argument('--ylabel', type=str, default = 'y')
|
|
51
|
+
parser.add_argument('--sharey', type=bool, default = True)
|
|
52
|
+
parser.add_argument('--figsize',type = list, default = (16,8))
|
|
53
|
+
args = parser.parse_args()
|
|
54
|
+
if args.filename is None:
|
|
55
|
+
print("No data given. Please pass filename as argument --filename 'data.csv'")
|
|
56
|
+
return None
|
|
57
|
+
C = ChewData(filename = args.filename,
|
|
58
|
+
namecol = args.namecol,
|
|
59
|
+
ycol = args.ycol,
|
|
60
|
+
timecol = args.timecol,
|
|
61
|
+
sep = args.sep,
|
|
62
|
+
margin = args.margin,
|
|
63
|
+
alpha = args.alpha,
|
|
64
|
+
initial_breakpoint = args.breakpoint,
|
|
65
|
+
breakpoints = args.breakpoints)
|
|
66
|
+
C.run()
|
|
67
|
+
if args.no_table is False:
|
|
68
|
+
C.save_to_file(filename = args.output, sep = args.sep)
|
|
69
|
+
if args.no_figures is False:
|
|
70
|
+
C.plot(f'output.{args.fig_format}',
|
|
71
|
+
nrows = args.nrows,
|
|
72
|
+
ncols = args.ncols,
|
|
73
|
+
sharey = args.sharey,
|
|
74
|
+
figsize = args.figsize
|
|
75
|
+
)
|
|
76
|
+
C.plot_individually(savingpath = args.path, format = args.fig_format)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
if __name__ == "__main__":
|
|
80
|
+
|
|
81
|
+
main()
|
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
"""Implementation of a Chow Classifier"""
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pandas as pd
|
|
5
|
+
import matplotlib.pyplot as plt
|
|
6
|
+
from typing import Union
|
|
7
|
+
import os
|
|
8
|
+
from tqdm import tqdm
|
|
9
|
+
from .utilities import infer_date_format, guess_timelapse_type
|
|
10
|
+
from .chow import Chow
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ChewData():
|
|
14
|
+
"""
|
|
15
|
+
Handle a full data frame with grouping of time series.
|
|
16
|
+
|
|
17
|
+
:param df: Dataset as a pandas DataFrame (optional if filename provided)
|
|
18
|
+
:param filename: Path to dataframe file (used if df not given)
|
|
19
|
+
:param namecol: Name of the column with group names (used in groupby)
|
|
20
|
+
:param initial_breakpoint: Initial breakpoint value
|
|
21
|
+
:param sep: Separator character in CSV files
|
|
22
|
+
:param timecol: Name of the time column in dataset
|
|
23
|
+
:param ycol: Name of the variable column in dataset
|
|
24
|
+
:param groupcol: Name of the grouping column in dataset
|
|
25
|
+
:param margin: Range of indexes around breakpoint/midpoint, see Chow.get_breakpoint_indices
|
|
26
|
+
:param alpha: Significance level at which to test the trends (Bonferroni correction will be applied later)
|
|
27
|
+
"""
|
|
28
|
+
def __init__(self,
|
|
29
|
+
df:pd.DataFrame = None,
|
|
30
|
+
filename:str = None,
|
|
31
|
+
groupcol:Union[None,str] = 'g',
|
|
32
|
+
namecol:str = 'name',
|
|
33
|
+
initial_breakpoint:float = None,
|
|
34
|
+
sep=',',
|
|
35
|
+
timecol:str = 'year',
|
|
36
|
+
ycol:str = 'value',
|
|
37
|
+
margin:int = 2,
|
|
38
|
+
alpha:float = 0.01):
|
|
39
|
+
"""
|
|
40
|
+
Initialize a ChewData object for analyzing multiple time series.
|
|
41
|
+
|
|
42
|
+
Either df or filename must be provided. The class will automatically
|
|
43
|
+
detect wide vs long format and convert if necessary.
|
|
44
|
+
"""
|
|
45
|
+
# get dataframe
|
|
46
|
+
|
|
47
|
+
self.df = df
|
|
48
|
+
self.namecol = namecol
|
|
49
|
+
self.timecol = timecol
|
|
50
|
+
self.groupcol = groupcol
|
|
51
|
+
if ycol is None:
|
|
52
|
+
self.ycol = 'value'
|
|
53
|
+
else:
|
|
54
|
+
self.ycol = ycol
|
|
55
|
+
self.margin = margin
|
|
56
|
+
self.alpha = alpha
|
|
57
|
+
self.initial_breakpoint = initial_breakpoint
|
|
58
|
+
if self.df is None:
|
|
59
|
+
self.get_df(filename, sep)
|
|
60
|
+
if self.namecol not in self.df.columns and\
|
|
61
|
+
len([k for k in self.df.columns if k not in ['id',ycol,timecol,'',np.nan]])>0:
|
|
62
|
+
#inferred that df is in wide format
|
|
63
|
+
try:
|
|
64
|
+
self.melt()
|
|
65
|
+
except ValueError as e:
|
|
66
|
+
raise ValueError("Could not melt the dataframe. Is it really in wide format? Check given namecol")
|
|
67
|
+
if pd.api.types.is_numeric_dtype(self.df[self.timecol]) is False:
|
|
68
|
+
date_format = infer_date_format(self.df.loc[self.df[self.timecol]!=np.nan,self.timecol][0])
|
|
69
|
+
print(f'Inferred format {date_format} for column {self.timecol}')
|
|
70
|
+
if date_format is not None:
|
|
71
|
+
self.parse_timecol(date_format)
|
|
72
|
+
def melt(self):
|
|
73
|
+
"""
|
|
74
|
+
Pivot the dataframe from wide to long format.
|
|
75
|
+
|
|
76
|
+
Converts a wide-format dataframe (with time series in columns) to
|
|
77
|
+
long format (with a variable column for series names).
|
|
78
|
+
"""
|
|
79
|
+
if self.groupcol in self.df.columns:
|
|
80
|
+
self.df = self.df.melt(id_vars=[self.timecol,self.groupcol], value_name=self.ycol).dropna()
|
|
81
|
+
else:
|
|
82
|
+
self.df = self.df.melt(id_vars=[self.timecol], value_name=self.ycol).dropna()
|
|
83
|
+
self.namecol = 'variable'
|
|
84
|
+
def get_df(self, filename, sep):
|
|
85
|
+
"""
|
|
86
|
+
Load dataframe from file.
|
|
87
|
+
|
|
88
|
+
Supports Excel (.xlsx, .xls), CSV (.csv), and TSV (.tsv) formats.
|
|
89
|
+
|
|
90
|
+
:param filename: Path to the file
|
|
91
|
+
:param sep: Separator character for CSV files
|
|
92
|
+
:raises ValueError: If file extension is not recognized
|
|
93
|
+
"""
|
|
94
|
+
ext = filename.split('.')[-1]
|
|
95
|
+
if ext == 'xlsx' or ext == 'xls':
|
|
96
|
+
self.df = pd.read_excel(filename, engine='openpyxl')
|
|
97
|
+
elif ext == 'csv':
|
|
98
|
+
self.df = pd.read_csv(filename,sep = sep)
|
|
99
|
+
elif ext == 'tsv':
|
|
100
|
+
self.df = pd.read_csv(filename, sep='\t')
|
|
101
|
+
else:
|
|
102
|
+
raise ValueError(f"Filename {filename} does not have a recognised extension")
|
|
103
|
+
def parse_timecol(self, date_format:str = '%Y-%m-%d'):
|
|
104
|
+
"""
|
|
105
|
+
Transform time column from dates to numeric values.
|
|
106
|
+
|
|
107
|
+
The time column must be numeric for analysis. If you have dates instead,
|
|
108
|
+
use this method to transform that column. Note: you need to adjust the
|
|
109
|
+
breakpoint accordingly!
|
|
110
|
+
|
|
111
|
+
:param date_format: Date format string (default '%Y-%m-%d')
|
|
112
|
+
"""
|
|
113
|
+
series,self.datetype = guess_timelapse_type(self.df[self.timecol], date_format = date_format)
|
|
114
|
+
self.df[self.datetype] = series
|
|
115
|
+
self.timecol = self.datetype
|
|
116
|
+
print(f'Indexed the time column {self.timecol} as {self.datetype}s')
|
|
117
|
+
def run_C(self, sub_df:pd.DataFrame, name:str, initial_breakpoint:Union[float,int])->(dict,Chow):
|
|
118
|
+
"""
|
|
119
|
+
Run the Chow classification on a single dataframe.
|
|
120
|
+
|
|
121
|
+
:param sub_df: DataFrame containing the time series data
|
|
122
|
+
:param name: Name identifier for the series
|
|
123
|
+
:param initial_breakpoint: Initial break point for Chow test (if None, will take midpoint of timecol)
|
|
124
|
+
:return: Tuple of (dictionary of parameters from the analysis, Chow object from the analysis)
|
|
125
|
+
"""
|
|
126
|
+
C = Chow(df = sub_df,
|
|
127
|
+
name = name,
|
|
128
|
+
initial_breakpoint = initial_breakpoint,
|
|
129
|
+
timecol = self.timecol,
|
|
130
|
+
ycol = self.ycol,
|
|
131
|
+
alpha = self.alpha,
|
|
132
|
+
margin = self.margin)
|
|
133
|
+
try:
|
|
134
|
+
C.run()
|
|
135
|
+
except ValueError:
|
|
136
|
+
return None,None
|
|
137
|
+
else:
|
|
138
|
+
params = C.params() # get the regressions parameters
|
|
139
|
+
params['name'] = name
|
|
140
|
+
return params, C
|
|
141
|
+
def run(self, initial_breakpoint = None):
|
|
142
|
+
"""
|
|
143
|
+
Run the Chow classification for each group in the dataset.
|
|
144
|
+
|
|
145
|
+
:param initial_breakpoint: Initial break point for Chow test (if None, will take midpoint of timecol)
|
|
146
|
+
"""
|
|
147
|
+
results = []
|
|
148
|
+
self.chows = []
|
|
149
|
+
self.failed = []
|
|
150
|
+
if initial_breakpoint is None and self.initial_breakpoint is None:
|
|
151
|
+
x = list(set(self.df[self.timecol]))
|
|
152
|
+
self.initial_breakpoint = x[int(len(x)/2)]
|
|
153
|
+
initial_breakpoint = self.initial_breakpoint
|
|
154
|
+
print(f"Set {initial_breakpoint = }")
|
|
155
|
+
elif initial_breakpoint is None:
|
|
156
|
+
initial_breakpoint = self.initial_breakpoint
|
|
157
|
+
print("Running analysis")
|
|
158
|
+
with tqdm(total = len(self.df.groupby(self.namecol))) as bar:
|
|
159
|
+
for name, sub_df in self.df.groupby(self.namecol):
|
|
160
|
+
if sub_df.shape[0] > 1:
|
|
161
|
+
params,C = self.run_C(sub_df, name = name, initial_breakpoint=initial_breakpoint)
|
|
162
|
+
if C is None:
|
|
163
|
+
self.failed.append(name)
|
|
164
|
+
else:
|
|
165
|
+
results.append(params)
|
|
166
|
+
self.chows.append(C)
|
|
167
|
+
else:
|
|
168
|
+
self.failed.append(name)
|
|
169
|
+
bar.update(1)
|
|
170
|
+
self.results = pd.DataFrame(results) # put the results in a dataframe
|
|
171
|
+
def plot(self,filename:Union[str,None]=None,
|
|
172
|
+
grouped:bool = False,
|
|
173
|
+
divide:int = 1,
|
|
174
|
+
nrows:Union[None,int] = None,
|
|
175
|
+
ncols:Union[None,int] = None,
|
|
176
|
+
cmap:Union[None,str] = None,
|
|
177
|
+
scatterparams:dict = {},
|
|
178
|
+
fillparams:dict = {},
|
|
179
|
+
fitparams:dict = {},
|
|
180
|
+
**params):
|
|
181
|
+
"""
|
|
182
|
+
Plot all trends in one image.
|
|
183
|
+
|
|
184
|
+
:param filename: Name of the output file with extension (e.g., 'name.pdf')
|
|
185
|
+
:param grouped: Whether to plot overall trends or grouped trends
|
|
186
|
+
:param divide: Divide the plots into n files
|
|
187
|
+
:param nrows: Number of rows in subplot grid
|
|
188
|
+
:param ncols: Number of columns in subplot grid
|
|
189
|
+
:param cmap: (grouped) Name of one of matplotlib cmaps
|
|
190
|
+
:param scatterparams: Dictionary of parameters for the scatter part of the plot
|
|
191
|
+
:param fillparams: Dictionary of parameters for the error fill area of the plot
|
|
192
|
+
:param fitparams: Dictionary of parameters for the fitted line
|
|
193
|
+
:param params: Named parameters passed to matplotlib (e.g., xlabel, ylabel, sharey, figsize, pad, plot_overall, plot_individual_fill, title)
|
|
194
|
+
"""
|
|
195
|
+
def get_n(m,_nplots = len(self.chows)):
|
|
196
|
+
n = _nplots/m
|
|
197
|
+
if n-int(n) == 0.:
|
|
198
|
+
return int(n)
|
|
199
|
+
return int(n)+1
|
|
200
|
+
if filename is not None:
|
|
201
|
+
f = filename.split(".")
|
|
202
|
+
filename = ".".join(f[0:-1])
|
|
203
|
+
ext = f[-1]
|
|
204
|
+
nplots = get_n(divide)
|
|
205
|
+
print("Plotting trends")
|
|
206
|
+
with tqdm(total = divide) as bar:
|
|
207
|
+
for i in range(0,divide):
|
|
208
|
+
if nrows is None and ncols is None:
|
|
209
|
+
ncols = 4
|
|
210
|
+
nrows = get_n(ncols,nplots)
|
|
211
|
+
elif nrows is not None:
|
|
212
|
+
ncols = get_n(nrows,nplots)
|
|
213
|
+
else:
|
|
214
|
+
nrows = get_n(ncols,nplots)
|
|
215
|
+
f, axes = plt.subplots(nrows=nrows,
|
|
216
|
+
ncols=ncols,
|
|
217
|
+
sharey=params.get("sharey",True),
|
|
218
|
+
figsize=params.get('figsize',(16,8)))
|
|
219
|
+
f.tight_layout(pad=params.get('pad',3.0)) # increase padding between sub figures
|
|
220
|
+
for chowObj, ax in zip(self.chows[i*nplots:(i+1)*nplots], axes.reshape(-1)):
|
|
221
|
+
if grouped is True and self.groupcol in chowObj.df.colnames:
|
|
222
|
+
plt.clf()
|
|
223
|
+
chowObj.plot_by_group(ax = ax,
|
|
224
|
+
xlabel=params.get('xlabel',self.timecol),
|
|
225
|
+
ylabel=params.get('ylabel',self.ycol),
|
|
226
|
+
cmap = cmap,
|
|
227
|
+
title=chowObj.name,
|
|
228
|
+
scatterparams = scatterparams,
|
|
229
|
+
fillparams = fillparams,
|
|
230
|
+
fitparams = fitparams,
|
|
231
|
+
**params)
|
|
232
|
+
else:
|
|
233
|
+
chowObj.plot(ax=ax,
|
|
234
|
+
xlabel=params.get('xlabel','X'),
|
|
235
|
+
ylabel=params.get('ylabel','y'),
|
|
236
|
+
title=chowObj.name,
|
|
237
|
+
scatterparams = scatterparams,
|
|
238
|
+
fillparams = fillparams,
|
|
239
|
+
fitparams = fitparams)
|
|
240
|
+
if filename is not None:
|
|
241
|
+
plt.savefig(f'{filename}_{i:02d}.{ext}')
|
|
242
|
+
bar.update(1)
|
|
243
|
+
if filename is None:
|
|
244
|
+
plt.show()
|
|
245
|
+
def plot_individually(self, savingpath:str = 'figs', format:str = 'pdf', **params):
|
|
246
|
+
"""
|
|
247
|
+
Plot all trends in individual files.
|
|
248
|
+
|
|
249
|
+
:param savingpath: Path where the plots will be saved
|
|
250
|
+
:param format: Format of the image (e.g., 'pdf', 'png')
|
|
251
|
+
:param params: Named parameters passed to matplotlib (e.g., xlabel, ylabel, prefix, scatterparams, fillparams, fitparams)
|
|
252
|
+
"""
|
|
253
|
+
# check if saving path exists:
|
|
254
|
+
if not os.path.isdir(savingpath):
|
|
255
|
+
os.makedirs(savingpath)
|
|
256
|
+
print("Plotting trends individually")
|
|
257
|
+
with tqdm(total = len(self.chows)) as bar:
|
|
258
|
+
for chowObj in self.chows:
|
|
259
|
+
plt.clf()
|
|
260
|
+
chowObj.plot(filename = f'{"/".join([savingpath, chowObj.name])}.{format}',
|
|
261
|
+
xlabel=params.get('xlabel', self.timecol),
|
|
262
|
+
ylabel=params.get('ylabel', self.ycol),
|
|
263
|
+
title=params.get('prefix','')+chowObj.name,
|
|
264
|
+
scatterparams = params.get('scatterparams',{}),
|
|
265
|
+
fillparams = params.get('fillparams',{}),
|
|
266
|
+
fitparams = params.get('fitparams',{}))
|
|
267
|
+
bar.update(1)
|
|
268
|
+
def plot_by_group(self,
|
|
269
|
+
savingpath:Union[None,str] = 'figs',
|
|
270
|
+
format:str = 'pdf',
|
|
271
|
+
cmap:str = 'Set1',
|
|
272
|
+
**params):
|
|
273
|
+
"""
|
|
274
|
+
Plot all trends in individual files, grouped by category.
|
|
275
|
+
|
|
276
|
+
:param savingpath: Path where the plots will be saved (None to show instead of save)
|
|
277
|
+
:param format: Format of the image (e.g., 'pdf', 'png')
|
|
278
|
+
:param cmap: Name of one of matplotlib cmaps
|
|
279
|
+
:param params: Named parameters passed to matplotlib and plot_by_group.
|
|
280
|
+
Includes: show_legend, plot_overall, plot_individual_fill,
|
|
281
|
+
xlabel, ylabel, title, overall_label
|
|
282
|
+
"""
|
|
283
|
+
if not os.path.isdir(savingpath):
|
|
284
|
+
os.makedirs(savingpath)
|
|
285
|
+
print("Plotting trends by group")
|
|
286
|
+
with tqdm(total = len(self.chows)) as bar:
|
|
287
|
+
for chowObj in self.chows:
|
|
288
|
+
plt.clf()
|
|
289
|
+
params['title'] = params.get('title', chowObj.name)
|
|
290
|
+
params['xlabel'] = params.get('xlabel', self.timecol)
|
|
291
|
+
params['ylabel'] = params.get('ylabel', self.ycol)
|
|
292
|
+
if savingpath is not None:
|
|
293
|
+
filename = f"{savingpath}/{chowObj.name}_bygroups.{format}"
|
|
294
|
+
else:
|
|
295
|
+
filename = None
|
|
296
|
+
chowObj.plot_by_group(filename = filename, groupcol = self.groupcol, cmap = cmap, **params)
|
|
297
|
+
bar.update(1)
|
|
298
|
+
if savingpath is None:
|
|
299
|
+
plt.show()
|
|
300
|
+
|
|
301
|
+
def results_df(self):
|
|
302
|
+
"""
|
|
303
|
+
Create dataframe with analysis results.
|
|
304
|
+
|
|
305
|
+
:return: DataFrame with multi-level columns containing results for all analyzed series
|
|
306
|
+
"""
|
|
307
|
+
d = {'name': [],
|
|
308
|
+
'breakpoint': [],
|
|
309
|
+
'score': [],
|
|
310
|
+
'model0': [],
|
|
311
|
+
'model1': [],
|
|
312
|
+
'model2': []
|
|
313
|
+
}
|
|
314
|
+
model_params = self.chows[0].params_names()
|
|
315
|
+
for c in self.chows:
|
|
316
|
+
params = c.params()
|
|
317
|
+
d['name'].append(c.name)
|
|
318
|
+
d['breakpoint'].append(c.initial_breakpoint)
|
|
319
|
+
d['score'].append(c.score)
|
|
320
|
+
for k,v in params.items():
|
|
321
|
+
d[k].append(v)
|
|
322
|
+
data, cols = [],[]
|
|
323
|
+
for k, v in d.items():
|
|
324
|
+
if not isinstance(v[0], dict):
|
|
325
|
+
cols.append((k, ''))
|
|
326
|
+
data.append(v)
|
|
327
|
+
else:
|
|
328
|
+
subdata = {}
|
|
329
|
+
for v1 in v:
|
|
330
|
+
for p in model_params:
|
|
331
|
+
subdata.setdefault(p,[]).append(v1.get(p,np.nan))
|
|
332
|
+
for k1,v1 in subdata.items():
|
|
333
|
+
cols.append((k, k1))
|
|
334
|
+
data.append(v1)
|
|
335
|
+
df = pd.DataFrame(list(zip(*data)), columns=pd.MultiIndex.from_tuples(cols))
|
|
336
|
+
return df
|
|
337
|
+
def to_excel(self, filename:str, **kwargs):
|
|
338
|
+
"""
|
|
339
|
+
Export results to Excel file.
|
|
340
|
+
|
|
341
|
+
Creates multiple sheets: 'results' (analysis results), 'data' (input data),
|
|
342
|
+
and 'failed' (list of failed analyses).
|
|
343
|
+
|
|
344
|
+
:param filename: Output filename with .xlsx or .xls extension
|
|
345
|
+
:param kwargs: Additional keyword arguments passed to pd.ExcelWriter
|
|
346
|
+
"""
|
|
347
|
+
df = self.results_df()
|
|
348
|
+
with pd.ExcelWriter(filename, **kwargs) as writer:
|
|
349
|
+
df.to_excel(writer, sheet_name = 'results')
|
|
350
|
+
self.df.to_excel(writer, sheet_name = 'data')
|
|
351
|
+
pd.DataFrame([[x] for x in self.failed],columns=['Name']).to_excel(writer, sheet_name='failed')
|
|
352
|
+
def to_csv(self, filename:str, **kwargs):
|
|
353
|
+
"""
|
|
354
|
+
Export results to CSV file.
|
|
355
|
+
|
|
356
|
+
:param filename: Output filename with .csv extension
|
|
357
|
+
:param kwargs: Additional keyword arguments passed to pd.DataFrame.to_csv
|
|
358
|
+
"""
|
|
359
|
+
|
|
360
|
+
df = self.results_df()
|
|
361
|
+
df.to_csv(filename, **kwargs)
|
|
362
|
+
def save_to_file(self, filename:str, **kwargs):
|
|
363
|
+
"""
|
|
364
|
+
Export results to Excel or CSV file based on extension.
|
|
365
|
+
|
|
366
|
+
Automatically detects file format from extension and calls the appropriate
|
|
367
|
+
export method.
|
|
368
|
+
|
|
369
|
+
:param filename: Output filename with extension (.xlsx, .xls, or .csv)
|
|
370
|
+
:param kwargs: Additional keyword arguments passed to export methods
|
|
371
|
+
:raises ValueError: If filename extension is not supported
|
|
372
|
+
"""
|
|
373
|
+
|
|
374
|
+
ext = filename.split('.')[-1]
|
|
375
|
+
if ext in ['xlsx','xls']:
|
|
376
|
+
self.to_excel(filename, engine='openpyxl')
|
|
377
|
+
elif ext == 'csv':
|
|
378
|
+
self.to_csv(filename,**kwargs)
|
|
379
|
+
else:
|
|
380
|
+
raise ValueError(f'filename {filename} does not have a supported extension. Should be .csv, .xlsx or .xls.')
|
|
381
|
+
|
|
382
|
+
|