plans 1.1.0.post1.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- plans/__init__.py +41 -0
- plans/analyst.py +3092 -0
- plans/config.py +103 -0
- plans/data/.gitkeep +2 -0
- plans/data/fields.csv +61 -0
- plans/data/files.csv +26 -0
- plans/data/tools.csv +2 -0
- plans/datasets/__init__.py +42 -0
- plans/datasets/chrono.py +540 -0
- plans/datasets/core.py +6224 -0
- plans/datasets/spatial.py +1298 -0
- plans/geo.py +1397 -0
- plans/hydrology/__init__.py +42 -0
- plans/hydrology/core.py +723 -0
- plans/hydrology/downscaled.py +1024 -0
- plans/hydrology/upscaled.py +3890 -0
- plans/parsers/__init__.py +40 -0
- plans/parsers/flare.py +484 -0
- plans/parsers/inmet.py +587 -0
- plans/parsers/snirh.py +219 -0
- plans/project.py +964 -0
- plans/qutils.py +3648 -0
- plans/root.py +2256 -0
- plans/tools/__init__.py +0 -0
- plans/tools/analysis_climate_series_lulc.py +231 -0
- plans/tools/analysis_dto.py +193 -0
- plans/tools/analysis_lulc_parameters.py +269 -0
- plans/tools/analysis_lulc_series.py +220 -0
- plans/tools/analysis_soils_parameters.py +253 -0
- plans/tools/core.py +559 -0
- plans/tools/demo.py +168 -0
- plans/viewer.py +414 -0
- plans-1.1.0.post1.dev0.dist-info/METADATA +120 -0
- plans-1.1.0.post1.dev0.dist-info/RECORD +37 -0
- plans-1.1.0.post1.dev0.dist-info/WHEEL +5 -0
- plans-1.1.0.post1.dev0.dist-info/licenses/LICENSE +674 -0
- plans-1.1.0.post1.dev0.dist-info/top_level.txt +1 -0
plans/analyst.py
ADDED
|
@@ -0,0 +1,3092 @@
|
|
|
1
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
2
|
+
#
|
|
3
|
+
# Copyright (C) 2025 The Project Authors
|
|
4
|
+
# See pyproject.toml for authors/maintainers.
|
|
5
|
+
# See LICENSE for license details.
|
|
6
|
+
"""
|
|
7
|
+
Classes designed to handle statistical analysis.
|
|
8
|
+
|
|
9
|
+
Overview
|
|
10
|
+
--------
|
|
11
|
+
|
|
12
|
+
# todo [major docstring improvement] -- overview
|
|
13
|
+
Mauris gravida ex quam, in porttitor lacus lobortis vitae.
|
|
14
|
+
In a lacinia nisl. Pellentesque habitant morbi tristique senectus
|
|
15
|
+
et netus et malesuada fames ac turpis egestas.
|
|
16
|
+
|
|
17
|
+
Example
|
|
18
|
+
-------
|
|
19
|
+
|
|
20
|
+
# todo [major docstring improvement] -- examples
|
|
21
|
+
Lorem ipsum dolor sit amet, consectetur adipiscing elit.
|
|
22
|
+
Nulla mollis tincidunt erat eget iaculis. Mauris gravida ex quam,
|
|
23
|
+
in porttitor lacus lobortis vitae. In a lacinia nisl.
|
|
24
|
+
|
|
25
|
+
.. code-block:: python
|
|
26
|
+
|
|
27
|
+
import numpy as np
|
|
28
|
+
from plans import analyst
|
|
29
|
+
|
|
30
|
+
# get data to a vector
|
|
31
|
+
data_vector = np.random.rand(1000)
|
|
32
|
+
|
|
33
|
+
# instantiate the Univar object
|
|
34
|
+
uni = analyst.Univar(sample=data_vector, name="my_data")
|
|
35
|
+
|
|
36
|
+
# view sample
|
|
37
|
+
uni.view()
|
|
38
|
+
|
|
39
|
+
Mauris gravida ex quam, in porttitor lacus lobortis vitae.
|
|
40
|
+
In a lacinia nisl.
|
|
41
|
+
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
# IMPORTS
|
|
45
|
+
# ***********************************************************************
|
|
46
|
+
# import modules from other libs
|
|
47
|
+
|
|
48
|
+
# Native imports
|
|
49
|
+
# =======================================================================
|
|
50
|
+
import os
|
|
51
|
+
|
|
52
|
+
# External imports
|
|
53
|
+
# =======================================================================
|
|
54
|
+
import numpy as np
|
|
55
|
+
import pandas as pd
|
|
56
|
+
import matplotlib as mpl
|
|
57
|
+
import matplotlib.pyplot as plt
|
|
58
|
+
import matplotlib.ticker as mticker
|
|
59
|
+
import matplotlib.colors as mcolors
|
|
60
|
+
|
|
61
|
+
# Project-level imports
|
|
62
|
+
# =======================================================================
|
|
63
|
+
from plans import viewer
|
|
64
|
+
from plans.root import DataSet
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
# FUNCTIONS
|
|
68
|
+
# ***********************************************************************
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def linear(x, c0, c1):
|
|
72
|
+
"""Linear function f(x) = c0 + c1 * x
|
|
73
|
+
|
|
74
|
+
:param x: function inputs
|
|
75
|
+
:type x: float | :class:`numpy.ndarray`
|
|
76
|
+
:param c0: translational parameter
|
|
77
|
+
:type c0: float
|
|
78
|
+
:param c1: scaling parameter
|
|
79
|
+
:type c1: float
|
|
80
|
+
:return: function output
|
|
81
|
+
:rtype: float | :class:`numpy.ndarray`
|
|
82
|
+
"""
|
|
83
|
+
return c0 + (x * c1)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def power(x, c0, c1, c2):
|
|
87
|
+
"""Power function f(x) = c2 * ((x + c0)^c1)
|
|
88
|
+
|
|
89
|
+
:param x: function inputs
|
|
90
|
+
:type x: float | :class:`numpy.ndarray`
|
|
91
|
+
:param c0: translational parameter
|
|
92
|
+
:type c0: float
|
|
93
|
+
:param c1: exponent parameter
|
|
94
|
+
:type c1: float
|
|
95
|
+
:param c2: scaling parameter
|
|
96
|
+
:type c2: float
|
|
97
|
+
:return: function output
|
|
98
|
+
:rtype: float | :class:`numpy.ndarray`
|
|
99
|
+
"""
|
|
100
|
+
return c2 * (np.power((x + c0), c1))
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def power_zero(x, c0, c1):
|
|
104
|
+
"""Power function with root in zero f(x) = c1 * ((x)^c0)
|
|
105
|
+
|
|
106
|
+
:param x: function inputs
|
|
107
|
+
:type x: float | :class:`numpy.ndarray`
|
|
108
|
+
:param c0: exponent parameter
|
|
109
|
+
:type c0: float
|
|
110
|
+
:param c1: scaling parameter
|
|
111
|
+
:type c1: float
|
|
112
|
+
:return: function output
|
|
113
|
+
:rtype: float | :class:`numpy.ndarray`
|
|
114
|
+
"""
|
|
115
|
+
return c1 * (np.power((x), c0))
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
# CLASSES
|
|
119
|
+
# ***********************************************************************
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class Univar(DataSet):
|
|
123
|
+
"""
|
|
124
|
+
The Univariate object
|
|
125
|
+
|
|
126
|
+
"""
|
|
127
|
+
|
|
128
|
+
def __init__(self, name="MyUnivar", alias="Uv0"):
|
|
129
|
+
# ------------ set defaults ----------- #
|
|
130
|
+
self.varfield = "V"
|
|
131
|
+
self.varalias = "Var"
|
|
132
|
+
self.varname = "Variable"
|
|
133
|
+
self.units = "units"
|
|
134
|
+
|
|
135
|
+
# ------------ call super ----------- #
|
|
136
|
+
super().__init__(name=name, alias=alias)
|
|
137
|
+
|
|
138
|
+
# overwriters
|
|
139
|
+
|
|
140
|
+
# attributes
|
|
141
|
+
self.stats_df = None
|
|
142
|
+
self.freq_df = None
|
|
143
|
+
self.weibull_df = None
|
|
144
|
+
|
|
145
|
+
def load_data(self, file_data):
|
|
146
|
+
"""
|
|
147
|
+
Load data from file. Expected to overwrite superior methods.
|
|
148
|
+
|
|
149
|
+
:param file_data: file path to data.
|
|
150
|
+
:type file_data: str
|
|
151
|
+
:return: None
|
|
152
|
+
:rtype: None
|
|
153
|
+
"""
|
|
154
|
+
|
|
155
|
+
# -------------- overwrite relative path inputs -------------- #
|
|
156
|
+
self.file_data = os.path.abspath(file_data)
|
|
157
|
+
|
|
158
|
+
# -------------- implement loading logic -------------- #
|
|
159
|
+
default_columns = {
|
|
160
|
+
#'DateTime': 'datetime64[1s]',
|
|
161
|
+
self.varfield: float,
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
# -------------- call loading function -------------- #
|
|
165
|
+
self.data = pd.read_csv(
|
|
166
|
+
self.file_data,
|
|
167
|
+
sep=self.file_csv_sep,
|
|
168
|
+
dtype=default_columns,
|
|
169
|
+
usecols=list(default_columns.keys()),
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
# -------------- post-loading logic -------------- #
|
|
173
|
+
self.data.dropna(inplace=True)
|
|
174
|
+
|
|
175
|
+
# -------------- update other mutables -------------- #
|
|
176
|
+
self.update()
|
|
177
|
+
|
|
178
|
+
# ... continues in downstream objects ... #
|
|
179
|
+
|
|
180
|
+
return None
|
|
181
|
+
|
|
182
|
+
def set_array(self, array):
|
|
183
|
+
"""
|
|
184
|
+
Set array to data
|
|
185
|
+
|
|
186
|
+
:param array: Numpy array
|
|
187
|
+
:type array: :class:`numpy.ndarray`
|
|
188
|
+
:return: None
|
|
189
|
+
:rtype: None
|
|
190
|
+
"""
|
|
191
|
+
self.data = pd.DataFrame({self.varfield: array})
|
|
192
|
+
self.update()
|
|
193
|
+
return None
|
|
194
|
+
|
|
195
|
+
def update(self):
|
|
196
|
+
# todo [docstring]
|
|
197
|
+
# ------------ call super ----------- #
|
|
198
|
+
super().update()
|
|
199
|
+
|
|
200
|
+
# update stats
|
|
201
|
+
if self.data is not None:
|
|
202
|
+
self.stats_df = self.get_basic_stats()
|
|
203
|
+
self.freq_df = self.get_frequency()
|
|
204
|
+
self.weibull_df = self.get_cdf_weibull()
|
|
205
|
+
|
|
206
|
+
# ... continues in downstream objects ... #
|
|
207
|
+
return None
|
|
208
|
+
|
|
209
|
+
# GET METHODS
|
|
210
|
+
# ===================================================================
|
|
211
|
+
|
|
212
|
+
def get_normality(self, clevel=0.95):
|
|
213
|
+
"""
|
|
214
|
+
Get normality analysis using standard tests
|
|
215
|
+
|
|
216
|
+
:return: dataframe of assessment results
|
|
217
|
+
:rtype: :class:`pandas.DataFrame`
|
|
218
|
+
"""
|
|
219
|
+
df_clean = self.data.dropna()
|
|
220
|
+
data = df_clean[self.varfield].values
|
|
221
|
+
|
|
222
|
+
# run tests
|
|
223
|
+
lst_tests = []
|
|
224
|
+
lst_tests.append(Univar.test_normality_ks(data, clevel=clevel))
|
|
225
|
+
lst_tests.append(Univar.test_normality_sw(data, clevel=clevel))
|
|
226
|
+
lst_tests.append(Univar.test_normality_dp(data, clevel=clevel))
|
|
227
|
+
# create dataframe
|
|
228
|
+
lst_names = []
|
|
229
|
+
lst_stats = []
|
|
230
|
+
lst_p = []
|
|
231
|
+
lst_clvl = []
|
|
232
|
+
lst_is = []
|
|
233
|
+
for e in lst_tests:
|
|
234
|
+
lst_names.append(e["Test"])
|
|
235
|
+
lst_stats.append(e["Statistic"])
|
|
236
|
+
lst_p.append(e["p-value"])
|
|
237
|
+
lst_clvl.append(e["Confidence"])
|
|
238
|
+
lst_is.append(e["is_normal"])
|
|
239
|
+
df_result = pd.DataFrame(
|
|
240
|
+
{
|
|
241
|
+
"Test": lst_names,
|
|
242
|
+
"Statistic": lst_stats,
|
|
243
|
+
"p-value": lst_p,
|
|
244
|
+
"is_normal": lst_is,
|
|
245
|
+
"Confidence": lst_clvl,
|
|
246
|
+
}
|
|
247
|
+
)
|
|
248
|
+
return df_result
|
|
249
|
+
|
|
250
|
+
def get_frequency(self):
|
|
251
|
+
"""
|
|
252
|
+
Get data frequencies
|
|
253
|
+
|
|
254
|
+
:return: result dataframe
|
|
255
|
+
:rtype: :class:`pandas.DataFrame`
|
|
256
|
+
"""
|
|
257
|
+
|
|
258
|
+
df_clean = self.data.dropna()
|
|
259
|
+
data = df_clean[self.varfield].values
|
|
260
|
+
|
|
261
|
+
# compute percentiles intevarls
|
|
262
|
+
vct_percentiles = np.arange(0, 100)
|
|
263
|
+
|
|
264
|
+
# get CFC values
|
|
265
|
+
vct_cfc = np.percentile(data, vct_percentiles)
|
|
266
|
+
|
|
267
|
+
# reverse to get exceedance
|
|
268
|
+
vct_exeed = 100 - vct_percentiles
|
|
269
|
+
|
|
270
|
+
# get count
|
|
271
|
+
vct_count = np.histogram(data, bins=len(vct_percentiles))[0]
|
|
272
|
+
|
|
273
|
+
# get empirical prob in the histogram
|
|
274
|
+
vct_empprob = vct_count / np.sum(vct_count)
|
|
275
|
+
|
|
276
|
+
df_result = pd.DataFrame(
|
|
277
|
+
{
|
|
278
|
+
"Percentiles": vct_percentiles,
|
|
279
|
+
"Exceedance": vct_exeed,
|
|
280
|
+
"Frequency": vct_count,
|
|
281
|
+
"Empirical Probability": vct_empprob,
|
|
282
|
+
"Values": vct_cfc,
|
|
283
|
+
}
|
|
284
|
+
)
|
|
285
|
+
return df_result
|
|
286
|
+
|
|
287
|
+
def get_basic_stats(self):
|
|
288
|
+
"""
|
|
289
|
+
Get basic statistics data of the variable field.
|
|
290
|
+
|
|
291
|
+
:return: DataFrame with statistics and their values.
|
|
292
|
+
:rtype: :class:`pandas.DataFrame`
|
|
293
|
+
"""
|
|
294
|
+
|
|
295
|
+
def _compute(data):
|
|
296
|
+
return {
|
|
297
|
+
"count": len(data),
|
|
298
|
+
"sum": np.sum(data),
|
|
299
|
+
"mean": np.mean(data),
|
|
300
|
+
"sd": np.std(data),
|
|
301
|
+
"min": np.min(data),
|
|
302
|
+
"p01": np.percentile(data, 1),
|
|
303
|
+
"p05": np.percentile(data, 5),
|
|
304
|
+
"p10": np.percentile(data, 10),
|
|
305
|
+
"p20": np.percentile(data, 20),
|
|
306
|
+
"p25": np.percentile(data, 25),
|
|
307
|
+
"p40": np.percentile(data, 40),
|
|
308
|
+
"p50": np.percentile(data, 50),
|
|
309
|
+
"p60": np.percentile(data, 60),
|
|
310
|
+
"p75": np.percentile(data, 75),
|
|
311
|
+
"p80": np.percentile(data, 80),
|
|
312
|
+
"p90": np.percentile(data, 90),
|
|
313
|
+
"p95": np.percentile(data, 95),
|
|
314
|
+
"p99": np.percentile(data, 99),
|
|
315
|
+
"max": np.max(data),
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
# handle data
|
|
319
|
+
df_clean = self.data.dropna()
|
|
320
|
+
# get data
|
|
321
|
+
data = df_clean[self.varfield].values
|
|
322
|
+
|
|
323
|
+
# handle void case
|
|
324
|
+
if len(df_clean) == 0:
|
|
325
|
+
dct = _compute(data=[0]) # feed dummy values
|
|
326
|
+
ls_values = [np.nan for key in dct]
|
|
327
|
+
ls_values[0] = 0 # fix for count
|
|
328
|
+
df_result = pd.DataFrame(
|
|
329
|
+
{"statistic": list(dct.keys()), "value": ls_values}
|
|
330
|
+
)
|
|
331
|
+
else:
|
|
332
|
+
dct = _compute(data=data)
|
|
333
|
+
ls_values = [dct[key] for key in dct]
|
|
334
|
+
df_result = pd.DataFrame({"statistic": list(dct.keys()), "value": ls_values})
|
|
335
|
+
return df_result
|
|
336
|
+
|
|
337
|
+
def get_cdf_weibull(self):
|
|
338
|
+
"""
|
|
339
|
+
Get the Weibull model for the data
|
|
340
|
+
|
|
341
|
+
:param x: function inputs
|
|
342
|
+
:type x: :class:`numpy.ndarray`
|
|
343
|
+
:return: model dataframe
|
|
344
|
+
:rtype: :class:`pandas.DataFrame`
|
|
345
|
+
"""
|
|
346
|
+
df_clean = self.data.dropna()
|
|
347
|
+
x = df_clean[self.varfield].values
|
|
348
|
+
v = np.sort(x)
|
|
349
|
+
vr = v[::-1]
|
|
350
|
+
r = np.arange(0, len(x)) + 1
|
|
351
|
+
px = Univar.weibull_px(ranks=r)
|
|
352
|
+
|
|
353
|
+
result_df = pd.DataFrame({"Data": vr, "P(X)": px, "F(X)": 1 - px})
|
|
354
|
+
|
|
355
|
+
return result_df
|
|
356
|
+
|
|
357
|
+
def get_cdf_gumbel(self):
|
|
358
|
+
"""
|
|
359
|
+
Get the Gumbel CDF for the data (it assumes that is a maxima dataset)
|
|
360
|
+
|
|
361
|
+
:return: multiple results from the assessment
|
|
362
|
+
:rtype: dict
|
|
363
|
+
"""
|
|
364
|
+
from scipy import stats
|
|
365
|
+
|
|
366
|
+
# clean data
|
|
367
|
+
df = self.data.dropna()
|
|
368
|
+
|
|
369
|
+
# get extra values
|
|
370
|
+
df = df.sort_values(by=self.varfield, ascending=False).reset_index(drop=True)
|
|
371
|
+
df["Rank"] = df.index + 1
|
|
372
|
+
df["P(X)_Empirical"] = Univar.empirical_px(ranks=df["Rank"].values)
|
|
373
|
+
df["P(X)_Weibull"] = Univar.weibull_px(ranks=df["Rank"].values)
|
|
374
|
+
df["T(X)_Weibull"] = 1 / df["P(X)_Weibull"]
|
|
375
|
+
df["P(X)_Gringorten"] = Univar.gringorten_px(ranks=df["Rank"].values)
|
|
376
|
+
|
|
377
|
+
# ---- model Gumbel using the method of moments ----
|
|
378
|
+
|
|
379
|
+
# using scipy
|
|
380
|
+
# Fit a Gumbel distribution to the data
|
|
381
|
+
params = stats.gumbel_r.fit(df[self.varfield].values, method="MM") # MLM or MM
|
|
382
|
+
gumbel_a = params[0]
|
|
383
|
+
gumbel_b = params[1]
|
|
384
|
+
|
|
385
|
+
# compute fit gumbel values
|
|
386
|
+
df["P(X)_Gumbel"] = 1 - Univar.gumbel_fx(
|
|
387
|
+
x=df[self.varfield].values, a=gumbel_a, b=gumbel_b
|
|
388
|
+
)
|
|
389
|
+
# return period
|
|
390
|
+
df["T(X)_Gumbel"] = 1 / df["P(X)_Gumbel"]
|
|
391
|
+
|
|
392
|
+
# compute uncertainty 90% bands for T(X) in the Annual Series
|
|
393
|
+
t = stats.t.ppf((1 + 0.9) / 2, df=len(df) - 1)
|
|
394
|
+
gumbel_se = Univar.gumbel_se(
|
|
395
|
+
std_sample=np.std(a=df[self.varfield].values),
|
|
396
|
+
n_sample=len(df),
|
|
397
|
+
tx=df["T(X)_Gumbel"],
|
|
398
|
+
)
|
|
399
|
+
# Standar error analysis
|
|
400
|
+
df["T(X)_Gumbel_SE90"] = t * gumbel_se
|
|
401
|
+
h_max = df[self.varfield] + df["T(X)_Gumbel_SE90"]
|
|
402
|
+
h_min = df[self.varfield] - df["T(X)_Gumbel_SE90"]
|
|
403
|
+
df["T(X)_Gumbel_P05"] = Univar.gumbel_tx(x=h_min, a=gumbel_a, b=gumbel_b)
|
|
404
|
+
df["T(X)_Gumbel_P95"] = Univar.gumbel_tx(x=h_max, a=gumbel_a, b=gumbel_b)
|
|
405
|
+
|
|
406
|
+
# -------- Return Period analysis
|
|
407
|
+
|
|
408
|
+
# evely spaced
|
|
409
|
+
txs = np.arange(2, 1001, step=1)
|
|
410
|
+
k = Univar.gumbel_freqfactor(txs)
|
|
411
|
+
|
|
412
|
+
# apply reverse formula
|
|
413
|
+
stages = np.mean(a=df[self.varfield]) + k * np.std(a=df[self.varfield])
|
|
414
|
+
|
|
415
|
+
# get standard error
|
|
416
|
+
gumbel_se = Univar.gumbel_se(
|
|
417
|
+
std_sample=np.std(a=df[self.varfield]), n_sample=len(df), tx=txs
|
|
418
|
+
)
|
|
419
|
+
# compute 50% uncertainty bands by Student's t distribution
|
|
420
|
+
t_50 = stats.t.ppf((1 + 0.5) / 2, df=len(df) - 1)
|
|
421
|
+
h_range_50 = t_50 * gumbel_se
|
|
422
|
+
|
|
423
|
+
# compute 90% uncertainty bands by Student's t distribution
|
|
424
|
+
t_90 = stats.t.ppf((1 + 0.9) / 2, df=len(df) - 1)
|
|
425
|
+
h_range_90 = t_90 * gumbel_se
|
|
426
|
+
|
|
427
|
+
# Set TX dataframe
|
|
428
|
+
df_tx = pd.DataFrame(
|
|
429
|
+
{
|
|
430
|
+
"T(X)_Gumbel": txs,
|
|
431
|
+
f"{self.varfield}": stages,
|
|
432
|
+
f"{self.varfield}_P25": stages - h_range_50,
|
|
433
|
+
f"{self.varfield}_P75": stages + h_range_50,
|
|
434
|
+
f"{self.varfield}_P05": stages - h_range_90,
|
|
435
|
+
f"{self.varfield}_P95": stages + h_range_90,
|
|
436
|
+
}
|
|
437
|
+
)
|
|
438
|
+
|
|
439
|
+
# -------- QQ plots
|
|
440
|
+
qq, qq_params = stats.probplot(
|
|
441
|
+
x=df[self.varfield].values, dist="gumbel_r", sparams=params
|
|
442
|
+
)
|
|
443
|
+
# Set QQ dataframe
|
|
444
|
+
df_qq = pd.DataFrame({self.varfield: qq[1], "T-Q": qq[0]})
|
|
445
|
+
|
|
446
|
+
# -------- Goodness of fit tests
|
|
447
|
+
ks_stat, ks_p_value = stats.kstest(
|
|
448
|
+
df[self.varfield].values, cdf="gumbel_r", args=params
|
|
449
|
+
)
|
|
450
|
+
# accept Null Hypothesis
|
|
451
|
+
is_gumbel = True
|
|
452
|
+
if ks_p_value <= 0.05:
|
|
453
|
+
is_gumbel = False
|
|
454
|
+
|
|
455
|
+
# -------- Metadata dataframe
|
|
456
|
+
df_meta = pd.DataFrame(
|
|
457
|
+
{
|
|
458
|
+
"Metadata": [
|
|
459
|
+
"N",
|
|
460
|
+
"Gumbel a",
|
|
461
|
+
"Gumbel b",
|
|
462
|
+
"KS-test s-value",
|
|
463
|
+
"KS-test p-value",
|
|
464
|
+
"KS-test Is Gumbel (NullH)",
|
|
465
|
+
"QQ-plot c0",
|
|
466
|
+
"QQ plot c1",
|
|
467
|
+
"QQ-plot r2",
|
|
468
|
+
],
|
|
469
|
+
"Value": [
|
|
470
|
+
len(df),
|
|
471
|
+
gumbel_a,
|
|
472
|
+
gumbel_b,
|
|
473
|
+
ks_stat,
|
|
474
|
+
ks_p_value,
|
|
475
|
+
is_gumbel,
|
|
476
|
+
qq_params[1],
|
|
477
|
+
qq_params[0],
|
|
478
|
+
qq_params[2],
|
|
479
|
+
],
|
|
480
|
+
}
|
|
481
|
+
)
|
|
482
|
+
|
|
483
|
+
return {"Data": df, "Data_T(X)": df_tx, "Data_QQ": df_qq, "Metadata": df_meta}
|
|
484
|
+
|
|
485
|
+
# PLOT METHODS
|
|
486
|
+
# ===================================================================
|
|
487
|
+
# Plotting methods
|
|
488
|
+
# todo [DRY] evaluate to discard or refactor for viewer styles
|
|
489
|
+
def plot_hist(
|
|
490
|
+
self,
|
|
491
|
+
bins=100,
|
|
492
|
+
colored=False,
|
|
493
|
+
annotated=False,
|
|
494
|
+
rule=None,
|
|
495
|
+
show=False,
|
|
496
|
+
folder="C:/sample",
|
|
497
|
+
filename="histogram",
|
|
498
|
+
specs=None,
|
|
499
|
+
dpi=300,
|
|
500
|
+
):
|
|
501
|
+
"""
|
|
502
|
+
Plot histogram of sample
|
|
503
|
+
|
|
504
|
+
:param bins: number of bins
|
|
505
|
+
:type bins: int
|
|
506
|
+
:param colored: Boolean to quantile-colored histogram
|
|
507
|
+
:type colored: bool
|
|
508
|
+
:param annotated: Boolean to plot stats texts over histogram
|
|
509
|
+
:type annotated: bool
|
|
510
|
+
:param rule: name of rule to compute bins
|
|
511
|
+
:type rule: str
|
|
512
|
+
:param show: Boolean to show instead of saving
|
|
513
|
+
:type show: bool
|
|
514
|
+
:param folder: output folder
|
|
515
|
+
:type folder: str
|
|
516
|
+
:param filename: image file name
|
|
517
|
+
:type filename: str
|
|
518
|
+
:param specs: specification dictionary
|
|
519
|
+
:type specs: dict
|
|
520
|
+
:param dpi: image resolution (default = 96)
|
|
521
|
+
:type dpi: int
|
|
522
|
+
"""
|
|
523
|
+
|
|
524
|
+
# get bins
|
|
525
|
+
if rule is None:
|
|
526
|
+
pass
|
|
527
|
+
else:
|
|
528
|
+
bins = self.get_nbins_by_rule(rule=rule)
|
|
529
|
+
|
|
530
|
+
# get specs
|
|
531
|
+
default_specs = {
|
|
532
|
+
"color": "tab:grey",
|
|
533
|
+
"title": "Histogram of {}".format(self.name),
|
|
534
|
+
"width": 4 * 1.618,
|
|
535
|
+
"height": 4,
|
|
536
|
+
"xlabel": "value",
|
|
537
|
+
"range": (0, 0.1),
|
|
538
|
+
"range_x": (
|
|
539
|
+
0.95 * np.min(self.data[self.varfield]),
|
|
540
|
+
1.05 * np.max(self.data[self.varfield]),
|
|
541
|
+
),
|
|
542
|
+
"subtitle": None,
|
|
543
|
+
"cmap": "viridis",
|
|
544
|
+
"colors": None,
|
|
545
|
+
"n_classes": 5,
|
|
546
|
+
"grid": False,
|
|
547
|
+
"quantiles": True,
|
|
548
|
+
}
|
|
549
|
+
# handle inputs specs
|
|
550
|
+
if specs is None:
|
|
551
|
+
pass
|
|
552
|
+
else: # override default
|
|
553
|
+
for k in specs:
|
|
554
|
+
default_specs[k] = specs[k]
|
|
555
|
+
specs = default_specs
|
|
556
|
+
|
|
557
|
+
# start plot
|
|
558
|
+
fig = plt.figure(figsize=(specs["width"], specs["height"])) # Width, Height
|
|
559
|
+
ax = plt.gca()
|
|
560
|
+
ax.set_position([0.15, 0.15, 0.75, 0.75])
|
|
561
|
+
if specs["subtitle"] is None:
|
|
562
|
+
plt.title(specs["title"])
|
|
563
|
+
else:
|
|
564
|
+
plt.title("{} | {}".format(specs["title"], specs["subtitle"]))
|
|
565
|
+
|
|
566
|
+
# Calculate quantiles
|
|
567
|
+
data = pd.Series(self.data[self.varfield])
|
|
568
|
+
# filter data to range x
|
|
569
|
+
data = data[data.between(specs["range_x"][0], specs["range_x"][1])]
|
|
570
|
+
if specs["quantiles"]:
|
|
571
|
+
quantiles = data.quantile(np.linspace(0, 1, specs["n_classes"] + 1))
|
|
572
|
+
else:
|
|
573
|
+
quantiles = pd.Series(
|
|
574
|
+
np.linspace(
|
|
575
|
+
start=data.min(), stop=data.max(), num=specs["n_classes"] + 1
|
|
576
|
+
)
|
|
577
|
+
)
|
|
578
|
+
|
|
579
|
+
# Determine the overall bin width you desire
|
|
580
|
+
bin_width = (
|
|
581
|
+
data.max() - data.min()
|
|
582
|
+
) / bins # for example, 40 equal width bins across the data range
|
|
583
|
+
# handle colored plots
|
|
584
|
+
if colored:
|
|
585
|
+
if specs["colors"] is None:
|
|
586
|
+
cmap = mpl.colormaps[specs["cmap"]]
|
|
587
|
+
colors = [cmap(i) for i in np.linspace(0, 1, len(quantiles) - 1)]
|
|
588
|
+
else:
|
|
589
|
+
colors = specs["colors"]
|
|
590
|
+
|
|
591
|
+
# Create the bins
|
|
592
|
+
binsv = np.arange(start=data.min(), stop=data.max(), step=bin_width)
|
|
593
|
+
binsv = np.append(
|
|
594
|
+
binsv, data.max()
|
|
595
|
+
) # Ensure the last bin includes the max value
|
|
596
|
+
|
|
597
|
+
# Plot the histogram
|
|
598
|
+
n, bins, patches = plt.hist(
|
|
599
|
+
self.data,
|
|
600
|
+
bins=binsv,
|
|
601
|
+
linewidth=0,
|
|
602
|
+
weights=np.ones(len(self.data)) / len(self.data),
|
|
603
|
+
)
|
|
604
|
+
|
|
605
|
+
# Color the bars based on the quantile
|
|
606
|
+
for patch, edge in zip(patches, bins[:-1]):
|
|
607
|
+
if edge < quantiles.values[1]:
|
|
608
|
+
plt.setp(patch, "facecolor", colors[0])
|
|
609
|
+
elif edge < quantiles.values[2]:
|
|
610
|
+
plt.setp(patch, "facecolor", colors[1])
|
|
611
|
+
elif edge < quantiles.values[3]:
|
|
612
|
+
plt.setp(patch, "facecolor", colors[2])
|
|
613
|
+
elif edge < quantiles.values[4]:
|
|
614
|
+
plt.setp(patch, "facecolor", colors[3])
|
|
615
|
+
else:
|
|
616
|
+
plt.setp(patch, "facecolor", colors[4])
|
|
617
|
+
else:
|
|
618
|
+
plt.hist(
|
|
619
|
+
self.data,
|
|
620
|
+
bins=bins,
|
|
621
|
+
weights=np.ones(len(self.data)) / len(self.data),
|
|
622
|
+
color=specs["color"],
|
|
623
|
+
)
|
|
624
|
+
|
|
625
|
+
plt.xlabel(specs["xlabel"])
|
|
626
|
+
plt.ylim(specs["range"])
|
|
627
|
+
plt.xlim(specs["range_x"])
|
|
628
|
+
plt.grid(specs["grid"])
|
|
629
|
+
# Optionally, add vertical lines for each quantile and annotate them
|
|
630
|
+
mu = np.mean(self.data)
|
|
631
|
+
plt.axvline(mu, color="r", linestyle="dashed", linewidth=1)
|
|
632
|
+
plt.text(
|
|
633
|
+
mu,
|
|
634
|
+
plt.gca().get_ylim()[1] * 0.92,
|
|
635
|
+
rf"$\mu$ = {mu:.1f}",
|
|
636
|
+
ha="left",
|
|
637
|
+
va="bottom",
|
|
638
|
+
fontsize=10,
|
|
639
|
+
rotation=0,
|
|
640
|
+
color="red",
|
|
641
|
+
)
|
|
642
|
+
if annotated:
|
|
643
|
+
aux = 0
|
|
644
|
+
for q in quantiles:
|
|
645
|
+
plt.axvline(q, color="k", linestyle="dashed", linewidth=1)
|
|
646
|
+
plt.text(
|
|
647
|
+
q,
|
|
648
|
+
plt.gca().get_ylim()[1] * (0.9 - aux),
|
|
649
|
+
f"{q:.1f}",
|
|
650
|
+
ha="right",
|
|
651
|
+
va="bottom",
|
|
652
|
+
fontsize=10,
|
|
653
|
+
rotation=0,
|
|
654
|
+
)
|
|
655
|
+
aux = aux + 0.05
|
|
656
|
+
# plt.tight_layout()
|
|
657
|
+
# show or save
|
|
658
|
+
if show:
|
|
659
|
+
plt.show()
|
|
660
|
+
else:
|
|
661
|
+
plt.savefig(
|
|
662
|
+
"{}/{}_{}.png".format(folder, self.name, filename),
|
|
663
|
+
dpi=dpi,
|
|
664
|
+
bbox_inches="tight",
|
|
665
|
+
)
|
|
666
|
+
|
|
667
|
+
# todo [evaluate]
|
|
668
|
+
def plot_qqplot(
|
|
669
|
+
self, show=True, folder="C:/sample", filename="qqplot", specs=None, dpi=300
|
|
670
|
+
):
|
|
671
|
+
"""
|
|
672
|
+
Plot Q-Q Plot on Normal distribution
|
|
673
|
+
|
|
674
|
+
:param show: Boolean to show instead of saving
|
|
675
|
+
:type show: bool
|
|
676
|
+
:param folder: output folder
|
|
677
|
+
:type folder: str
|
|
678
|
+
:param filename: image file name
|
|
679
|
+
:type filename: str
|
|
680
|
+
:param specs: specification dictionary
|
|
681
|
+
:type specs: dct
|
|
682
|
+
:param dpi: image resolution (default = 300)
|
|
683
|
+
:type dpi: int
|
|
684
|
+
:return: None
|
|
685
|
+
:rtype: None
|
|
686
|
+
"""
|
|
687
|
+
df_clean = self.data.dropna()
|
|
688
|
+
data = df_clean[self.varfield]
|
|
689
|
+
# get specs
|
|
690
|
+
default_specs = {
|
|
691
|
+
"color": "tab:grey",
|
|
692
|
+
"title": "Q-Q Plot of {}".format(self.name),
|
|
693
|
+
"width": 4 * 1.618,
|
|
694
|
+
"height": 4,
|
|
695
|
+
"xlabel": "value",
|
|
696
|
+
"range": (0.95 * np.min(data), 1.05 * np.max(data)),
|
|
697
|
+
"range_x": (-3, 3),
|
|
698
|
+
"subtitle": None,
|
|
699
|
+
}
|
|
700
|
+
# handle inputs specs
|
|
701
|
+
if specs is None:
|
|
702
|
+
pass
|
|
703
|
+
else: # override default
|
|
704
|
+
for k in specs:
|
|
705
|
+
default_specs[k] = specs[k]
|
|
706
|
+
specs = default_specs
|
|
707
|
+
|
|
708
|
+
# process quantiles
|
|
709
|
+
_df = Univar.qqplot(data=data)
|
|
710
|
+
# plot
|
|
711
|
+
fig = plt.figure(figsize=(specs["width"], specs["height"])) # Width, Height
|
|
712
|
+
# grid
|
|
713
|
+
gs = mpl.gridspec.GridSpec(
|
|
714
|
+
1, 1, wspace=0.4, hspace=0.5, left=0.1, bottom=0.15, top=0.9, right=0.95
|
|
715
|
+
) # nrows, ncols
|
|
716
|
+
# scatter plot
|
|
717
|
+
ax = fig.add_subplot(gs[0, 0])
|
|
718
|
+
plt.title(specs["title"])
|
|
719
|
+
plt.scatter(_df["T-Quantiles"], _df["Data"], marker=".", color="tab:grey")
|
|
720
|
+
plt.ylim([_df["Data"].min(), _df["Data"].max()])
|
|
721
|
+
plt.xlim(specs["range_x"])
|
|
722
|
+
plt.xlabel("Normal Theoretical Quantiles")
|
|
723
|
+
plt.ylabel("Data Empirical Quantiles")
|
|
724
|
+
plt.gca().set_aspect(
|
|
725
|
+
(specs["range_x"][1] - specs["range_x"][0])
|
|
726
|
+
/ (_df["Data"].max() - _df["Data"].min())
|
|
727
|
+
)
|
|
728
|
+
|
|
729
|
+
# show or save
|
|
730
|
+
if show:
|
|
731
|
+
plt.show()
|
|
732
|
+
else:
|
|
733
|
+
plt.savefig("{}/{}_{}.png".format(folder, self.name, filename), dpi=dpi)
|
|
734
|
+
|
|
735
|
+
def _plot(self, fig, gs, specs):
|
|
736
|
+
# todo [docstring]
|
|
737
|
+
|
|
738
|
+
# from matplotlib.ticker import PercentFormatter # Import PercentFormatter
|
|
739
|
+
formatter = mticker.ScalarFormatter(useOffset=False, useMathText=False)
|
|
740
|
+
formatter.set_scientific(False) # Ensure scientific notation is off
|
|
741
|
+
|
|
742
|
+
# AUX DATA
|
|
743
|
+
# -------------------------------------------------------------------
|
|
744
|
+
if self.stats_df is None:
|
|
745
|
+
self.stats_df = self.get_basic_stats()
|
|
746
|
+
if self.weibull_df is None:
|
|
747
|
+
self.weibull_df = self.get_cdf_weibull()
|
|
748
|
+
# get mean
|
|
749
|
+
plot_mean = specs["plot_mean"]
|
|
750
|
+
y_mu = np.nanmean(self.data[self.varfield])
|
|
751
|
+
|
|
752
|
+
# RANGE
|
|
753
|
+
# -------------------------------------------------------------------
|
|
754
|
+
range_data = specs["range"]
|
|
755
|
+
# handle if none
|
|
756
|
+
if specs["range"] is None:
|
|
757
|
+
p_low = self.stats_df.loc[
|
|
758
|
+
self.stats_df["statistic"] == "p01", "value"
|
|
759
|
+
].values[0]
|
|
760
|
+
p_hi = self.stats_df.loc[
|
|
761
|
+
self.stats_df["statistic"] == "p99", "value"
|
|
762
|
+
].values[0]
|
|
763
|
+
rng = p_hi - p_low
|
|
764
|
+
rng_margin = rng * 0.1
|
|
765
|
+
range_data = (p_low - rng_margin, p_hi + rng_margin)
|
|
766
|
+
specs["range"] = range_data
|
|
767
|
+
|
|
768
|
+
# TITLE
|
|
769
|
+
# -------------------------------------------------------------------
|
|
770
|
+
if specs["title"] is not None:
|
|
771
|
+
plt.suptitle(specs["title"], fontsize=9)
|
|
772
|
+
|
|
773
|
+
# AXES
|
|
774
|
+
# -------------------------------------------------------------------
|
|
775
|
+
fig = self._build_axes(fig=fig, gs=gs, specs=specs)
|
|
776
|
+
all_axes = fig.get_axes()
|
|
777
|
+
|
|
778
|
+
# DATA
|
|
779
|
+
ax_data = False
|
|
780
|
+
if specs["ax_data"]:
|
|
781
|
+
ax_data = all_axes[specs["ax_data"]]
|
|
782
|
+
|
|
783
|
+
# HIST
|
|
784
|
+
ax_histh = False
|
|
785
|
+
if specs["ax_histh"]:
|
|
786
|
+
ax_histh = all_axes[specs["ax_histh"]]
|
|
787
|
+
|
|
788
|
+
# CDF
|
|
789
|
+
ax_cdf = False
|
|
790
|
+
if specs["ax_cdf"]:
|
|
791
|
+
ax_cdf = all_axes[specs["ax_cdf"]]
|
|
792
|
+
|
|
793
|
+
# CLEANUP
|
|
794
|
+
# -------------------------------------------------------------------
|
|
795
|
+
df_clean = self.data.dropna()
|
|
796
|
+
data = df_clean[self.varfield].values
|
|
797
|
+
|
|
798
|
+
# PLOT DATA
|
|
799
|
+
# -------------------------------------------------------------------
|
|
800
|
+
Univar.plot_scatter(
|
|
801
|
+
data=data,
|
|
802
|
+
ax=ax_data,
|
|
803
|
+
specs=specs,
|
|
804
|
+
formatter=formatter,
|
|
805
|
+
x_factor=2,
|
|
806
|
+
)
|
|
807
|
+
|
|
808
|
+
# PLOT HIST
|
|
809
|
+
# -------------------------------------------------------------------
|
|
810
|
+
Univar.plot_histh(
|
|
811
|
+
data=data,
|
|
812
|
+
ax=ax_histh,
|
|
813
|
+
specs=specs,
|
|
814
|
+
formatter=formatter,
|
|
815
|
+
)
|
|
816
|
+
|
|
817
|
+
# PLOT CDF
|
|
818
|
+
# -------------------------------------------------------------------
|
|
819
|
+
Univar.plot_cdf(
|
|
820
|
+
cdf_df=self.weibull_df,
|
|
821
|
+
ax=ax_cdf,
|
|
822
|
+
specs=specs,
|
|
823
|
+
formatter=formatter,
|
|
824
|
+
)
|
|
825
|
+
|
|
826
|
+
# PLOT STATS
|
|
827
|
+
# -------------------------------------------------------------------
|
|
828
|
+
if specs["mode"] == "full":
|
|
829
|
+
x_stats = specs["x_stats"]
|
|
830
|
+
x_stats_sep = specs["x_stats_sep"]
|
|
831
|
+
y_stats = specs["y_stats"]
|
|
832
|
+
fig = Univar.plot_stats(
|
|
833
|
+
fig=fig, stats_df=self.stats_df, x=x_stats, y=y_stats, sep=x_stats_sep
|
|
834
|
+
)
|
|
835
|
+
|
|
836
|
+
# RETURN FIG
|
|
837
|
+
# -------------------------------------------------------------------
|
|
838
|
+
return fig
|
|
839
|
+
|
|
840
|
+
def _build_axes(self, fig, gs, specs):
|
|
841
|
+
# todo [docstring]
|
|
842
|
+
# ------------ setup axes ------------
|
|
843
|
+
if specs["mode"] == "full":
|
|
844
|
+
fig.add_subplot(gs[2:10, 2:10])
|
|
845
|
+
fig.add_subplot(gs[2:10, 13:18])
|
|
846
|
+
fig.add_subplot(gs[2:10, 19:])
|
|
847
|
+
elif specs["mode"] == "mini":
|
|
848
|
+
fig.add_subplot(gs[2:10, 2:10])
|
|
849
|
+
fig.add_subplot(gs[2:10, 13:18])
|
|
850
|
+
fig.add_subplot(gs[2:10, 19:])
|
|
851
|
+
else:
|
|
852
|
+
fig.add_subplot(gs[2:10, 2:10])
|
|
853
|
+
fig.add_subplot(gs[2:10, 13:18])
|
|
854
|
+
fig.add_subplot(gs[2:10, 19:])
|
|
855
|
+
return fig
|
|
856
|
+
|
|
857
|
+
def _get_fig_specs(self):
|
|
858
|
+
# todo [docstring]
|
|
859
|
+
# handle specs
|
|
860
|
+
specs = self.view_specs.copy()
|
|
861
|
+
|
|
862
|
+
# handle mode
|
|
863
|
+
mode = specs["mode"]
|
|
864
|
+
if mode == "full":
|
|
865
|
+
specs_aux = {
|
|
866
|
+
"ncols": 24,
|
|
867
|
+
"nrows": 16,
|
|
868
|
+
"width": viewer.FIG_SIZES["M"]["w"],
|
|
869
|
+
"height": viewer.FIG_SIZES["M"]["h"],
|
|
870
|
+
}
|
|
871
|
+
else:
|
|
872
|
+
specs_aux = {
|
|
873
|
+
"ncols": 24,
|
|
874
|
+
"nrows": 12,
|
|
875
|
+
"width": viewer.FIG_SIZES["M2"]["w"],
|
|
876
|
+
"height": viewer.FIG_SIZES["M2"]["h"],
|
|
877
|
+
}
|
|
878
|
+
# update sizes
|
|
879
|
+
specs.update(specs_aux)
|
|
880
|
+
|
|
881
|
+
# update gridspecs
|
|
882
|
+
specs.update(viewer.GRID_SPECS)
|
|
883
|
+
|
|
884
|
+
return specs
|
|
885
|
+
|
|
886
|
+
def _set_view_specs(self):
|
|
887
|
+
"""
|
|
888
|
+
Set view specifications. Expected to overwrite superior methods.
|
|
889
|
+
|
|
890
|
+
:return: None
|
|
891
|
+
:rtype: None
|
|
892
|
+
"""
|
|
893
|
+
super()._set_view_specs()
|
|
894
|
+
self.view_specs.update(
|
|
895
|
+
{
|
|
896
|
+
# layout
|
|
897
|
+
"style": "wien",
|
|
898
|
+
"mode": "full",
|
|
899
|
+
"width": viewer.FIG_SIZES["M"]["w"],
|
|
900
|
+
"height": viewer.FIG_SIZES["M"]["h"],
|
|
901
|
+
# titles
|
|
902
|
+
"subtitle_data": "Distribution",
|
|
903
|
+
"subtitle_hist": "Histogram",
|
|
904
|
+
"subtitle_cdf": "CDF",
|
|
905
|
+
# fields
|
|
906
|
+
"yvar_field": self.varfield,
|
|
907
|
+
"xvar": "i",
|
|
908
|
+
"yvar": self.varname,
|
|
909
|
+
# axis labels
|
|
910
|
+
"xlabel": self.varname,
|
|
911
|
+
"xlabel_cdf": "P(X)",
|
|
912
|
+
"ylabel": self.units,
|
|
913
|
+
# data labels
|
|
914
|
+
"data_label": self.name,
|
|
915
|
+
"data_legend": False,
|
|
916
|
+
# colors
|
|
917
|
+
"color_hist": "tab:grey",
|
|
918
|
+
"color_cdf": "black",
|
|
919
|
+
"color_mean": "red",
|
|
920
|
+
"color_mean_text": None,
|
|
921
|
+
# cmaps
|
|
922
|
+
"scheme_cmap": None,
|
|
923
|
+
"cmap": "viridis",
|
|
924
|
+
"colorize_scatter": False,
|
|
925
|
+
# alphas
|
|
926
|
+
"alpha": 0.4,
|
|
927
|
+
"alpha_hist": 1,
|
|
928
|
+
"alpha_cdf": 1,
|
|
929
|
+
# ranges
|
|
930
|
+
"range": None,
|
|
931
|
+
"range_x": None,
|
|
932
|
+
# mean
|
|
933
|
+
"plot_mean": True,
|
|
934
|
+
"linestyle_mean": "solid",
|
|
935
|
+
"pad_mean": 1,
|
|
936
|
+
# histogram
|
|
937
|
+
"hist_density": True, # todo check this may be deprecated
|
|
938
|
+
"bins": 20,
|
|
939
|
+
"bins_density": False,
|
|
940
|
+
# axes
|
|
941
|
+
"ax_data": 0,
|
|
942
|
+
"ax_histh": 1,
|
|
943
|
+
"ax_cdf": 2,
|
|
944
|
+
# stats locators
|
|
945
|
+
"x_stats": 0.01,
|
|
946
|
+
"x_stats_sep": 0.2,
|
|
947
|
+
"y_stats": 0.3,
|
|
948
|
+
# zorder
|
|
949
|
+
"zorder_data": 1,
|
|
950
|
+
"zorder_histh": 1,
|
|
951
|
+
"zorder_cdf": 1,
|
|
952
|
+
}
|
|
953
|
+
)
|
|
954
|
+
return None
|
|
955
|
+
|
|
956
|
+
def view(self, show=True, return_fig=False):
|
|
957
|
+
# todo [docstring]
|
|
958
|
+
|
|
959
|
+
# SPECS
|
|
960
|
+
# -------------------------------------------------------------------
|
|
961
|
+
specs = self._get_fig_specs()
|
|
962
|
+
|
|
963
|
+
# BUILD
|
|
964
|
+
# -------------------------------------------------------------------
|
|
965
|
+
# build from specs
|
|
966
|
+
fig, gs = viewer.build_fig(specs=specs)
|
|
967
|
+
|
|
968
|
+
# PLOT
|
|
969
|
+
# -------------------------------------------------------------------
|
|
970
|
+
fig = self._plot(fig=fig, gs=gs, specs=specs)
|
|
971
|
+
|
|
972
|
+
# SHIP
|
|
973
|
+
# -------------------------------------------------------------------
|
|
974
|
+
# create path
|
|
975
|
+
file_path = "{}/{}.{}".format(
|
|
976
|
+
specs["folder"], specs["filename"], specs["fig_format"]
|
|
977
|
+
)
|
|
978
|
+
if return_fig:
|
|
979
|
+
return fig
|
|
980
|
+
else:
|
|
981
|
+
viewer.ship_fig(fig=fig, show=show, file_output=file_path, dpi=specs["dpi"])
|
|
982
|
+
|
|
983
|
+
# STATIC METHODS
|
|
984
|
+
# ===================================================================
|
|
985
|
+
|
|
986
|
+
@staticmethod
|
|
987
|
+
def plot_mean(ax, y_mu, xmin, xmax, specs):
|
|
988
|
+
# todo [docstring]
|
|
989
|
+
if not ax:
|
|
990
|
+
return None
|
|
991
|
+
|
|
992
|
+
ax.hlines(
|
|
993
|
+
y=y_mu,
|
|
994
|
+
xmin=xmin,
|
|
995
|
+
xmax=xmax,
|
|
996
|
+
colors=specs["color_mean"],
|
|
997
|
+
linestyle=specs["linestyle_mean"],
|
|
998
|
+
)
|
|
999
|
+
|
|
1000
|
+
if specs["color_mean_text"] is None:
|
|
1001
|
+
specs["color_mean_text"] = "black"
|
|
1002
|
+
|
|
1003
|
+
pad = specs["pad_mean"] * " "
|
|
1004
|
+
s = r"{}$\mu$ = {}".format(pad, round(y_mu, 2))
|
|
1005
|
+
ax.annotate(
|
|
1006
|
+
s,
|
|
1007
|
+
xy=(0, y_mu),
|
|
1008
|
+
xytext=(1, 3),
|
|
1009
|
+
textcoords="offset points",
|
|
1010
|
+
color=specs["color_mean_text"],
|
|
1011
|
+
)
|
|
1012
|
+
return None
|
|
1013
|
+
|
|
1014
|
+
@staticmethod
|
|
1015
|
+
def plot_scatter(data, ax, specs, formatter=None, x_factor=4):
|
|
1016
|
+
# todo [docstring]
|
|
1017
|
+
if not ax:
|
|
1018
|
+
return None
|
|
1019
|
+
x_values = np.arange(len(data))
|
|
1020
|
+
# handle scheme
|
|
1021
|
+
if specs["colorize_scatter"]:
|
|
1022
|
+
color_data = Univar.classify(
|
|
1023
|
+
data=data, n_classes=5, scheme=specs["scheme_cmap"]
|
|
1024
|
+
)
|
|
1025
|
+
color_cmap = viewer.get_discrete_cmap(
|
|
1026
|
+
n_classes=5, base_cmap_name=specs["cmap"]
|
|
1027
|
+
)
|
|
1028
|
+
# plot
|
|
1029
|
+
ax.scatter(
|
|
1030
|
+
x_values,
|
|
1031
|
+
data,
|
|
1032
|
+
marker=".",
|
|
1033
|
+
c=color_data,
|
|
1034
|
+
cmap=color_cmap,
|
|
1035
|
+
alpha=specs["alpha"],
|
|
1036
|
+
s=0.8 * viewer.MM_TO_PT,
|
|
1037
|
+
zorder=specs["zorder_data"],
|
|
1038
|
+
)
|
|
1039
|
+
else:
|
|
1040
|
+
ax.scatter(
|
|
1041
|
+
x_values,
|
|
1042
|
+
data,
|
|
1043
|
+
marker=".",
|
|
1044
|
+
color=specs["color"],
|
|
1045
|
+
alpha=specs["alpha"],
|
|
1046
|
+
s=0.8 * viewer.MM_TO_PT,
|
|
1047
|
+
zorder=specs["zorder_data"],
|
|
1048
|
+
)
|
|
1049
|
+
|
|
1050
|
+
if specs["subtitle_data"] is not None:
|
|
1051
|
+
ax.set_title(specs["subtitle_data"], loc="left")
|
|
1052
|
+
|
|
1053
|
+
ax.set_ylim(specs["range"])
|
|
1054
|
+
ax.set_xlabel(specs["xlabel"])
|
|
1055
|
+
ax.set_ylabel(specs["ylabel"])
|
|
1056
|
+
ax.grid(axis="x", visible=False)
|
|
1057
|
+
|
|
1058
|
+
if formatter is not None:
|
|
1059
|
+
ax.yaxis.set_major_formatter(formatter)
|
|
1060
|
+
|
|
1061
|
+
# handle x axis
|
|
1062
|
+
n_size = len(data)
|
|
1063
|
+
x_low = -x_factor * n_size
|
|
1064
|
+
x_hi = n_size + (x_factor * n_size)
|
|
1065
|
+
ax.set_xlim(x_low, x_hi)
|
|
1066
|
+
ax.set_xticks([])
|
|
1067
|
+
|
|
1068
|
+
# handle cbars
|
|
1069
|
+
# ------------ cbar plot ------------
|
|
1070
|
+
if specs["colorize_scatter"]:
|
|
1071
|
+
Univar.plot_cbar(
|
|
1072
|
+
data=data,
|
|
1073
|
+
ax=ax,
|
|
1074
|
+
scheme=specs["scheme_cmap"],
|
|
1075
|
+
cmap=specs["cmap"],
|
|
1076
|
+
side="right",
|
|
1077
|
+
)
|
|
1078
|
+
|
|
1079
|
+
# PLOT MEAN LINE
|
|
1080
|
+
# -------------------------------------------------------------------
|
|
1081
|
+
if specs["plot_mean"]:
|
|
1082
|
+
# data
|
|
1083
|
+
Univar.plot_mean(
|
|
1084
|
+
ax,
|
|
1085
|
+
y_mu=np.nanmean(data),
|
|
1086
|
+
xmin=x_low,
|
|
1087
|
+
xmax=x_hi,
|
|
1088
|
+
specs=specs,
|
|
1089
|
+
)
|
|
1090
|
+
|
|
1091
|
+
return (x_low, x_hi)
|
|
1092
|
+
|
|
1093
|
+
@staticmethod
|
|
1094
|
+
def plot_histh(data, ax, specs, formatter=None):
|
|
1095
|
+
# todo [docstring]
|
|
1096
|
+
# todo dry --- evaluate to remove the range_data parameter
|
|
1097
|
+
# ------- local functions --------
|
|
1098
|
+
def _get_max(std_values, data):
|
|
1099
|
+
data_max = np.max(data)
|
|
1100
|
+
closest_max_index = np.argmin(np.abs(np.array(std_values) - data_max))
|
|
1101
|
+
_id = closest_max_index + 1
|
|
1102
|
+
if _id >= len(std_values):
|
|
1103
|
+
_id = len(std_values) - 1
|
|
1104
|
+
return std_values[_id]
|
|
1105
|
+
|
|
1106
|
+
h = None
|
|
1107
|
+
|
|
1108
|
+
if not ax:
|
|
1109
|
+
return None
|
|
1110
|
+
|
|
1111
|
+
# Standard x-axis maximum values for hist
|
|
1112
|
+
sdmax1 = list(np.linspace(1, 9, 9) / 100)
|
|
1113
|
+
sdmax2 = list(np.linspace(10, 100, 10) / 100)
|
|
1114
|
+
standard_max_values = sdmax1 + sdmax2
|
|
1115
|
+
|
|
1116
|
+
# handle data range
|
|
1117
|
+
data_range = specs["range"]
|
|
1118
|
+
if data_range is None:
|
|
1119
|
+
data_range = [np.nanmin(data), np.nanmax(data)]
|
|
1120
|
+
|
|
1121
|
+
# handle bins
|
|
1122
|
+
n_bins = specs["bins"]
|
|
1123
|
+
if specs["bins_density"]:
|
|
1124
|
+
n_units = n_bins
|
|
1125
|
+
d_range = [np.nanmin(data), np.nanmax(data)]
|
|
1126
|
+
n_bins = Univar.get_nbins_density(n_units=n_units, data_range=d_range)
|
|
1127
|
+
|
|
1128
|
+
# plot horizontal hist
|
|
1129
|
+
h = ax.hist(
|
|
1130
|
+
data,
|
|
1131
|
+
bins=n_bins,
|
|
1132
|
+
color=specs["color_hist"],
|
|
1133
|
+
alpha=specs["alpha_hist"],
|
|
1134
|
+
orientation="horizontal",
|
|
1135
|
+
weights=np.ones(len(data)) / len(data),
|
|
1136
|
+
zorder=specs["zorder_histh"],
|
|
1137
|
+
)
|
|
1138
|
+
ax.xaxis.set_major_formatter(mticker.PercentFormatter(1))
|
|
1139
|
+
if specs["subtitle_hist"] is not None:
|
|
1140
|
+
ax.set_title(specs["subtitle_hist"], loc="left")
|
|
1141
|
+
|
|
1142
|
+
# Get the maximum value from the data
|
|
1143
|
+
xmax = _get_max(standard_max_values, h[0]) * 1.1
|
|
1144
|
+
ax.set_xlim(0, xmax)
|
|
1145
|
+
ax.set_ylim(data_range)
|
|
1146
|
+
ax.set_xlabel("%")
|
|
1147
|
+
ax.set_ylabel(specs["ylabel"])
|
|
1148
|
+
if formatter is not None:
|
|
1149
|
+
ax.yaxis.set_major_formatter(formatter)
|
|
1150
|
+
|
|
1151
|
+
# PLOT MEAN
|
|
1152
|
+
# --------------------------------------------------
|
|
1153
|
+
if specs["plot_mean"]:
|
|
1154
|
+
# hist
|
|
1155
|
+
Univar.plot_mean(
|
|
1156
|
+
ax,
|
|
1157
|
+
y_mu=np.nanmean(data),
|
|
1158
|
+
xmin=0,
|
|
1159
|
+
xmax=1,
|
|
1160
|
+
specs=specs,
|
|
1161
|
+
)
|
|
1162
|
+
|
|
1163
|
+
return h
|
|
1164
|
+
|
|
1165
|
+
@staticmethod
|
|
1166
|
+
def plot_cdf(cdf_df, ax, specs, formatter=None, px_field="P(X)", var_field="Data"):
|
|
1167
|
+
# todo [docstring]
|
|
1168
|
+
|
|
1169
|
+
if not ax:
|
|
1170
|
+
return None
|
|
1171
|
+
|
|
1172
|
+
# PLOT CDF
|
|
1173
|
+
# --------------------------------------------------
|
|
1174
|
+
ax.plot(
|
|
1175
|
+
cdf_df[px_field],
|
|
1176
|
+
cdf_df[var_field],
|
|
1177
|
+
color=specs["color_cdf"],
|
|
1178
|
+
alpha=specs["alpha_cdf"],
|
|
1179
|
+
zorder=specs["zorder_cdf"],
|
|
1180
|
+
)
|
|
1181
|
+
|
|
1182
|
+
if specs["subtitle_cdf"] is not None:
|
|
1183
|
+
ax.set_title(specs["subtitle_cdf"], loc="left")
|
|
1184
|
+
|
|
1185
|
+
ax.set_xlabel(specs["xlabel_cdf"])
|
|
1186
|
+
ax.set_xlim(-0.05, 1.05)
|
|
1187
|
+
ax.set_ylim(specs["range"])
|
|
1188
|
+
|
|
1189
|
+
if formatter is not None:
|
|
1190
|
+
ax.yaxis.set_major_formatter(formatter)
|
|
1191
|
+
ax.set_yticklabels([])
|
|
1192
|
+
ax.xaxis.set_major_formatter(mticker.PercentFormatter(1))
|
|
1193
|
+
|
|
1194
|
+
# PLOT MEAN
|
|
1195
|
+
# --------------------------------------------------
|
|
1196
|
+
if specs["plot_mean"]:
|
|
1197
|
+
Univar.plot_mean(
|
|
1198
|
+
ax,
|
|
1199
|
+
y_mu=cdf_df[var_field].mean(),
|
|
1200
|
+
xmin=-0.05,
|
|
1201
|
+
xmax=1.05,
|
|
1202
|
+
specs=specs,
|
|
1203
|
+
)
|
|
1204
|
+
|
|
1205
|
+
return None
|
|
1206
|
+
|
|
1207
|
+
@staticmethod
|
|
1208
|
+
def plot_stats(fig, stats_df, x=0.5, y=0.4, sep=0.2):
|
|
1209
|
+
# todo [docstring]
|
|
1210
|
+
|
|
1211
|
+
# Helper function to plot a single column of stats
|
|
1212
|
+
def _plot_column(x_position, stats_list, start_y):
|
|
1213
|
+
current_y = start_y
|
|
1214
|
+
for stat in stats_list:
|
|
1215
|
+
# Handle "Median" special case if it's coming from ls_stats that uses it
|
|
1216
|
+
display_stat = "p50" if stat == "median" else stat
|
|
1217
|
+
|
|
1218
|
+
# Ensure the stat exists in the DataFrame before attempting to access
|
|
1219
|
+
stat_row = stats_df.loc[stats_df["statistic"] == display_stat]
|
|
1220
|
+
if not stat_row.empty:
|
|
1221
|
+
s_value = stat_row["value"].values[0]
|
|
1222
|
+
s_head = stat # Use the original name for display if needed, or display_stat
|
|
1223
|
+
|
|
1224
|
+
# Determine formatting based on value magnitude
|
|
1225
|
+
if abs(s_value) > 1_000_000:
|
|
1226
|
+
s_line = "{:>10}: {:<10.2e}".format(s_head, s_value)
|
|
1227
|
+
elif stat == "count":
|
|
1228
|
+
s_line = "{:>10}: {:<10}".format(s_head, int(s_value))
|
|
1229
|
+
else:
|
|
1230
|
+
s_line = "{:>10}: {:<10.2f}".format(s_head, s_value)
|
|
1231
|
+
|
|
1232
|
+
current_y -= n_step
|
|
1233
|
+
plt.text(
|
|
1234
|
+
x=x_position,
|
|
1235
|
+
y=current_y,
|
|
1236
|
+
s=s_line,
|
|
1237
|
+
fontdict={"family": "monospace"},
|
|
1238
|
+
transform=fig.transFigure,
|
|
1239
|
+
)
|
|
1240
|
+
else:
|
|
1241
|
+
# Optional: Handle cases where a stat might not be found
|
|
1242
|
+
# print(f"Warning: Statistic '{display_stat}' not found in DataFrame.")
|
|
1243
|
+
pass
|
|
1244
|
+
|
|
1245
|
+
n_step = 0.03
|
|
1246
|
+
|
|
1247
|
+
# First column of statistics
|
|
1248
|
+
ls_stats_col1 = [
|
|
1249
|
+
"count",
|
|
1250
|
+
"sum",
|
|
1251
|
+
"mean",
|
|
1252
|
+
"median", # This will be mapped to "P50" internally by the helper
|
|
1253
|
+
"sd",
|
|
1254
|
+
"min",
|
|
1255
|
+
"max",
|
|
1256
|
+
]
|
|
1257
|
+
_plot_column(x, ls_stats_col1, y - 0.01)
|
|
1258
|
+
|
|
1259
|
+
# Second column of statistics
|
|
1260
|
+
ls_stats_col2 = [
|
|
1261
|
+
"p05",
|
|
1262
|
+
"p10",
|
|
1263
|
+
"p25",
|
|
1264
|
+
"p50",
|
|
1265
|
+
"p75",
|
|
1266
|
+
"p90",
|
|
1267
|
+
"p95",
|
|
1268
|
+
]
|
|
1269
|
+
_plot_column(x + sep, ls_stats_col2, y - 0.01)
|
|
1270
|
+
|
|
1271
|
+
return fig
|
|
1272
|
+
|
|
1273
|
+
@staticmethod
|
|
1274
|
+
def plot_cbar(
|
|
1275
|
+
data, ax, scheme, cmap="viridis", n_classes=5, side="right", width_factor=16
|
|
1276
|
+
):
|
|
1277
|
+
# todo [docstring]
|
|
1278
|
+
|
|
1279
|
+
# handle bins
|
|
1280
|
+
bins = Univar.get_bins(data=data, n_bins=n_classes, scheme=scheme)
|
|
1281
|
+
|
|
1282
|
+
# handle egde cases
|
|
1283
|
+
n_bins = len(bins)
|
|
1284
|
+
if n_bins <= n_classes:
|
|
1285
|
+
n_classes = n_bins - 1
|
|
1286
|
+
|
|
1287
|
+
# handle colors
|
|
1288
|
+
custom_cmap = viewer.get_discrete_cmap(n_classes=n_classes, base_cmap_name=cmap)
|
|
1289
|
+
colors = []
|
|
1290
|
+
for i in range(n_classes):
|
|
1291
|
+
colors.append(custom_cmap.colors[i])
|
|
1292
|
+
|
|
1293
|
+
# handle position
|
|
1294
|
+
xlims = ax.get_xlim()
|
|
1295
|
+
bar_w = abs(xlims[1] - xlims[0]) / width_factor
|
|
1296
|
+
if side == "left":
|
|
1297
|
+
vrange = [xlims[0], xlims[0] + bar_w]
|
|
1298
|
+
elif side == "right":
|
|
1299
|
+
vrange = [xlims[1] - bar_w, xlims[1]]
|
|
1300
|
+
else:
|
|
1301
|
+
vrange = [xlims[0], xlims[0] + bar_w]
|
|
1302
|
+
|
|
1303
|
+
# plotting loop
|
|
1304
|
+
n_classes = len(colors)
|
|
1305
|
+
for i in range(n_classes):
|
|
1306
|
+
lower_bound = bins[i]
|
|
1307
|
+
upper_bound = bins[i + 1]
|
|
1308
|
+
color = colors[i]
|
|
1309
|
+
ax.fill_between(
|
|
1310
|
+
vrange, lower_bound, upper_bound, color=color, edgecolor="none"
|
|
1311
|
+
)
|
|
1312
|
+
|
|
1313
|
+
return None
|
|
1314
|
+
|
|
1315
|
+
@staticmethod
|
|
1316
|
+
def test_distribution(test_name, stat, p, clevel=0.95, distr="normal"):
|
|
1317
|
+
"""
|
|
1318
|
+
Util function for statistical testing
|
|
1319
|
+
|
|
1320
|
+
:param test_name: name of test
|
|
1321
|
+
:type test_name: str
|
|
1322
|
+
:param stat: statistic
|
|
1323
|
+
:type stat: float
|
|
1324
|
+
:param p: p-value
|
|
1325
|
+
:type p: float
|
|
1326
|
+
:param clevel: confidence level
|
|
1327
|
+
:type clevel: float
|
|
1328
|
+
:param distr: name of distribution
|
|
1329
|
+
:type distr: str
|
|
1330
|
+
:return: summary of test
|
|
1331
|
+
:rtype: dict
|
|
1332
|
+
"""
|
|
1333
|
+
# built output dict
|
|
1334
|
+
dct_out = {
|
|
1335
|
+
"Test": test_name,
|
|
1336
|
+
"Statistic": stat,
|
|
1337
|
+
"p-value": p,
|
|
1338
|
+
"Confidence": clevel,
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
if p > (1 - clevel):
|
|
1342
|
+
dct_out["is_{}".format(distr.lower())] = True
|
|
1343
|
+
else:
|
|
1344
|
+
dct_out["is_{}".format(distr.lower())] = False
|
|
1345
|
+
|
|
1346
|
+
return dct_out
|
|
1347
|
+
|
|
1348
|
+
@staticmethod
|
|
1349
|
+
def test_normality_ks(data, clevel=0.95):
|
|
1350
|
+
"""
|
|
1351
|
+
Test for normality using the Kolmogorov-Smirnov test
|
|
1352
|
+
|
|
1353
|
+
Kolmogorov-Smirnov Test: This test compares the observed distribution with
|
|
1354
|
+
the expected normal distribution using a test statistic and a p-value.
|
|
1355
|
+
A p-value less than 0.05 indicates that the null hypothesis should be rejected.
|
|
1356
|
+
|
|
1357
|
+
:param data: vector of data without nan values
|
|
1358
|
+
:type data: :class:`numpy.ndarray`
|
|
1359
|
+
:return: test result dictionary. Keys: Statistic, p-value and Is normal
|
|
1360
|
+
:rtype: dct
|
|
1361
|
+
"""
|
|
1362
|
+
from scipy.stats import kstest
|
|
1363
|
+
|
|
1364
|
+
vct_norm = (data - np.mean(data)) / np.std(data)
|
|
1365
|
+
|
|
1366
|
+
# test
|
|
1367
|
+
result = kstest(vct_norm, "norm")
|
|
1368
|
+
return Univar.test_distribution(
|
|
1369
|
+
test_name="Kolmogorov-Smirnov",
|
|
1370
|
+
stat=result.statistic,
|
|
1371
|
+
p=result.pvalue,
|
|
1372
|
+
clevel=clevel,
|
|
1373
|
+
distr="normal",
|
|
1374
|
+
)
|
|
1375
|
+
|
|
1376
|
+
@staticmethod
|
|
1377
|
+
def test_normality_sw(data, clevel=0.95):
|
|
1378
|
+
"""
|
|
1379
|
+
Test for normality using the Shapiro-Wilk test.
|
|
1380
|
+
|
|
1381
|
+
:param data: vector of data without nan values
|
|
1382
|
+
:type data: :class:`numpy.ndarray`
|
|
1383
|
+
:return: test result dictionary. Keys: Statistic, p-value and Is normal
|
|
1384
|
+
:rtype: dct
|
|
1385
|
+
"""
|
|
1386
|
+
from scipy.stats import shapiro
|
|
1387
|
+
|
|
1388
|
+
# test
|
|
1389
|
+
stat, p = shapiro(data)
|
|
1390
|
+
|
|
1391
|
+
return Univar.test_distribution(
|
|
1392
|
+
test_name="Shapiro-Wilk", stat=stat, p=p, clevel=clevel, distr="normal"
|
|
1393
|
+
)
|
|
1394
|
+
|
|
1395
|
+
@staticmethod
|
|
1396
|
+
def test_normality_dp(data, clevel=0.95):
|
|
1397
|
+
"""
|
|
1398
|
+
Test for normality using the D'Agostino-Pearson test.
|
|
1399
|
+
|
|
1400
|
+
:param data: vector of data without nan values
|
|
1401
|
+
:type data: :class:`numpy.ndarray`
|
|
1402
|
+
:return: test result dictionary. Keys: Statistic, p-value and Is normal
|
|
1403
|
+
:rtype: dct
|
|
1404
|
+
"""
|
|
1405
|
+
from scipy.stats import normaltest
|
|
1406
|
+
|
|
1407
|
+
# test
|
|
1408
|
+
stat, p = normaltest(data)
|
|
1409
|
+
|
|
1410
|
+
return Univar.test_distribution(
|
|
1411
|
+
test_name="D'Agostino-Pearson",
|
|
1412
|
+
stat=stat,
|
|
1413
|
+
p=p,
|
|
1414
|
+
clevel=clevel,
|
|
1415
|
+
distr="normal",
|
|
1416
|
+
)
|
|
1417
|
+
|
|
1418
|
+
@staticmethod
|
|
1419
|
+
def get_tx(fx):
|
|
1420
|
+
"""
|
|
1421
|
+
Simple function for computing the Return Period from the CDF
|
|
1422
|
+
|
|
1423
|
+
:param fx: CDF (FX)
|
|
1424
|
+
:type fx: float | :class:`numpy.ndarray`
|
|
1425
|
+
:return: Return Period
|
|
1426
|
+
:rtype: float | :class:`numpy.ndarray`
|
|
1427
|
+
"""
|
|
1428
|
+
return 1 / (1 - fx)
|
|
1429
|
+
|
|
1430
|
+
@staticmethod
|
|
1431
|
+
def gumbel_fx(x, a, b):
|
|
1432
|
+
"""
|
|
1433
|
+
Gumbel probability distribution F(X)
|
|
1434
|
+
|
|
1435
|
+
:param x: function inputs
|
|
1436
|
+
:type x: float | :class:`numpy.ndarray`
|
|
1437
|
+
:param a: distribution parameter a
|
|
1438
|
+
:type a: float
|
|
1439
|
+
:param b: distribution parameter b
|
|
1440
|
+
:type b: float
|
|
1441
|
+
:return: function output
|
|
1442
|
+
:rtype: float | :class:`numpy.ndarray`
|
|
1443
|
+
"""
|
|
1444
|
+
aux_1 = (x - a) / b
|
|
1445
|
+
aux_2 = np.exp(-aux_1)
|
|
1446
|
+
g_fx = np.exp(-aux_2)
|
|
1447
|
+
return g_fx
|
|
1448
|
+
|
|
1449
|
+
@staticmethod
|
|
1450
|
+
def gumbel_tx(x, a, b):
|
|
1451
|
+
"""
|
|
1452
|
+
Gumbel return period distribution T(X)
|
|
1453
|
+
|
|
1454
|
+
:param x: function inputs
|
|
1455
|
+
:type x: float | :class:`numpy.ndarray`
|
|
1456
|
+
:param a: distribution parameter a
|
|
1457
|
+
:type a: float
|
|
1458
|
+
:param b: distribution parameter b
|
|
1459
|
+
:type b: float
|
|
1460
|
+
:return: function output
|
|
1461
|
+
:rtype: float | :class:`numpy.ndarray`
|
|
1462
|
+
"""
|
|
1463
|
+
g_fx = Univar.gumbel_fx(x, a, b)
|
|
1464
|
+
return Univar.get_tx(fx=g_fx)
|
|
1465
|
+
|
|
1466
|
+
@staticmethod
|
|
1467
|
+
def gumbel_freqfactor(tx=2):
|
|
1468
|
+
"""
|
|
1469
|
+
Gumbel Frequency Factor K(T)
|
|
1470
|
+
|
|
1471
|
+
:param tx: return period T
|
|
1472
|
+
:type tx: float | :class:`numpy.ndarray`
|
|
1473
|
+
:return: function output
|
|
1474
|
+
:rtype: float | :class:`numpy.ndarray`
|
|
1475
|
+
"""
|
|
1476
|
+
aux1 = tx / (tx - 1)
|
|
1477
|
+
aux2 = np.log(aux1)
|
|
1478
|
+
aux3 = np.log(aux2)
|
|
1479
|
+
aux4 = 0.5772 + aux3
|
|
1480
|
+
aux5 = -np.sqrt(6) * aux4 / np.pi
|
|
1481
|
+
return aux5
|
|
1482
|
+
|
|
1483
|
+
@staticmethod
|
|
1484
|
+
def gumbel_se(std_sample, n_sample, tx):
|
|
1485
|
+
"""
|
|
1486
|
+
Gumbel Standard Error for the MM fitted Gumbel function
|
|
1487
|
+
|
|
1488
|
+
:param std_sample: sample standard deviation
|
|
1489
|
+
:type std_sample: float
|
|
1490
|
+
:param n_sample: sample size
|
|
1491
|
+
:type n_sample: int
|
|
1492
|
+
:param tx: return period T
|
|
1493
|
+
:type tx: float | :class:`numpy.ndarray`
|
|
1494
|
+
:return: function output
|
|
1495
|
+
:rtype: float | :class:`numpy.ndarray`
|
|
1496
|
+
"""
|
|
1497
|
+
aux1 = std_sample / np.sqrt(n_sample)
|
|
1498
|
+
k = Univar.gumbel_freqfactor(tx=tx)
|
|
1499
|
+
aux2 = 1 + (1.14 * k) + (1.1 * np.square(k))
|
|
1500
|
+
aux3 = np.sqrt(aux2)
|
|
1501
|
+
return aux1 * aux3
|
|
1502
|
+
|
|
1503
|
+
@staticmethod
|
|
1504
|
+
def empirical_px(ranks):
|
|
1505
|
+
"""
|
|
1506
|
+
Get the empirical exceedance probability P(X)
|
|
1507
|
+
|
|
1508
|
+
:param ranks: vector of ranks
|
|
1509
|
+
:type ranks: class:`numpy.array`
|
|
1510
|
+
:return: empirical exceedance probability P(X)
|
|
1511
|
+
:rtype: class:`numpy.array`
|
|
1512
|
+
"""
|
|
1513
|
+
return ranks / len(ranks)
|
|
1514
|
+
|
|
1515
|
+
@staticmethod
|
|
1516
|
+
def weibull_px(ranks):
|
|
1517
|
+
"""
|
|
1518
|
+
Get the Weibull exceedance probability P(X)
|
|
1519
|
+
|
|
1520
|
+
:param ranks: vector of ranks
|
|
1521
|
+
:type ranks: class:`numpy.array`
|
|
1522
|
+
:return: Weibull exceedance probability P(X)
|
|
1523
|
+
:rtype: class:`numpy.array`
|
|
1524
|
+
"""
|
|
1525
|
+
return ranks / (len(ranks) + 1)
|
|
1526
|
+
|
|
1527
|
+
@staticmethod
|
|
1528
|
+
def gringorten_px(ranks):
|
|
1529
|
+
"""
|
|
1530
|
+
Get the Gringorten exceedance probability P(X)
|
|
1531
|
+
|
|
1532
|
+
:param ranks: vector of ranks
|
|
1533
|
+
:type ranks: class:`numpy.array`
|
|
1534
|
+
:return: Gringorten exceedance probability P(X)
|
|
1535
|
+
:rtype: class:`numpy.array`
|
|
1536
|
+
"""
|
|
1537
|
+
return (ranks - 0.44) / (len(ranks) + 0.12)
|
|
1538
|
+
|
|
1539
|
+
@staticmethod
|
|
1540
|
+
def sample_gamma(size, shape=3, scale=1, shape_mode=None, n_min=None, n_max=None):
|
|
1541
|
+
"""
|
|
1542
|
+
Sample Gamma distribution
|
|
1543
|
+
|
|
1544
|
+
:param size: Sample size
|
|
1545
|
+
:type size: int
|
|
1546
|
+
:param shape: Shape parameter
|
|
1547
|
+
:type shape: float
|
|
1548
|
+
:param scale: Scale parameter
|
|
1549
|
+
:type scale: float
|
|
1550
|
+
:param shape_mode: Shape mode (overwrites shape value). See dict for options
|
|
1551
|
+
:type shape_mode: str or None
|
|
1552
|
+
:param n_min: Normalization lower value
|
|
1553
|
+
:type n_min: float or None
|
|
1554
|
+
:param n_max: Normalization upper value
|
|
1555
|
+
:type n_max: float or None
|
|
1556
|
+
:return: Sampled array
|
|
1557
|
+
:rtype: :class:`numpy.ndarray`
|
|
1558
|
+
"""
|
|
1559
|
+
from plans.geo import normalize
|
|
1560
|
+
|
|
1561
|
+
# set gamma shapes
|
|
1562
|
+
dc_gamma_shapes = {
|
|
1563
|
+
"decay": 1,
|
|
1564
|
+
"asymmetrical_extra": 2,
|
|
1565
|
+
"asymmetrical": 3,
|
|
1566
|
+
"symmetrical": 20,
|
|
1567
|
+
}
|
|
1568
|
+
# handle shapemode
|
|
1569
|
+
if shape_mode is not None:
|
|
1570
|
+
shape = dc_gamma_shapes[shape_mode]
|
|
1571
|
+
|
|
1572
|
+
# sample
|
|
1573
|
+
sample = np.random.gamma(shape=shape, scale=scale, size=size)
|
|
1574
|
+
|
|
1575
|
+
# handle normalization
|
|
1576
|
+
if n_min is None:
|
|
1577
|
+
n_min = np.min(sample)
|
|
1578
|
+
if n_max is None:
|
|
1579
|
+
n_max = np.max(sample)
|
|
1580
|
+
|
|
1581
|
+
# normalize
|
|
1582
|
+
sample_norm = normalize(array=sample, min_value=n_min, max_value=n_max)
|
|
1583
|
+
|
|
1584
|
+
return sample_norm
|
|
1585
|
+
|
|
1586
|
+
@staticmethod
|
|
1587
|
+
def get_nbins_fd(data):
|
|
1588
|
+
"""
|
|
1589
|
+
This function computes the number of bins for histograms using the Freedman-Diaconis rule, which takes into
|
|
1590
|
+
account the interquartile range (IQR) of the sample, in addition to its range.
|
|
1591
|
+
|
|
1592
|
+
:param data: vector of data without nan values
|
|
1593
|
+
:type data: :class:`numpy.ndarray`
|
|
1594
|
+
:return: number of bins for histogram using the Freedman-Diaconis rule
|
|
1595
|
+
:rtype: int
|
|
1596
|
+
"""
|
|
1597
|
+
iqr = np.subtract(*np.percentile(data, [75, 25]))
|
|
1598
|
+
binsize = 2 * iqr * len(data) ** (-1 / 3)
|
|
1599
|
+
# hack for non infinite values
|
|
1600
|
+
if binsize == 0:
|
|
1601
|
+
binsize = 100
|
|
1602
|
+
return int(np.ceil((max(data) - min(data)) / binsize))
|
|
1603
|
+
|
|
1604
|
+
@staticmethod
|
|
1605
|
+
def get_nbins_sturges(data):
|
|
1606
|
+
"""
|
|
1607
|
+
This function computes the number of bins using the Sturges rule,
|
|
1608
|
+
which assumes that the data follows a normal distribution and computes
|
|
1609
|
+
the number of bins based on its data runsize.
|
|
1610
|
+
|
|
1611
|
+
:param data: vector of data without nan values
|
|
1612
|
+
:type data: :class:`numpy.ndarray`
|
|
1613
|
+
:return: number of bins using the Sturges rule
|
|
1614
|
+
:rtype: int
|
|
1615
|
+
"""
|
|
1616
|
+
return int(np.ceil(np.log2(len(data)) + 1))
|
|
1617
|
+
|
|
1618
|
+
@staticmethod
|
|
1619
|
+
def get_nbins_scott(data):
|
|
1620
|
+
"""
|
|
1621
|
+
This function computes the number of bins using the Scott rule,
|
|
1622
|
+
which is similar to the Freedman-Diaconis rule, but uses the standard deviation
|
|
1623
|
+
of the data to compute the bin runsize.
|
|
1624
|
+
|
|
1625
|
+
:param data: vector of data without nan values
|
|
1626
|
+
:type data: :class:`numpy.ndarray`
|
|
1627
|
+
:return: number of bins using the Scott rule
|
|
1628
|
+
:rtype: int
|
|
1629
|
+
"""
|
|
1630
|
+
binsize = 3.5 * np.nanstd(data) * len(data) ** (-1 / 3)
|
|
1631
|
+
return int(np.ceil((max(data) - min(data)) / binsize))
|
|
1632
|
+
|
|
1633
|
+
@staticmethod
|
|
1634
|
+
def get_nbins_density(n_units, data_range):
|
|
1635
|
+
# todo docstring
|
|
1636
|
+
n_range = data_range[1] - data_range[0]
|
|
1637
|
+
bins = int(n_range / n_units)
|
|
1638
|
+
if bins < 1:
|
|
1639
|
+
return 1
|
|
1640
|
+
else:
|
|
1641
|
+
return bins
|
|
1642
|
+
|
|
1643
|
+
@staticmethod
|
|
1644
|
+
def get_nbins_by_rule(data, rule=None):
|
|
1645
|
+
"""
|
|
1646
|
+
Util function for rule-based nbins computation
|
|
1647
|
+
|
|
1648
|
+
:param data: vector of data without nan values
|
|
1649
|
+
:type data: :class:`numpy.ndarray`
|
|
1650
|
+
:param rule: rule code (sturges, fd, scott)
|
|
1651
|
+
:type rule: str
|
|
1652
|
+
:return: number of bins for histogram
|
|
1653
|
+
:rtype: int
|
|
1654
|
+
"""
|
|
1655
|
+
bins = 10
|
|
1656
|
+
if rule is None:
|
|
1657
|
+
pass
|
|
1658
|
+
else:
|
|
1659
|
+
if rule.lower() == "sturges":
|
|
1660
|
+
bins = Univar.get_nbins_sturges()
|
|
1661
|
+
elif rule.lower() == "fd":
|
|
1662
|
+
bins = Univar.get_nbins_fd()
|
|
1663
|
+
elif rule.lower() == "scott":
|
|
1664
|
+
bins = Univar.get_nbins_scott()
|
|
1665
|
+
else:
|
|
1666
|
+
pass
|
|
1667
|
+
return bins
|
|
1668
|
+
|
|
1669
|
+
@staticmethod
|
|
1670
|
+
def get_histogram(data, bins=100, rule=None):
|
|
1671
|
+
"""
|
|
1672
|
+
Compute the histogram of the sample
|
|
1673
|
+
|
|
1674
|
+
:param data: vector of data without nan values
|
|
1675
|
+
:type data: :class:`numpy.ndarray`
|
|
1676
|
+
:param bins: number of bins
|
|
1677
|
+
:type bins: int
|
|
1678
|
+
:param rule: rule to define the number of bins. If 'none', uses bins parameters.
|
|
1679
|
+
:type rule: str
|
|
1680
|
+
:return: dataframe of histogram
|
|
1681
|
+
:rtype: :class:`pandas.DataFrame`
|
|
1682
|
+
"""
|
|
1683
|
+
|
|
1684
|
+
if bins is None:
|
|
1685
|
+
if rule is None:
|
|
1686
|
+
pass
|
|
1687
|
+
else:
|
|
1688
|
+
bins = Univar.get_nbins_by_rule(rule=rule)
|
|
1689
|
+
# compute histogram
|
|
1690
|
+
vct_hist, vct_bins = np.histogram(data, bins=bins)
|
|
1691
|
+
# get dataframe
|
|
1692
|
+
df_hist = pd.DataFrame({"Upper value": vct_bins[1:], "Count": vct_hist})
|
|
1693
|
+
return df_hist
|
|
1694
|
+
|
|
1695
|
+
@staticmethod
|
|
1696
|
+
def qqplot(data):
|
|
1697
|
+
"""
|
|
1698
|
+
Calculate the QQ-plot of data against normal distribution
|
|
1699
|
+
|
|
1700
|
+
:param data: vector of data without nan values
|
|
1701
|
+
:type data: :class:`numpy.ndarray`
|
|
1702
|
+
:return: dataframe of QQ plot
|
|
1703
|
+
:rtype: :class:`pandas.DataFrame`
|
|
1704
|
+
"""
|
|
1705
|
+
from scipy.stats import norm
|
|
1706
|
+
|
|
1707
|
+
# process quantiles
|
|
1708
|
+
_df = pd.DataFrame(
|
|
1709
|
+
{
|
|
1710
|
+
"Data": np.sort(data),
|
|
1711
|
+
"E-Quantiles": (np.arange(1, len(data) + 1) - 0.5) / len(data),
|
|
1712
|
+
}
|
|
1713
|
+
)
|
|
1714
|
+
# get theoretical
|
|
1715
|
+
_df["T-Quantiles"] = norm.ppf(_df["E-Quantiles"])
|
|
1716
|
+
return _df
|
|
1717
|
+
|
|
1718
|
+
@staticmethod
|
|
1719
|
+
def trace_variance(data):
|
|
1720
|
+
"""
|
|
1721
|
+
Trace the mean variance from sample
|
|
1722
|
+
|
|
1723
|
+
:param data: vector of data without nan values
|
|
1724
|
+
:type data: :class:`numpy.ndarray`
|
|
1725
|
+
:return: dataframe of accumulated variance
|
|
1726
|
+
:rtype: :class:`pandas.DataFrame`
|
|
1727
|
+
"""
|
|
1728
|
+
vct_variance_mean = np.zeros(len(data))
|
|
1729
|
+
for i in range(2, len(data)):
|
|
1730
|
+
vct_variance_mean[i] = np.var(data[:i])
|
|
1731
|
+
|
|
1732
|
+
# process quantiles
|
|
1733
|
+
_df = pd.DataFrame(
|
|
1734
|
+
{
|
|
1735
|
+
"Data": data,
|
|
1736
|
+
"Variance": vct_variance_mean,
|
|
1737
|
+
}
|
|
1738
|
+
)
|
|
1739
|
+
|
|
1740
|
+
return _df
|
|
1741
|
+
|
|
1742
|
+
@staticmethod
|
|
1743
|
+
def get_bins(data, n_bins=5, scheme="equal"):
|
|
1744
|
+
# todo [docstring]
|
|
1745
|
+
dc = {
|
|
1746
|
+
"linear": Univar.bins_equal,
|
|
1747
|
+
"equal": Univar.bins_equal,
|
|
1748
|
+
"quantiles": Univar.bins_quantiles,
|
|
1749
|
+
"quantile": Univar.bins_quantiles,
|
|
1750
|
+
}
|
|
1751
|
+
v = dc[scheme](data=data, n_bins=n_bins)
|
|
1752
|
+
return v
|
|
1753
|
+
|
|
1754
|
+
@staticmethod
|
|
1755
|
+
def bins_equal(data, n_bins):
|
|
1756
|
+
"""
|
|
1757
|
+
Calculates equally spaced (linear) bins for a 1D NumPy array.
|
|
1758
|
+
|
|
1759
|
+
:param data: The 1D NumPy array to analyze.
|
|
1760
|
+
:type data: numpy.ndarray
|
|
1761
|
+
:param n_bins: The desired number of bins/intervals.
|
|
1762
|
+
:type n_bins: int
|
|
1763
|
+
:returns: A NumPy array containing the linear bin boundary values.
|
|
1764
|
+
:rtype: numpy.ndarray
|
|
1765
|
+
"""
|
|
1766
|
+
min_val = np.min(data)
|
|
1767
|
+
max_val = np.max(data)
|
|
1768
|
+
|
|
1769
|
+
# np.linspace creates 'n_classes + 1' points to define 'n_classes' intervals
|
|
1770
|
+
linear_bins = np.linspace(min_val, max_val, n_bins + 1)
|
|
1771
|
+
return linear_bins
|
|
1772
|
+
|
|
1773
|
+
@staticmethod
|
|
1774
|
+
def bins_quantiles(data, n_bins=5):
|
|
1775
|
+
"""
|
|
1776
|
+
Calculates the numerical quantile boundaries (bins) for a 1D NumPy array.
|
|
1777
|
+
|
|
1778
|
+
:param data: The 1D NumPy array to analyze.
|
|
1779
|
+
:type data: numpy.ndarray
|
|
1780
|
+
:param n_bins: The number of quantile classes.
|
|
1781
|
+
:type n_bins: int
|
|
1782
|
+
:returns: A NumPy array containing the quantile boundary values.
|
|
1783
|
+
:rtype: numpy.ndarray
|
|
1784
|
+
"""
|
|
1785
|
+
# pd.qcut with retbins=True returns (labels, bins)
|
|
1786
|
+
_, bins = pd.qcut(
|
|
1787
|
+
data,
|
|
1788
|
+
q=n_bins,
|
|
1789
|
+
retbins=True,
|
|
1790
|
+
duplicates="drop", # Essential for handling non-unique quantiles
|
|
1791
|
+
)
|
|
1792
|
+
return bins
|
|
1793
|
+
|
|
1794
|
+
@staticmethod
|
|
1795
|
+
def classify(data, bins=None, n_classes=5, scheme="equal"):
|
|
1796
|
+
"""
|
|
1797
|
+
Classifies a 1D NumPy array into 0-indexed bins based on provided bin boundaries.
|
|
1798
|
+
|
|
1799
|
+
Values `x` are assigned to class `i` if `bins[i] < x <= bins[i+1]`.
|
|
1800
|
+
Values less than or equal to `bins[0]` are assigned to class 0.
|
|
1801
|
+
Values greater than `bins[-1]` are assigned to the last class (`len(bins) - 2`).
|
|
1802
|
+
|
|
1803
|
+
:param data: The 1D NumPy array to classify.
|
|
1804
|
+
:type data: numpy.ndarray
|
|
1805
|
+
:param bins: A 1D NumPy array specifying the sorted bin edges.
|
|
1806
|
+
E.g., `[min_val, boundary1, boundary2, ..., max_val]`.
|
|
1807
|
+
Must contain at least two elements.
|
|
1808
|
+
:type bins: numpy.ndarray
|
|
1809
|
+
:returns: A 1D NumPy array of integers representing the 0-indexed class for
|
|
1810
|
+
each element in `data_array`.
|
|
1811
|
+
:rtype: numpy.ndarray
|
|
1812
|
+
"""
|
|
1813
|
+
# handle bins
|
|
1814
|
+
if bins is None:
|
|
1815
|
+
bins = Univar.get_bins(data=data, n_bins=n_classes, scheme=scheme)
|
|
1816
|
+
|
|
1817
|
+
# Number of intervals (classes) will be one less than the number of bins
|
|
1818
|
+
num_classes = len(bins) - 1
|
|
1819
|
+
|
|
1820
|
+
# np.searchsorted(a, v, side='right') returns index 'i' such that all v in a[0:i] <= v
|
|
1821
|
+
# This means `bins[i-1] < x <= bins[i]`.
|
|
1822
|
+
# For example, if bins = [0, 10, 20]:
|
|
1823
|
+
# x=5 => searchsorted returns 1 (meaning it's after bins[0])
|
|
1824
|
+
# x=10 => searchsorted returns 1 (meaning it's after bins[0])
|
|
1825
|
+
# x=15 => searchsorted returns 2 (meaning it's after bins[1])
|
|
1826
|
+
# x=20 => searchsorted returns 2 (meaning it's after bins[1])
|
|
1827
|
+
# Subtracting 1 yields the 0-indexed class for [bins[i-1], bins[i]] like intervals.
|
|
1828
|
+
|
|
1829
|
+
# `classes` will be 0 for x <= bins[0], 1 for x in (bins[0], bins[1]], etc.
|
|
1830
|
+
# For x > bins[-1], it will be `num_classes` (len(bins) - 1).
|
|
1831
|
+
classes = np.searchsorted(bins, data, side="right") - 1
|
|
1832
|
+
|
|
1833
|
+
# Clamp the results to ensure they fall within the valid 0 to num_classes-1 range.
|
|
1834
|
+
# This handles values outside the `bins` range by assigning them to the
|
|
1835
|
+
# first (0) or last (num_classes-1) available class.
|
|
1836
|
+
classified_array = np.clip(classes, 0, num_classes - 1)
|
|
1837
|
+
|
|
1838
|
+
return classified_array
|
|
1839
|
+
|
|
1840
|
+
@staticmethod
|
|
1841
|
+
def quantiles_classify(data, n_classes=5):
|
|
1842
|
+
"""
|
|
1843
|
+
Classifies a 1D NumPy array into quantile classes.
|
|
1844
|
+
|
|
1845
|
+
:param data: The 1D NumPy array to classify.
|
|
1846
|
+
:type data: numpy.ndarray
|
|
1847
|
+
:param n_classes: The number of quantile classes.
|
|
1848
|
+
:type n_classes: int
|
|
1849
|
+
:returns: A 1D NumPy array containing the quantile class (0 to n_classes-1) for each element.
|
|
1850
|
+
:rtype: numpy.ndarray
|
|
1851
|
+
"""
|
|
1852
|
+
|
|
1853
|
+
bins = Univar.get_bins(data=data, n_bins=n_classes, scheme="quantiles")
|
|
1854
|
+
|
|
1855
|
+
values = Univar.classify(data, bins=bins)
|
|
1856
|
+
|
|
1857
|
+
"""
|
|
1858
|
+
# Use pandas qcut, which is efficient and handles duplicates well
|
|
1859
|
+
# labels=False ensures integer class labels (0 to n_classes-1)
|
|
1860
|
+
# duplicates='drop' handles cases where fewer unique quantiles than n_classes exist
|
|
1861
|
+
quantile_classes = pd.qcut(
|
|
1862
|
+
data,
|
|
1863
|
+
q=n_classes,
|
|
1864
|
+
labels=False,
|
|
1865
|
+
duplicates='drop'
|
|
1866
|
+
)#.to_numpy() # Convert the pandas Series back to a NumPy array
|
|
1867
|
+
"""
|
|
1868
|
+
|
|
1869
|
+
return quantile_classes
|
|
1870
|
+
|
|
1871
|
+
|
|
1872
|
+
class GeoUnivar(Univar):
|
|
1873
|
+
|
|
1874
|
+
def __init__(self, name="MyGeoUnivar", alias="GV0"):
|
|
1875
|
+
# call super
|
|
1876
|
+
super().__init__(name=name, alias=alias)
|
|
1877
|
+
self.layer_name = None
|
|
1878
|
+
|
|
1879
|
+
def _set_fields(self):
|
|
1880
|
+
# ------------ call super ----------- #
|
|
1881
|
+
super()._set_fields()
|
|
1882
|
+
# Attribute fields
|
|
1883
|
+
self.field_geometry = "geometry"
|
|
1884
|
+
# ... continues in downstream objects ... #
|
|
1885
|
+
|
|
1886
|
+
def _set_view_specs(self):
|
|
1887
|
+
super()._set_view_specs()
|
|
1888
|
+
self.view_specs.update(
|
|
1889
|
+
{
|
|
1890
|
+
"subtitle_map": "Map",
|
|
1891
|
+
"ax_map": 0,
|
|
1892
|
+
"ax_data": 1,
|
|
1893
|
+
"ax_histh": 2,
|
|
1894
|
+
"ax_cdf": 3,
|
|
1895
|
+
"x_stats": 0.55,
|
|
1896
|
+
"zorder_map": 0,
|
|
1897
|
+
"empty_map": False,
|
|
1898
|
+
"grid_map": False,
|
|
1899
|
+
"marker_map": "o",
|
|
1900
|
+
"markersize_map": 10,
|
|
1901
|
+
"edgecolor_map": "none",
|
|
1902
|
+
"colorize_map": True,
|
|
1903
|
+
"cbar_scatter": True,
|
|
1904
|
+
"cbar_histh": True,
|
|
1905
|
+
}
|
|
1906
|
+
)
|
|
1907
|
+
return None
|
|
1908
|
+
|
|
1909
|
+
def load_data(self, file_data, layer_name):
|
|
1910
|
+
"""
|
|
1911
|
+
Load data from file. Expected to overwrite superior methods.
|
|
1912
|
+
|
|
1913
|
+
:param file_data: file path to data.
|
|
1914
|
+
:type file_data: str
|
|
1915
|
+
:param layer_name:
|
|
1916
|
+
:type layer_name: str
|
|
1917
|
+
:return: None
|
|
1918
|
+
:rtype: None
|
|
1919
|
+
"""
|
|
1920
|
+
# include geopandas
|
|
1921
|
+
import geopandas as gpd
|
|
1922
|
+
|
|
1923
|
+
# -------------- overwrite relative path inputs -------------- #
|
|
1924
|
+
self.file_data = os.path.abspath(file_data)
|
|
1925
|
+
self.layer_name = layer_name
|
|
1926
|
+
|
|
1927
|
+
# -------------- implement loading logic -------------- #
|
|
1928
|
+
default_columns = {
|
|
1929
|
+
#'DateTime': 'datetime64[1s]',
|
|
1930
|
+
self.varfield: float,
|
|
1931
|
+
}
|
|
1932
|
+
ls_cols = [self.varfield, self.field_geometry]
|
|
1933
|
+
# -------------- call loading function -------------- #
|
|
1934
|
+
|
|
1935
|
+
# handle data formats
|
|
1936
|
+
self.data = gpd.read_file(self.file_data, layer=layer_name, columns=ls_cols)
|
|
1937
|
+
|
|
1938
|
+
# -------------- post-loading logic -------------- #
|
|
1939
|
+
self.data.dropna(inplace=True)
|
|
1940
|
+
|
|
1941
|
+
# -------------- update other mutables -------------- #
|
|
1942
|
+
self.update()
|
|
1943
|
+
|
|
1944
|
+
# ... continues in downstream objects ... #
|
|
1945
|
+
|
|
1946
|
+
return None
|
|
1947
|
+
|
|
1948
|
+
# todo [dev] -- export method
|
|
1949
|
+
|
|
1950
|
+
def _build_axes(self, fig, gs, specs):
|
|
1951
|
+
# todo [docstring]
|
|
1952
|
+
# ------------ setup axes ------------
|
|
1953
|
+
if specs["mode"] == "full":
|
|
1954
|
+
fig.add_subplot(gs[2:15, 1:14])
|
|
1955
|
+
fig.add_subplot(gs[2:10, 16:21])
|
|
1956
|
+
fig.add_subplot(gs[2:10, 23:28])
|
|
1957
|
+
fig.add_subplot(gs[2:10, 29:])
|
|
1958
|
+
elif specs["mode"] == "mini":
|
|
1959
|
+
fig.add_subplot(gs[2:10, 2:10])
|
|
1960
|
+
fig.add_subplot(gs[2:10, 16:21])
|
|
1961
|
+
fig.add_subplot(gs[2:10, 23:28])
|
|
1962
|
+
fig.add_subplot(gs[2:10, 29:])
|
|
1963
|
+
return fig
|
|
1964
|
+
|
|
1965
|
+
def _get_fig_specs(self):
|
|
1966
|
+
# todo [docstring]
|
|
1967
|
+
# handle specs
|
|
1968
|
+
specs = self.view_specs.copy()
|
|
1969
|
+
|
|
1970
|
+
# handle mode
|
|
1971
|
+
mode = specs["mode"]
|
|
1972
|
+
if mode == "full":
|
|
1973
|
+
specs_aux = {
|
|
1974
|
+
"ncols": 34,
|
|
1975
|
+
"nrows": 16,
|
|
1976
|
+
"width": viewer.FIG_SIZES["L"]["w"],
|
|
1977
|
+
"height": viewer.FIG_SIZES["L"]["h"],
|
|
1978
|
+
}
|
|
1979
|
+
else:
|
|
1980
|
+
specs_aux = {
|
|
1981
|
+
"ncols": 34,
|
|
1982
|
+
"nrows": 12,
|
|
1983
|
+
"width": viewer.FIG_SIZES["L2"]["w"],
|
|
1984
|
+
"height": viewer.FIG_SIZES["L2"]["h"],
|
|
1985
|
+
}
|
|
1986
|
+
# update sizes
|
|
1987
|
+
specs.update(specs_aux)
|
|
1988
|
+
|
|
1989
|
+
# update gridspecs
|
|
1990
|
+
specs.update(viewer.GRID_SPECS)
|
|
1991
|
+
|
|
1992
|
+
return specs
|
|
1993
|
+
|
|
1994
|
+
def _plot(self, fig, gs, specs):
|
|
1995
|
+
# todo [docstring]
|
|
1996
|
+
fig = super()._plot(fig=fig, gs=gs, specs=specs)
|
|
1997
|
+
|
|
1998
|
+
all_axes = fig.get_axes()
|
|
1999
|
+
|
|
2000
|
+
# ------------ map plot ------------
|
|
2001
|
+
ax_map = all_axes[specs["ax_map"]]
|
|
2002
|
+
|
|
2003
|
+
GeoUnivar.plot_map(data=self.data, ax=ax_map, column=self.varfield, specs=specs)
|
|
2004
|
+
|
|
2005
|
+
# ------------ scatter cbar plot ------------
|
|
2006
|
+
# handle plotting conflict
|
|
2007
|
+
if specs["colorize_scatter"]:
|
|
2008
|
+
specs["cbar_scatter"] = False
|
|
2009
|
+
if specs["cbar_scatter"]:
|
|
2010
|
+
Univar.plot_cbar(
|
|
2011
|
+
data=self.data[specs["yvar_field"]].values,
|
|
2012
|
+
ax=all_axes[specs["ax_data"]],
|
|
2013
|
+
scheme=specs["scheme_cmap"],
|
|
2014
|
+
cmap=specs["cmap"],
|
|
2015
|
+
)
|
|
2016
|
+
|
|
2017
|
+
# ------------ hist cbar plot ------------
|
|
2018
|
+
if specs["cbar_histh"]:
|
|
2019
|
+
Univar.plot_cbar(
|
|
2020
|
+
data=self.data[specs["yvar_field"]].values,
|
|
2021
|
+
ax=all_axes[specs["ax_histh"]],
|
|
2022
|
+
scheme=specs["scheme_cmap"],
|
|
2023
|
+
cmap=specs["cmap"],
|
|
2024
|
+
)
|
|
2025
|
+
|
|
2026
|
+
return fig
|
|
2027
|
+
|
|
2028
|
+
@staticmethod
|
|
2029
|
+
def plot_map(data, ax, column, specs, legend=False):
|
|
2030
|
+
if not specs["empty_map"]:
|
|
2031
|
+
|
|
2032
|
+
# handle scheme
|
|
2033
|
+
if specs["colorize_map"]:
|
|
2034
|
+
color_data = Univar.classify(
|
|
2035
|
+
data=data[column].values, n_classes=5, scheme=specs["scheme_cmap"]
|
|
2036
|
+
)
|
|
2037
|
+
color_cmap = viewer.get_discrete_cmap(
|
|
2038
|
+
n_classes=5, base_cmap_name=specs["cmap"]
|
|
2039
|
+
)
|
|
2040
|
+
# set new column
|
|
2041
|
+
data[f"{column}_plot"] = color_data
|
|
2042
|
+
|
|
2043
|
+
data.plot(
|
|
2044
|
+
ax=ax,
|
|
2045
|
+
column=f"{column}_plot",
|
|
2046
|
+
zorder=specs["zorder_map"],
|
|
2047
|
+
cmap=color_cmap,
|
|
2048
|
+
legend=legend,
|
|
2049
|
+
marker=specs["marker_map"], # Use a circle marker for dots
|
|
2050
|
+
markersize=specs[
|
|
2051
|
+
"markersize_map"
|
|
2052
|
+
], # Set the size of the points (adjust as needed)
|
|
2053
|
+
edgecolor=specs["edgecolor_map"],
|
|
2054
|
+
)
|
|
2055
|
+
else:
|
|
2056
|
+
data.plot(
|
|
2057
|
+
ax=ax,
|
|
2058
|
+
column=column,
|
|
2059
|
+
zorder=specs["zorder_map"],
|
|
2060
|
+
cmap=specs["cmap"],
|
|
2061
|
+
legend=legend,
|
|
2062
|
+
marker=specs["marker_map"], # Use a circle marker for dots
|
|
2063
|
+
markersize=specs[
|
|
2064
|
+
"markersize_map"
|
|
2065
|
+
], # Set the size of the points (adjust as needed)
|
|
2066
|
+
edgecolor=specs["edgecolor_map"],
|
|
2067
|
+
)
|
|
2068
|
+
# Set the Y-axis tick labels orientation
|
|
2069
|
+
ax.tick_params(
|
|
2070
|
+
axis="y", # Apply to the y-axis
|
|
2071
|
+
labelrotation=90, # Rotate labels by 90 degrees (vertical)
|
|
2072
|
+
)
|
|
2073
|
+
|
|
2074
|
+
for label in ax.get_yticklabels():
|
|
2075
|
+
# Option A: Often looks good for vertical Y-labels on the left
|
|
2076
|
+
label.set_horizontalalignment("right")
|
|
2077
|
+
label.set_verticalalignment("center")
|
|
2078
|
+
if specs["subtitle_map"] is not None:
|
|
2079
|
+
ax.set_title(specs["subtitle_map"], loc="left")
|
|
2080
|
+
ax.grid(visible=specs["grid_map"])
|
|
2081
|
+
|
|
2082
|
+
return None
|
|
2083
|
+
|
|
2084
|
+
|
|
2085
|
+
# todo [upgrade] make it a child of Dataset()
|
|
2086
|
+
class Bivar:
|
|
2087
|
+
# todo [docstring]
|
|
2088
|
+
"""
|
|
2089
|
+
The Bivariate analyst object
|
|
2090
|
+
|
|
2091
|
+
# [major docstring]
|
|
2092
|
+
|
|
2093
|
+
"""
|
|
2094
|
+
|
|
2095
|
+
def __init__(self, df_data, x_name="x", y_name="y", name="myvars"):
|
|
2096
|
+
|
|
2097
|
+
# set inputs attributes
|
|
2098
|
+
self.xname = x_name
|
|
2099
|
+
self.yname = y_name
|
|
2100
|
+
self.name = name
|
|
2101
|
+
|
|
2102
|
+
# set sorted data and reset index
|
|
2103
|
+
df_data = df_data[[x_name, y_name]]
|
|
2104
|
+
df_data = df_data.dropna()
|
|
2105
|
+
self.data = df_data.sort_values(by=self.xname).reset_index(drop=True)
|
|
2106
|
+
|
|
2107
|
+
# models setup
|
|
2108
|
+
self.models = {
|
|
2109
|
+
"Linear": {
|
|
2110
|
+
"Function": linear,
|
|
2111
|
+
"Formula": "f(x) = c0 + c1 * x",
|
|
2112
|
+
"Setup": pd.DataFrame(
|
|
2113
|
+
{"Parameters": ["c_0", "c_1"], "Mean": [0, 1], "SD": [0.1, 0.1]}
|
|
2114
|
+
),
|
|
2115
|
+
"Data": None,
|
|
2116
|
+
"RMSE": None,
|
|
2117
|
+
},
|
|
2118
|
+
"Power": {
|
|
2119
|
+
"Function": power,
|
|
2120
|
+
"Formula": "f(x) = c2 * ((x + c0)^c1)",
|
|
2121
|
+
"Setup": pd.DataFrame(
|
|
2122
|
+
{
|
|
2123
|
+
"Parameters": ["c_0", "c_1", "c_2"],
|
|
2124
|
+
"Mean": [0, 1, 1],
|
|
2125
|
+
"SD": [0.1, 0.1, 0.1],
|
|
2126
|
+
}
|
|
2127
|
+
),
|
|
2128
|
+
"Data": None,
|
|
2129
|
+
"RMSE": None,
|
|
2130
|
+
},
|
|
2131
|
+
"Power_zero": {
|
|
2132
|
+
"Function": power_zero,
|
|
2133
|
+
"Formula": "f(x) = c1 * (x^c0)",
|
|
2134
|
+
"Setup": pd.DataFrame(
|
|
2135
|
+
{"Parameters": ["c_0", "c_1"], "Mean": [1, 1], "SD": [0.1, 0.1]}
|
|
2136
|
+
),
|
|
2137
|
+
"Data": None,
|
|
2138
|
+
"RMSE": None,
|
|
2139
|
+
},
|
|
2140
|
+
}
|
|
2141
|
+
|
|
2142
|
+
"""
|
|
2143
|
+
"Gumbel": {
|
|
2144
|
+
"Function": gumbel,
|
|
2145
|
+
"Formula": "f(x) = exp(-exp(-((x-a)/b)))",
|
|
2146
|
+
"Setup": pd.DataFrame(
|
|
2147
|
+
{"Parameters": ["a", "b"], "Mean": [1, 1], "SD": [0.1, 0.1]}
|
|
2148
|
+
),
|
|
2149
|
+
"Data": None,
|
|
2150
|
+
"RMSE": None,
|
|
2151
|
+
},
|
|
2152
|
+
"""
|
|
2153
|
+
|
|
2154
|
+
def fit(self, model_type="Linear"):
|
|
2155
|
+
"""
|
|
2156
|
+
Fit model to bivariate object
|
|
2157
|
+
|
|
2158
|
+
:param model_type: model type. options: Linear, Power, Power_zero
|
|
2159
|
+
:type model_type: str
|
|
2160
|
+
:return: None
|
|
2161
|
+
:rtype: None
|
|
2162
|
+
"""
|
|
2163
|
+
from scipy.optimize import curve_fit
|
|
2164
|
+
|
|
2165
|
+
popt, pcov = curve_fit(
|
|
2166
|
+
f=self.models[model_type]["Function"],
|
|
2167
|
+
xdata=self.data[self.xname],
|
|
2168
|
+
ydata=self.data[self.yname],
|
|
2169
|
+
p0=self.models[model_type]["Setup"]["Mean"].values,
|
|
2170
|
+
)
|
|
2171
|
+
pstd = np.sqrt(np.diag(pcov))
|
|
2172
|
+
# update model
|
|
2173
|
+
self.update_model(params_mean=popt, params_sd=pstd, model_type=model_type)
|
|
2174
|
+
return None
|
|
2175
|
+
|
|
2176
|
+
def update_model(self, params_mean, params_sd=None, model_type="Linear"):
|
|
2177
|
+
"""Update model based on parameters
|
|
2178
|
+
|
|
2179
|
+
:param params_mean: list of mean of parameters
|
|
2180
|
+
:type params_mean: list
|
|
2181
|
+
:param params_sd: list of standard deviation of parameters
|
|
2182
|
+
:type params_sd: list
|
|
2183
|
+
:param model_type: model type. options: Linear, Power, Power_zero
|
|
2184
|
+
:type model_type: str
|
|
2185
|
+
:return: None
|
|
2186
|
+
:rtype: None
|
|
2187
|
+
"""
|
|
2188
|
+
# Setup
|
|
2189
|
+
self.models[model_type]["Setup"]["Mean"] = params_mean
|
|
2190
|
+
if params_sd is None:
|
|
2191
|
+
pass
|
|
2192
|
+
else:
|
|
2193
|
+
self.models[model_type]["Setup"]["SD"] = params_sd
|
|
2194
|
+
# update sample
|
|
2195
|
+
self.updata_model_data(model_type=model_type)
|
|
2196
|
+
# update model metrics
|
|
2197
|
+
vct_e = self.models[model_type]["Data"]["e_Mean"].values
|
|
2198
|
+
self.models[model_type]["RMSE"] = np.square(np.mean(np.square(vct_e)))
|
|
2199
|
+
return None
|
|
2200
|
+
|
|
2201
|
+
def updata_model_data(self, model_type="Linear"):
|
|
2202
|
+
"""Update only model data output
|
|
2203
|
+
|
|
2204
|
+
:param model_type: model type. options: Linear, Power, Power_zero
|
|
2205
|
+
:type model_type: str
|
|
2206
|
+
:return: None
|
|
2207
|
+
:rtype: None
|
|
2208
|
+
"""
|
|
2209
|
+
popt = self.models[model_type]["Setup"]["Mean"].values
|
|
2210
|
+
_df = self.data.copy()
|
|
2211
|
+
s_ymodel = "{}_Mean".format(self.yname)
|
|
2212
|
+
# compute model on sample:
|
|
2213
|
+
_df[s_ymodel] = self.models[model_type]["Function"](
|
|
2214
|
+
self.data[self.xname], *popt
|
|
2215
|
+
)
|
|
2216
|
+
# compute error
|
|
2217
|
+
_df["e_Mean".format(model_type)] = _df[s_ymodel] - _df[self.yname]
|
|
2218
|
+
# set attribute
|
|
2219
|
+
self.models[model_type]["Data"] = _df.copy()
|
|
2220
|
+
del _df
|
|
2221
|
+
return None
|
|
2222
|
+
|
|
2223
|
+
def view(
|
|
2224
|
+
self,
|
|
2225
|
+
show=True,
|
|
2226
|
+
folder="C:/sample",
|
|
2227
|
+
filename="view",
|
|
2228
|
+
specs=None,
|
|
2229
|
+
fig_format="jpg",
|
|
2230
|
+
dpi=300,
|
|
2231
|
+
):
|
|
2232
|
+
"""Plot basic view of Bivar base_object
|
|
2233
|
+
|
|
2234
|
+
:param show: Boolean to show instead of saving
|
|
2235
|
+
:type show: bool
|
|
2236
|
+
:param folder: output folder
|
|
2237
|
+
:type folder: str
|
|
2238
|
+
:param filename: image file name
|
|
2239
|
+
:type filename: str
|
|
2240
|
+
:param specs: specification dictionary
|
|
2241
|
+
:type specs: dict
|
|
2242
|
+
:param dpi: image resolution (default = 300)
|
|
2243
|
+
:type dpi: int
|
|
2244
|
+
:param fig_format: image fig_format (ex: png or jpg). Default jpg
|
|
2245
|
+
:type fig_format: str
|
|
2246
|
+
:return: None
|
|
2247
|
+
:rtype: None
|
|
2248
|
+
"""
|
|
2249
|
+
|
|
2250
|
+
# get specs
|
|
2251
|
+
default_specs = {
|
|
2252
|
+
"color": "tab:grey",
|
|
2253
|
+
"color_scatter": "tab:blue",
|
|
2254
|
+
"title": "View of {}".format(self.name),
|
|
2255
|
+
"width": 6,
|
|
2256
|
+
"height": 6,
|
|
2257
|
+
"range_x": [self.data[self.xname].min(), self.data[self.xname].max()],
|
|
2258
|
+
"range": [self.data[self.yname].min(), self.data[self.yname].max()],
|
|
2259
|
+
"xlabel": self.xname,
|
|
2260
|
+
"ylabel": self.yname,
|
|
2261
|
+
}
|
|
2262
|
+
# handle inputs specs
|
|
2263
|
+
if specs is None:
|
|
2264
|
+
pass
|
|
2265
|
+
else: # override default
|
|
2266
|
+
for k in specs:
|
|
2267
|
+
default_specs[k] = specs[k]
|
|
2268
|
+
specs = default_specs
|
|
2269
|
+
|
|
2270
|
+
# start plot
|
|
2271
|
+
fig = plt.figure(figsize=(specs["width"], specs["height"])) # Width, Height
|
|
2272
|
+
|
|
2273
|
+
plt.suptitle(specs["title"])
|
|
2274
|
+
|
|
2275
|
+
# grid
|
|
2276
|
+
gs = mpl.gridspec.GridSpec(
|
|
2277
|
+
3, 3, wspace=0.3, hspace=0.3, left=0.12, bottom=0.1, top=0.90, right=0.95
|
|
2278
|
+
) # nrows, ncols
|
|
2279
|
+
|
|
2280
|
+
# scatter plot
|
|
2281
|
+
ax = fig.add_subplot(gs[1:, :2])
|
|
2282
|
+
plt.scatter(
|
|
2283
|
+
self.data[self.xname],
|
|
2284
|
+
self.data[self.yname],
|
|
2285
|
+
marker=".",
|
|
2286
|
+
color=specs["color_scatter"],
|
|
2287
|
+
)
|
|
2288
|
+
plt.xlabel(specs["xlabel"])
|
|
2289
|
+
plt.ylabel(specs["ylabel"])
|
|
2290
|
+
plt.xlim(specs["range_x"])
|
|
2291
|
+
plt.ylim(specs["range"])
|
|
2292
|
+
|
|
2293
|
+
# x hist
|
|
2294
|
+
ax_histx = fig.add_subplot(gs[0, :2], sharex=ax)
|
|
2295
|
+
plt.ylabel("p({})".format(self.xname))
|
|
2296
|
+
xuni = Univar()
|
|
2297
|
+
xuni.data = self.data[self.xname].values
|
|
2298
|
+
plt.hist(
|
|
2299
|
+
self.data[self.xname],
|
|
2300
|
+
bins=xuni.get_nbins_fd(data=xuni.data),
|
|
2301
|
+
color=specs["color"],
|
|
2302
|
+
alpha=1,
|
|
2303
|
+
weights=np.ones(len(self.data)) / len(self.data),
|
|
2304
|
+
)
|
|
2305
|
+
plt.xlim(specs["range_x"])
|
|
2306
|
+
|
|
2307
|
+
# y hist
|
|
2308
|
+
ay_histy = fig.add_subplot(gs[1:, 2], sharey=ax)
|
|
2309
|
+
plt.xlabel("p({})".format(self.yname))
|
|
2310
|
+
yuni = Univar()
|
|
2311
|
+
yuni.data = self.data[self.yname].values
|
|
2312
|
+
plt.hist(
|
|
2313
|
+
yuni.data,
|
|
2314
|
+
bins=yuni.get_nbins_fd(data=yuni.data),
|
|
2315
|
+
color=specs["color"],
|
|
2316
|
+
alpha=1,
|
|
2317
|
+
orientation="horizontal",
|
|
2318
|
+
weights=np.ones(len(self.data)) / len(self.data),
|
|
2319
|
+
)
|
|
2320
|
+
plt.ylim(specs["range"])
|
|
2321
|
+
|
|
2322
|
+
# show or save
|
|
2323
|
+
if show:
|
|
2324
|
+
plt.show()
|
|
2325
|
+
else:
|
|
2326
|
+
if filename is None:
|
|
2327
|
+
filename = self.name
|
|
2328
|
+
plt.savefig("{}/{}.{}".format(folder, filename, fig_format), dpi=dpi)
|
|
2329
|
+
plt.close(fig)
|
|
2330
|
+
|
|
2331
|
+
def view_model(
|
|
2332
|
+
self,
|
|
2333
|
+
model_type="Power",
|
|
2334
|
+
show=True,
|
|
2335
|
+
folder="C:/sample",
|
|
2336
|
+
filename=None,
|
|
2337
|
+
specs=None,
|
|
2338
|
+
dpi=300,
|
|
2339
|
+
fig_format="jpg",
|
|
2340
|
+
):
|
|
2341
|
+
"""Plot pannel for model analysis
|
|
2342
|
+
|
|
2343
|
+
:param show: Boolean to show instead of saving
|
|
2344
|
+
:type show: bool
|
|
2345
|
+
:param folder: output folder
|
|
2346
|
+
:type folder: str
|
|
2347
|
+
:param filename: image file name
|
|
2348
|
+
:type filename: str
|
|
2349
|
+
:param specs: specification dictionary
|
|
2350
|
+
:type specs: dict
|
|
2351
|
+
:param dpi: image resolution (default = 300)
|
|
2352
|
+
:type dpi: int
|
|
2353
|
+
:param fig_format: image fig_format (ex: png or jpg). Default jpg
|
|
2354
|
+
:type fig_format: str
|
|
2355
|
+
:return: None
|
|
2356
|
+
:rtype: None
|
|
2357
|
+
"""
|
|
2358
|
+
|
|
2359
|
+
# ensure data is update
|
|
2360
|
+
self.updata_model_data(model_type=model_type)
|
|
2361
|
+
|
|
2362
|
+
# get specs
|
|
2363
|
+
default_specs = {
|
|
2364
|
+
"color": "tab:grey",
|
|
2365
|
+
"color_scatter": "tab:blue",
|
|
2366
|
+
"color_model": "black",
|
|
2367
|
+
"color_variance": "darkred",
|
|
2368
|
+
"title": "View of {} {} Model".format(self.name, model_type),
|
|
2369
|
+
"width": 8,
|
|
2370
|
+
"height": 8,
|
|
2371
|
+
"range_x": [self.data[self.xname].min(), self.data[self.xname].max()],
|
|
2372
|
+
"range": [self.data[self.yname].min(), self.data[self.yname].max()],
|
|
2373
|
+
"elim": [
|
|
2374
|
+
-1.5 * self.models[model_type]["Data"]["e_Mean"].max(),
|
|
2375
|
+
1.5 * self.models[model_type]["Data"]["e_Mean"].max(),
|
|
2376
|
+
],
|
|
2377
|
+
"alpha_xy": 0.75,
|
|
2378
|
+
"alpha_e": 0.75,
|
|
2379
|
+
}
|
|
2380
|
+
# handle inputs specs
|
|
2381
|
+
if specs is None:
|
|
2382
|
+
pass
|
|
2383
|
+
else: # override default
|
|
2384
|
+
for k in specs:
|
|
2385
|
+
default_specs[k] = specs[k]
|
|
2386
|
+
specs = default_specs
|
|
2387
|
+
|
|
2388
|
+
# get some ranges
|
|
2389
|
+
e_range = specs["elim"][1] - specs["elim"][0]
|
|
2390
|
+
e_range_10 = e_range / 20
|
|
2391
|
+
x_range = specs["range_x"][1] - specs["range_x"][0]
|
|
2392
|
+
x_range_10 = x_range / 10
|
|
2393
|
+
|
|
2394
|
+
# -------------- start plot
|
|
2395
|
+
fig = plt.figure(figsize=(specs["width"], specs["height"])) # Width, Height
|
|
2396
|
+
plt.suptitle(specs["title"])
|
|
2397
|
+
# grid
|
|
2398
|
+
gs = mpl.gridspec.GridSpec(
|
|
2399
|
+
4, 4, wspace=0.3, hspace=0.45, left=0.12, bottom=0.1, top=0.90, right=0.95
|
|
2400
|
+
) # nrows, ncols
|
|
2401
|
+
|
|
2402
|
+
# ----------------- XY plot -----------------
|
|
2403
|
+
ax = fig.add_subplot(gs[:2, :2])
|
|
2404
|
+
|
|
2405
|
+
plt.scatter(
|
|
2406
|
+
self.data[self.xname],
|
|
2407
|
+
self.data[self.yname],
|
|
2408
|
+
marker=".",
|
|
2409
|
+
color=specs["color_scatter"],
|
|
2410
|
+
alpha=specs["alpha_xy"],
|
|
2411
|
+
zorder=1,
|
|
2412
|
+
)
|
|
2413
|
+
plt.plot(
|
|
2414
|
+
self.models[model_type]["Data"][self.xname],
|
|
2415
|
+
self.models[model_type]["Data"]["{}_Mean".format(self.yname)],
|
|
2416
|
+
color=specs["color_model"],
|
|
2417
|
+
zorder=2,
|
|
2418
|
+
linestyle="--",
|
|
2419
|
+
)
|
|
2420
|
+
plt.xlabel(self.xname)
|
|
2421
|
+
plt.ylabel(self.yname)
|
|
2422
|
+
plt.xlim(specs["range_x"])
|
|
2423
|
+
plt.ylim(specs["range"])
|
|
2424
|
+
|
|
2425
|
+
# ----------------- error -----------------
|
|
2426
|
+
e_uni = Univar(data=self.models[model_type]["Data"]["e_Mean"].values)
|
|
2427
|
+
# get mean
|
|
2428
|
+
e_mean = np.mean(e_uni.data)
|
|
2429
|
+
|
|
2430
|
+
ax = fig.add_subplot(gs[2, :2])
|
|
2431
|
+
plt.scatter(
|
|
2432
|
+
self.models[model_type]["Data"][self.xname],
|
|
2433
|
+
self.models[model_type]["Data"]["e_Mean"].values,
|
|
2434
|
+
marker=".",
|
|
2435
|
+
alpha=specs["alpha_e"],
|
|
2436
|
+
color=specs["color"],
|
|
2437
|
+
)
|
|
2438
|
+
# mean line
|
|
2439
|
+
plt.hlines(
|
|
2440
|
+
y=e_mean,
|
|
2441
|
+
xmin=specs["range_x"][0],
|
|
2442
|
+
xmax=specs["range_x"][1],
|
|
2443
|
+
color="tab:red",
|
|
2444
|
+
alpha=0.4,
|
|
2445
|
+
)
|
|
2446
|
+
plt.text(
|
|
2447
|
+
y=e_mean + e_range_10,
|
|
2448
|
+
x=specs["range_x"][1] - (3 * x_range_10),
|
|
2449
|
+
s=r"$\mu$: " + str(round(e_mean, 2)),
|
|
2450
|
+
color="tab:red",
|
|
2451
|
+
)
|
|
2452
|
+
plt.xlabel(self.xname)
|
|
2453
|
+
plt.ylabel(r"$\epsilon$")
|
|
2454
|
+
plt.xlim(specs["range_x"])
|
|
2455
|
+
plt.ylim(specs["elim"])
|
|
2456
|
+
|
|
2457
|
+
# ----------------- error hist -----------------
|
|
2458
|
+
ax = fig.add_subplot(gs[2, 2])
|
|
2459
|
+
plt.hist(
|
|
2460
|
+
e_uni.data,
|
|
2461
|
+
bins=e_uni.get_nbins_fd(),
|
|
2462
|
+
color=specs["color"],
|
|
2463
|
+
alpha=1,
|
|
2464
|
+
orientation="horizontal",
|
|
2465
|
+
weights=np.ones(len(self.data)) / len(self.data),
|
|
2466
|
+
)
|
|
2467
|
+
# mean line
|
|
2468
|
+
plt.hlines(y=e_mean, xmin=0, xmax=0.15, color="tab:red", alpha=0.4)
|
|
2469
|
+
plt.ylim(specs["elim"])
|
|
2470
|
+
plt.xlabel(r"p($\epsilon$)")
|
|
2471
|
+
|
|
2472
|
+
# ----------------- qq plot error -----------------
|
|
2473
|
+
df_qq = e_uni.qqplot()
|
|
2474
|
+
ax = fig.add_subplot(gs[2, 3])
|
|
2475
|
+
plt.scatter(
|
|
2476
|
+
x=df_qq["T-Quantiles"],
|
|
2477
|
+
y=df_qq["Data"],
|
|
2478
|
+
marker=".",
|
|
2479
|
+
alpha=specs["alpha_e"],
|
|
2480
|
+
color=specs["color"],
|
|
2481
|
+
)
|
|
2482
|
+
# mean line
|
|
2483
|
+
plt.hlines(
|
|
2484
|
+
y=e_mean,
|
|
2485
|
+
xmin=df_qq["T-Quantiles"].min(),
|
|
2486
|
+
xmax=df_qq["T-Quantiles"].max(),
|
|
2487
|
+
color="tab:red",
|
|
2488
|
+
alpha=0.4,
|
|
2489
|
+
)
|
|
2490
|
+
plt.ylim(specs["elim"])
|
|
2491
|
+
plt.xlabel("normal quantiles")
|
|
2492
|
+
|
|
2493
|
+
# ----------------- variance tracing -----------------
|
|
2494
|
+
ax = fig.add_subplot(gs[3, :2])
|
|
2495
|
+
vct_var = e_uni.trace_variance()
|
|
2496
|
+
plt.plot(
|
|
2497
|
+
self.models[model_type]["Data"][self.xname],
|
|
2498
|
+
vct_var,
|
|
2499
|
+
color=specs["color_variance"],
|
|
2500
|
+
)
|
|
2501
|
+
|
|
2502
|
+
plt.xlabel(self.xname)
|
|
2503
|
+
plt.ylabel(r"$\sigma^2$")
|
|
2504
|
+
plt.xlim(specs["range_x"])
|
|
2505
|
+
plt.ylim([0, 1.5 * np.max(vct_var)])
|
|
2506
|
+
|
|
2507
|
+
# show or save
|
|
2508
|
+
if show:
|
|
2509
|
+
plt.show()
|
|
2510
|
+
else:
|
|
2511
|
+
if filename is None:
|
|
2512
|
+
filename = self.name
|
|
2513
|
+
plt.savefig("{}/{}.{}".format(folder, filename, fig_format), dpi=dpi)
|
|
2514
|
+
plt.close(fig)
|
|
2515
|
+
return None
|
|
2516
|
+
|
|
2517
|
+
def correlation(self):
|
|
2518
|
+
"""Compute the R correlation coefficient of the base_object
|
|
2519
|
+
|
|
2520
|
+
:return: R correlation coefficient
|
|
2521
|
+
:rtype: float
|
|
2522
|
+
"""
|
|
2523
|
+
corr_df = self.data.corr().loc[self.xname, self.yname]
|
|
2524
|
+
return corr_df
|
|
2525
|
+
|
|
2526
|
+
def prediction_bands(
|
|
2527
|
+
self, lst_bounds=None, n_sim=100, n_grid=100, n_seed=None, p0=None
|
|
2528
|
+
):
|
|
2529
|
+
"""Run Monte Carlo Simulation to get prediciton bands
|
|
2530
|
+
|
|
2531
|
+
:param lst_bounds: list of prediction bounds [min, max], if None, 3x the date range is used
|
|
2532
|
+
:type lst_bounds: list
|
|
2533
|
+
:param n_sim: number of simulation runs
|
|
2534
|
+
:type n_sim: int
|
|
2535
|
+
:param n_grid: number of prediction intervals
|
|
2536
|
+
:type n_grid: int
|
|
2537
|
+
:param n_seed: number of random seed for reproducibility. Default = None
|
|
2538
|
+
:type n_seed: int
|
|
2539
|
+
:param p0: list of initial values to search. Default: None
|
|
2540
|
+
:type p0: list
|
|
2541
|
+
:return: base_object with result dataframes
|
|
2542
|
+
:rtype: dict
|
|
2543
|
+
"""
|
|
2544
|
+
from scipy.optimize import curve_fit
|
|
2545
|
+
|
|
2546
|
+
# todo this seems to be deprecated
|
|
2547
|
+
# handle None bounds
|
|
2548
|
+
if lst_bounds is None:
|
|
2549
|
+
_min = self.data[self.xname].min()
|
|
2550
|
+
_max = self.data[self.xname].max()
|
|
2551
|
+
# extrapolation zone 2x the interpolation zone
|
|
2552
|
+
lst_bounds = [_min, _max + (2 * _min)]
|
|
2553
|
+
# handle None seed
|
|
2554
|
+
if n_seed is None:
|
|
2555
|
+
from datetime import datetime
|
|
2556
|
+
|
|
2557
|
+
_seed = int(datetime.now().timestamp())
|
|
2558
|
+
n_seed = np.random.seed(_seed)
|
|
2559
|
+
|
|
2560
|
+
# --- set dataframes
|
|
2561
|
+
# scale of simulations
|
|
2562
|
+
n_scale = int(np.log10(n_sim)) + 1
|
|
2563
|
+
|
|
2564
|
+
# simulation labels
|
|
2565
|
+
lst_mcsim = ["MC{}".format(str(i).zfill(n_scale)) for i in range(n_sim + 1)]
|
|
2566
|
+
s_refsim = lst_mcsim[0]
|
|
2567
|
+
|
|
2568
|
+
# parameter labels
|
|
2569
|
+
_lst_params = list(self.linear_model["Parameter"].values)
|
|
2570
|
+
_labels = ["Mean", "Std"]
|
|
2571
|
+
lst_params = list()
|
|
2572
|
+
for i in range(len(_labels)):
|
|
2573
|
+
_label = _labels[i]
|
|
2574
|
+
for j in range(len(_lst_params)):
|
|
2575
|
+
_param = _lst_params[j]
|
|
2576
|
+
lst_params.append("{}_{}".format(_param, _label))
|
|
2577
|
+
|
|
2578
|
+
# models dataframe
|
|
2579
|
+
_cols = lst_params.copy()
|
|
2580
|
+
_cols.insert(0, "Model")
|
|
2581
|
+
_data = np.zeros(shape=(n_sim + 1, len(_cols)))
|
|
2582
|
+
df_models = pd.DataFrame(data=_data, columns=_cols)
|
|
2583
|
+
df_models["Model"] = lst_mcsim
|
|
2584
|
+
|
|
2585
|
+
# prediction dataframe
|
|
2586
|
+
_cols = ["{}_{}".format(self.yname, s) for s in lst_mcsim]
|
|
2587
|
+
_cols.insert(0, self.xname)
|
|
2588
|
+
_data = np.zeros(shape=(n_grid, len(_cols)))
|
|
2589
|
+
df_preds = pd.DataFrame(data=_data, columns=_cols)
|
|
2590
|
+
_min = np.min(lst_bounds)
|
|
2591
|
+
_max = np.max(lst_bounds)
|
|
2592
|
+
df_preds[self.xname] = np.linspace(_min, _max, n_grid)
|
|
2593
|
+
|
|
2594
|
+
# bands dataframe
|
|
2595
|
+
_cols = ["Min", "Max", "Mean", "p01", "p05", "p25", "p50", "p75", "p95", "p99"]
|
|
2596
|
+
_cols.insert(0, self.xname)
|
|
2597
|
+
_data = np.zeros(shape=(n_grid, len(_cols)))
|
|
2598
|
+
df_bands = pd.DataFrame(data=_data, columns=_cols)
|
|
2599
|
+
df_bands[self.xname] = df_preds[self.xname].values
|
|
2600
|
+
|
|
2601
|
+
# set reference simulation parameters
|
|
2602
|
+
_lst = ["Mean", "Std"]
|
|
2603
|
+
for s in _lst:
|
|
2604
|
+
for i in range(len(self.linear_model)):
|
|
2605
|
+
_s_par = self.linear_model["Parameter"].values[i]
|
|
2606
|
+
_s_key = "{}_{}".format(_s_par, s)
|
|
2607
|
+
_value = self.linear_model["Fit_{}".format(s)].values[i]
|
|
2608
|
+
df_models[_s_key].values[0] = _value
|
|
2609
|
+
|
|
2610
|
+
# set reference simulation prediction
|
|
2611
|
+
_x = df_preds[self.xname].values
|
|
2612
|
+
_params = self.linear_model["Fit_Mean"].values
|
|
2613
|
+
df_preds["y_{}".format(s_refsim)] = linear(_x, *_params)
|
|
2614
|
+
|
|
2615
|
+
# get standard deviation from fit sample
|
|
2616
|
+
n_std_e = self.linear_model_data["e"].std()
|
|
2617
|
+
vct_yfit = self.linear_model_data["{}_fit".format(self.yname)].values
|
|
2618
|
+
|
|
2619
|
+
# simulation loop:
|
|
2620
|
+
for n in range(1, n_sim + 1):
|
|
2621
|
+
_sim = lst_mcsim[n]
|
|
2622
|
+
|
|
2623
|
+
# get new error
|
|
2624
|
+
vct_e = np.random.normal(0.0, n_std_e, len(self.linear_model_data))
|
|
2625
|
+
# get new y obs
|
|
2626
|
+
vct_yobs = vct_yfit + vct_e
|
|
2627
|
+
|
|
2628
|
+
# fit new model
|
|
2629
|
+
if p0 is None:
|
|
2630
|
+
popt, pcov = curve_fit(
|
|
2631
|
+
f=linear, xdata=self.linear_model_data[self.xname], ydata=vct_yobs
|
|
2632
|
+
)
|
|
2633
|
+
else:
|
|
2634
|
+
popt, pcov = curve_fit(
|
|
2635
|
+
f=linear,
|
|
2636
|
+
xdata=self.linear_model_data[self.xname],
|
|
2637
|
+
ydata=vct_yobs,
|
|
2638
|
+
p0=p0,
|
|
2639
|
+
)
|
|
2640
|
+
pstd = np.sqrt(np.diag(pcov))
|
|
2641
|
+
|
|
2642
|
+
# store model values
|
|
2643
|
+
_dct = {"Mean": popt, "Std": pstd}
|
|
2644
|
+
for s in _dct:
|
|
2645
|
+
for i in range(len(self.linear_model)):
|
|
2646
|
+
_s_par = self.linear_model["Parameter"].values[i]
|
|
2647
|
+
_s_key = "{}_{}".format(_s_par, s)
|
|
2648
|
+
_value = _dct[s][i]
|
|
2649
|
+
df_models[_s_key].values[n] = _value
|
|
2650
|
+
|
|
2651
|
+
# store prediction values
|
|
2652
|
+
_x = df_preds[self.xname].values
|
|
2653
|
+
df_preds["y_{}".format(_sim)] = linear(_x, *popt)
|
|
2654
|
+
|
|
2655
|
+
# get bands
|
|
2656
|
+
for i in range(len(df_preds)):
|
|
2657
|
+
_values = df_preds.values[i, 1:]
|
|
2658
|
+
df_bands["Min"].values[i] = np.min(_values)
|
|
2659
|
+
df_bands["Max"].values[i] = np.max(_values)
|
|
2660
|
+
df_bands["Mean"].values[i] = np.mean(_values)
|
|
2661
|
+
df_bands["p01"].values[i] = np.quantile(_values, 0.01)
|
|
2662
|
+
df_bands["p05"].values[i] = np.quantile(_values, 0.05)
|
|
2663
|
+
df_bands["p25"].values[i] = np.quantile(_values, 0.25)
|
|
2664
|
+
df_bands["p50"].values[i] = np.quantile(_values, 0.5)
|
|
2665
|
+
df_bands["p75"].values[i] = np.quantile(_values, 0.75)
|
|
2666
|
+
df_bands["p95"].values[i] = np.quantile(_values, 0.95)
|
|
2667
|
+
df_bands["p99"].values[i] = np.quantile(_values, 0.99)
|
|
2668
|
+
|
|
2669
|
+
# return base_object
|
|
2670
|
+
return {"Models": df_models, "Predictions": df_preds, "Bands": df_bands}
|
|
2671
|
+
|
|
2672
|
+
def assess_error_normality(self, model_type="Linear", clevel=0.95):
|
|
2673
|
+
self.updata_model_data(model_type=model_type)
|
|
2674
|
+
vct_e = self.models[model_type]["Data"]["e_Mean"].values
|
|
2675
|
+
uni = Univar(data=vct_e, name=self.name)
|
|
2676
|
+
df = uni.get_normality(clevel=clevel)
|
|
2677
|
+
return df
|
|
2678
|
+
|
|
2679
|
+
@staticmethod
|
|
2680
|
+
def bias(pred, obs):
|
|
2681
|
+
"""Compute the Bias between predicted and observated sample
|
|
2682
|
+
|
|
2683
|
+
:param pred: :class:`numpy.ndarray`
|
|
2684
|
+
vector of prediction vales
|
|
2685
|
+
:type pred: :class:`numpy.ndarray`
|
|
2686
|
+
:param obs: :class:`numpy.ndarray`
|
|
2687
|
+
vector of observed values
|
|
2688
|
+
:type obs: :class:`numpy.ndarray`
|
|
2689
|
+
:return: Bias value
|
|
2690
|
+
:rtype: float
|
|
2691
|
+
"""
|
|
2692
|
+
return 100 * np.sum(pred - obs) / np.sum(obs)
|
|
2693
|
+
|
|
2694
|
+
@staticmethod
|
|
2695
|
+
def rmse(pred, obs):
|
|
2696
|
+
"""Compute the RMSE metric between predicted and observated sample
|
|
2697
|
+
|
|
2698
|
+
:param pred: :class:`numpy.ndarray`
|
|
2699
|
+
vector of prediction vales
|
|
2700
|
+
:type pred: :class:`numpy.ndarray`
|
|
2701
|
+
:param obs: :class:`numpy.ndarray`
|
|
2702
|
+
vector of observed values
|
|
2703
|
+
:type obs: :class:`numpy.ndarray`
|
|
2704
|
+
:return: RMSE value
|
|
2705
|
+
:rtype: float
|
|
2706
|
+
"""
|
|
2707
|
+
return np.sqrt(np.sum(np.power(pred - obs, 2)) / len(pred))
|
|
2708
|
+
|
|
2709
|
+
@staticmethod
|
|
2710
|
+
def mae(pred, obs):
|
|
2711
|
+
"""Compute the Mean Absolute Error (MAE) between predicted and observated sample
|
|
2712
|
+
|
|
2713
|
+
:param pred: :class:`numpy.ndarray`
|
|
2714
|
+
vector of prediction vales
|
|
2715
|
+
:type pred: :class:`numpy.ndarray`
|
|
2716
|
+
:param obs: :class:`numpy.ndarray`
|
|
2717
|
+
vector of observed values
|
|
2718
|
+
:type obs: :class:`numpy.ndarray`
|
|
2719
|
+
:return: MAE value
|
|
2720
|
+
:rtype: float
|
|
2721
|
+
"""
|
|
2722
|
+
return np.sum(np.abs(pred - obs)) / len(pred)
|
|
2723
|
+
|
|
2724
|
+
@staticmethod
|
|
2725
|
+
def rsq(pred, obs):
|
|
2726
|
+
"""Compute the R-square between predicted and observated sample
|
|
2727
|
+
|
|
2728
|
+
:param pred: :class:`numpy.ndarray`
|
|
2729
|
+
vector of prediction vales
|
|
2730
|
+
:type pred: :class:`numpy.ndarray`
|
|
2731
|
+
:param obs: :class:`numpy.ndarray`
|
|
2732
|
+
vector of observed values
|
|
2733
|
+
:type obs: :class:`numpy.ndarray`
|
|
2734
|
+
:return: R-square value
|
|
2735
|
+
:rtype: float
|
|
2736
|
+
"""
|
|
2737
|
+
return 1 - (
|
|
2738
|
+
np.sum(np.power(pred - obs, 2)) / np.sum(np.power(obs - np.mean(obs), 2))
|
|
2739
|
+
)
|
|
2740
|
+
|
|
2741
|
+
|
|
2742
|
+
# todo [upgrade] make it a child of some MbaE class
|
|
2743
|
+
class Bayes:
|
|
2744
|
+
# todo [docstring] improve major docstring
|
|
2745
|
+
"""
|
|
2746
|
+
The Bayes Theorem Analyst Object
|
|
2747
|
+
"""
|
|
2748
|
+
|
|
2749
|
+
def __init__(self, df_hypotheses, name="myBayes", nomenclature=None, gridsize=100):
|
|
2750
|
+
"""Deploy the Bayes Analyst
|
|
2751
|
+
:param df_hypotheses: dataframe listing all model hypotheses. Must contain a field Name (of parameter), Min and Max.
|
|
2752
|
+
:type df_hypotheses: :class:pandas.DataFrame
|
|
2753
|
+
:param name: name of analyst
|
|
2754
|
+
:type name: str
|
|
2755
|
+
:param nomenclature: dictionary for rename nomenclature
|
|
2756
|
+
:type nomenclature: dict
|
|
2757
|
+
:param gridsize: grid resolution in histrograms (bins)
|
|
2758
|
+
:type gridsize: int
|
|
2759
|
+
"""
|
|
2760
|
+
self.hypotheses = df_hypotheses
|
|
2761
|
+
# list of hypotheses by step
|
|
2762
|
+
self.bands = list()
|
|
2763
|
+
self.bands.append(self.hypotheses.copy())
|
|
2764
|
+
# other parameters
|
|
2765
|
+
self.name = name
|
|
2766
|
+
self.gridsize = gridsize
|
|
2767
|
+
|
|
2768
|
+
# zero
|
|
2769
|
+
self.zero = 1 / gridsize
|
|
2770
|
+
|
|
2771
|
+
# set labels
|
|
2772
|
+
self.shyp = "H"
|
|
2773
|
+
self.sevid = "E"
|
|
2774
|
+
self.sprior = "P(H)"
|
|
2775
|
+
self.slike = "P(E | H)"
|
|
2776
|
+
self.spost = "P(H | E)"
|
|
2777
|
+
self.saux = self.sprior + self.slike
|
|
2778
|
+
if nomenclature is not None:
|
|
2779
|
+
self._reset_nomenclature(dct_names=nomenclature)
|
|
2780
|
+
|
|
2781
|
+
# set omega
|
|
2782
|
+
self.steps = list()
|
|
2783
|
+
# create step 0
|
|
2784
|
+
self._insert_new_step()
|
|
2785
|
+
|
|
2786
|
+
def __str__(self):
|
|
2787
|
+
s1 = self.hypotheses.to_string()
|
|
2788
|
+
return s1
|
|
2789
|
+
|
|
2790
|
+
def _reset_nomenclature(self, dct_names):
|
|
2791
|
+
"""Reset nomenclature
|
|
2792
|
+
:param dct_names: dictionary to rename nomenclatures
|
|
2793
|
+
:type dct_names: dict
|
|
2794
|
+
:return: None
|
|
2795
|
+
:rtype: None
|
|
2796
|
+
"""
|
|
2797
|
+
for k in dct_names:
|
|
2798
|
+
self.sevid = self.sevid.replace(k, dct_names[k])
|
|
2799
|
+
self.shyp = self.shyp.replace(k, dct_names[k])
|
|
2800
|
+
self.sprior = self.sprior.replace(k, dct_names[k])
|
|
2801
|
+
self.slike = self.slike.replace(k, dct_names[k])
|
|
2802
|
+
self.spost = self.spost.replace(k, dct_names[k])
|
|
2803
|
+
self.saux = self.sprior + self.slike
|
|
2804
|
+
|
|
2805
|
+
def _insert_new_step(self):
|
|
2806
|
+
"""convenience void function for inserting new step objects
|
|
2807
|
+
:return: None
|
|
2808
|
+
:rtype:
|
|
2809
|
+
"""
|
|
2810
|
+
self.steps.append(dict())
|
|
2811
|
+
self.steps[len(self.steps) - 1]["Omega"] = dict()
|
|
2812
|
+
self.steps[len(self.steps) - 1]["Evidence"] = dict()
|
|
2813
|
+
self.steps[len(self.steps) - 1]["Bands"] = dict()
|
|
2814
|
+
for h in self.hypotheses["Name"].values:
|
|
2815
|
+
n_min = self.hypotheses.loc[self.hypotheses["Name"] == h, "Min"].values[0]
|
|
2816
|
+
n_max = self.hypotheses.loc[self.hypotheses["Name"] == h, "Max"].values[0]
|
|
2817
|
+
self.steps[len(self.steps) - 1]["Omega"][h] = pd.DataFrame(
|
|
2818
|
+
{
|
|
2819
|
+
self.shyp: np.linspace(n_min, n_max, self.gridsize),
|
|
2820
|
+
self.sprior: np.ones(self.gridsize) / self.gridsize,
|
|
2821
|
+
self.slike: np.zeros(self.gridsize),
|
|
2822
|
+
self.saux: np.zeros(self.gridsize),
|
|
2823
|
+
self.spost: np.zeros(self.gridsize),
|
|
2824
|
+
}
|
|
2825
|
+
)
|
|
2826
|
+
return None
|
|
2827
|
+
|
|
2828
|
+
def _accumulate(self, n_step):
|
|
2829
|
+
"""convenience void function for accumulate probability
|
|
2830
|
+
:param n_step: step number to accumulate
|
|
2831
|
+
:type n_step: int
|
|
2832
|
+
:return: None
|
|
2833
|
+
:rtype: none
|
|
2834
|
+
"""
|
|
2835
|
+
lst_labels1 = [self.sprior, self.slike, self.spost]
|
|
2836
|
+
for h in self.hypotheses["Name"].values:
|
|
2837
|
+
for i in range(len(lst_labels1)):
|
|
2838
|
+
s_field0 = lst_labels1[i]
|
|
2839
|
+
s_field1 = "{}_acc".format(lst_labels1[i])
|
|
2840
|
+
self.steps[n_step]["Omega"][h][s_field1] = 0.0
|
|
2841
|
+
for j in range(len(self.steps[n_step]["Omega"][h])):
|
|
2842
|
+
if j == 0:
|
|
2843
|
+
self.steps[n_step]["Omega"][h][s_field1].values[j] = self.steps[
|
|
2844
|
+
n_step
|
|
2845
|
+
]["Omega"][h][s_field0].values[j]
|
|
2846
|
+
else:
|
|
2847
|
+
self.steps[n_step]["Omega"][h][s_field1].values[j] = (
|
|
2848
|
+
self.steps[n_step]["Omega"][h][s_field1].values[j - 1]
|
|
2849
|
+
+ self.steps[n_step]["Omega"][h][s_field0].values[j]
|
|
2850
|
+
)
|
|
2851
|
+
return None
|
|
2852
|
+
|
|
2853
|
+
def conditionalize(self, dct_evidence, s_varfield="E", s_weightfield="W"):
|
|
2854
|
+
"""Conditionalize procedure of the Bayes Theorem
|
|
2855
|
+
:param dct_evidence: base_object of evidence dataframes
|
|
2856
|
+
:type dct_evidence: dict
|
|
2857
|
+
:param s_varfield: name of variable field in evidence dataframes
|
|
2858
|
+
:type s_varfield: str
|
|
2859
|
+
:param s_weightfield: name of weights field in evidence dataframes
|
|
2860
|
+
:type s_weightfield: str
|
|
2861
|
+
:return: None
|
|
2862
|
+
:rtype: none
|
|
2863
|
+
"""
|
|
2864
|
+
# instantiate new step
|
|
2865
|
+
n_step = len(self.steps)
|
|
2866
|
+
self._insert_new_step()
|
|
2867
|
+
|
|
2868
|
+
# set evidence dict for step
|
|
2869
|
+
self.steps[n_step]["Evidence"] = dct_evidence
|
|
2870
|
+
|
|
2871
|
+
# main loop
|
|
2872
|
+
for h in self.hypotheses["Name"].values:
|
|
2873
|
+
# get prior from last step
|
|
2874
|
+
if n_step > 1:
|
|
2875
|
+
self.steps[n_step]["Omega"][h][self.sprior] = self.steps[n_step - 1][
|
|
2876
|
+
"Omega"
|
|
2877
|
+
][h][self.spost].values
|
|
2878
|
+
# get likelihood
|
|
2879
|
+
hist, edges = np.histogram(
|
|
2880
|
+
a=dct_evidence[h][s_varfield],
|
|
2881
|
+
bins=self.steps[n_step]["Omega"][h][self.shyp],
|
|
2882
|
+
weights=dct_evidence[h][s_weightfield],
|
|
2883
|
+
)
|
|
2884
|
+
hist = hist / np.sum(hist)
|
|
2885
|
+
hist = list(hist)
|
|
2886
|
+
hist.append(0)
|
|
2887
|
+
self.steps[n_step]["Omega"][h][self.slike] = hist
|
|
2888
|
+
# insert baseline and normalize
|
|
2889
|
+
v_like = self.steps[n_step]["Omega"][h][self.slike].values
|
|
2890
|
+
v_like = v_like + self.zero
|
|
2891
|
+
self.steps[n_step]["Omega"][h][self.slike] = v_like / np.sum(v_like)
|
|
2892
|
+
|
|
2893
|
+
# get posterior with Bayes Theorem
|
|
2894
|
+
v_prior = self.steps[n_step]["Omega"][h][self.sprior].values
|
|
2895
|
+
v_like = self.steps[n_step]["Omega"][h][self.slike].values
|
|
2896
|
+
v_posterior = (v_prior * v_like) / np.sum(v_prior * v_like)
|
|
2897
|
+
self.steps[n_step]["Omega"][h][self.spost] = v_posterior
|
|
2898
|
+
|
|
2899
|
+
# get accumulated values
|
|
2900
|
+
self._accumulate(n_step)
|
|
2901
|
+
|
|
2902
|
+
self.steps[n_step]["Bands"][h] = dict()
|
|
2903
|
+
|
|
2904
|
+
# get percentiles of accumulated
|
|
2905
|
+
lst_aux = [self.sprior, self.slike, self.spost]
|
|
2906
|
+
for s in lst_aux:
|
|
2907
|
+
self.steps[n_step]["Bands"][h][s] = dict()
|
|
2908
|
+
# 90% band
|
|
2909
|
+
s_query = "`{}` > 0.05 and `{}` <= 0.95".format(s + "_acc", s + "_acc")
|
|
2910
|
+
df_aux = self.steps[n_step]["Omega"][h].query(s_query)
|
|
2911
|
+
self.steps[n_step]["Bands"][h][s]["p05"] = df_aux[self.shyp].min()
|
|
2912
|
+
self.steps[n_step]["Bands"][h][s]["p95"] = df_aux[self.shyp].max()
|
|
2913
|
+
# 50% band
|
|
2914
|
+
s_query = "`{}` > 0.25 and `{}` <= 0.75".format(s + "_acc", s + "_acc")
|
|
2915
|
+
df_aux = self.steps[n_step]["Omega"][h].query(s_query)
|
|
2916
|
+
self.steps[n_step]["Bands"][h][s]["p25"] = df_aux[self.shyp].min()
|
|
2917
|
+
self.steps[n_step]["Bands"][h][s]["p75"] = df_aux[self.shyp].max()
|
|
2918
|
+
|
|
2919
|
+
# return
|
|
2920
|
+
return None
|
|
2921
|
+
|
|
2922
|
+
def plot_step(
|
|
2923
|
+
self,
|
|
2924
|
+
n_step,
|
|
2925
|
+
folder="C:/sample",
|
|
2926
|
+
filename="bayes",
|
|
2927
|
+
specs=None,
|
|
2928
|
+
dpi=300,
|
|
2929
|
+
show=False,
|
|
2930
|
+
):
|
|
2931
|
+
"""Void function for plot pannel of conditionalization step
|
|
2932
|
+
:param n_step: step number
|
|
2933
|
+
:type n_step: int
|
|
2934
|
+
:param folder: export folder
|
|
2935
|
+
:type folder: str
|
|
2936
|
+
:param filename: file name
|
|
2937
|
+
:type filename: str
|
|
2938
|
+
:param specs: plot specs base_object
|
|
2939
|
+
:type specs: dict
|
|
2940
|
+
:param dpi: plot resolution
|
|
2941
|
+
:type dpi: int
|
|
2942
|
+
:param show: control to show plot instead of saving
|
|
2943
|
+
:type show: bool
|
|
2944
|
+
:return: None
|
|
2945
|
+
:rtype: None
|
|
2946
|
+
"""
|
|
2947
|
+
# get specs
|
|
2948
|
+
default_specs = {
|
|
2949
|
+
"color": "tab:grey",
|
|
2950
|
+
"title": "Conditionalization",
|
|
2951
|
+
"width": 10,
|
|
2952
|
+
"height": 5,
|
|
2953
|
+
"xlabel": "value",
|
|
2954
|
+
"range": (0, 1),
|
|
2955
|
+
"range_x": (0, 1),
|
|
2956
|
+
"subtitle": None,
|
|
2957
|
+
}
|
|
2958
|
+
# handle inputs specs
|
|
2959
|
+
if specs is None:
|
|
2960
|
+
pass
|
|
2961
|
+
else: # override default
|
|
2962
|
+
for k in specs:
|
|
2963
|
+
default_specs[k] = specs[k]
|
|
2964
|
+
specs = default_specs
|
|
2965
|
+
|
|
2966
|
+
# hunt some parameters
|
|
2967
|
+
lst_pmax = list()
|
|
2968
|
+
for h in self.hypotheses["Name"].values: # get objects
|
|
2969
|
+
_df = self.steps[n_step]["Omega"][h]
|
|
2970
|
+
# get max p
|
|
2971
|
+
lst_pmax.append(
|
|
2972
|
+
np.max(
|
|
2973
|
+
[
|
|
2974
|
+
_df[self.sprior].max(),
|
|
2975
|
+
_df[self.slike].max(),
|
|
2976
|
+
_df[self.spost].max(),
|
|
2977
|
+
]
|
|
2978
|
+
)
|
|
2979
|
+
)
|
|
2980
|
+
n_pmax = 1.5 * np.max(lst_pmax)
|
|
2981
|
+
|
|
2982
|
+
# main loop
|
|
2983
|
+
for h in self.hypotheses["Name"].values:
|
|
2984
|
+
specs["subtitle"] = "${}$".format(h)
|
|
2985
|
+
# get objects
|
|
2986
|
+
_df = self.steps[n_step]["Omega"][h]
|
|
2987
|
+
_bands = self.steps[n_step]["Bands"][h]
|
|
2988
|
+
|
|
2989
|
+
# get width
|
|
2990
|
+
_wid = 0.7 * (_df[self.shyp].values[1] - _df[self.shyp].values[0])
|
|
2991
|
+
|
|
2992
|
+
# get min max
|
|
2993
|
+
n_min = self.hypotheses.loc[self.hypotheses["Name"] == h, "Min"].values[0]
|
|
2994
|
+
n_max = self.hypotheses.loc[self.hypotheses["Name"] == h, "Max"].values[0]
|
|
2995
|
+
# start plot
|
|
2996
|
+
|
|
2997
|
+
fig = plt.figure(figsize=(specs["width"], specs["height"])) # Width, Height
|
|
2998
|
+
|
|
2999
|
+
if specs["subtitle"] is None:
|
|
3000
|
+
plt.suptitle(specs["title"])
|
|
3001
|
+
else:
|
|
3002
|
+
plt.suptitle("{} | {}".format(specs["title"], specs["subtitle"]))
|
|
3003
|
+
# grid
|
|
3004
|
+
gs = mpl.gridspec.GridSpec(
|
|
3005
|
+
2, 3, wspace=0.4, hspace=0.5, left=0.1, bottom=0.1, top=0.85, right=0.95
|
|
3006
|
+
) # nrows, ncols
|
|
3007
|
+
|
|
3008
|
+
# grid loop
|
|
3009
|
+
lst_aux = [self.sprior, self.slike, self.spost]
|
|
3010
|
+
for i in range(len(lst_aux)):
|
|
3011
|
+
# Histogram
|
|
3012
|
+
ax = fig.add_subplot(gs[0, i])
|
|
3013
|
+
plt.title(lst_aux[i])
|
|
3014
|
+
plt.bar(
|
|
3015
|
+
_df[self.shyp],
|
|
3016
|
+
_df[lst_aux[i]],
|
|
3017
|
+
width=_wid,
|
|
3018
|
+
color=specs["color"],
|
|
3019
|
+
align="edge",
|
|
3020
|
+
)
|
|
3021
|
+
plt.ylim(0, n_pmax)
|
|
3022
|
+
plt.xlim(n_min, n_max)
|
|
3023
|
+
plt.ylabel("$p({})$".format(h))
|
|
3024
|
+
plt.xlabel("${}$".format(h))
|
|
3025
|
+
# plt.gca().set_aspect(1 / (1.5))
|
|
3026
|
+
|
|
3027
|
+
if i == 1:
|
|
3028
|
+
# Accumulated
|
|
3029
|
+
ax = fig.add_subplot(gs[1, i])
|
|
3030
|
+
# plt.title(lst_aux[i])
|
|
3031
|
+
plt.scatter(
|
|
3032
|
+
self.steps[n_step]["Evidence"][h]["E"],
|
|
3033
|
+
self.steps[n_step]["Evidence"][h]["W"],
|
|
3034
|
+
marker=".",
|
|
3035
|
+
c="k",
|
|
3036
|
+
alpha=0.5,
|
|
3037
|
+
linewidths=0.0,
|
|
3038
|
+
edgecolors=None,
|
|
3039
|
+
)
|
|
3040
|
+
plt.xlim(n_min, n_max)
|
|
3041
|
+
plt.ylabel("$W({})$".format(h))
|
|
3042
|
+
plt.xlabel("${}$".format(h))
|
|
3043
|
+
# plt.gca().set_aspect(1 / (1.5))
|
|
3044
|
+
else:
|
|
3045
|
+
# Accumulated
|
|
3046
|
+
ax = fig.add_subplot(gs[1, i])
|
|
3047
|
+
# plt.title(lst_aux[i])
|
|
3048
|
+
plt.plot(
|
|
3049
|
+
_df[self.shyp],
|
|
3050
|
+
_df["{}_acc".format(lst_aux[i])],
|
|
3051
|
+
color="tab:blue",
|
|
3052
|
+
zorder=3,
|
|
3053
|
+
)
|
|
3054
|
+
plt.fill_betweenx(
|
|
3055
|
+
[-1, 2],
|
|
3056
|
+
x1=self.steps[n_step]["Bands"][h][lst_aux[i]]["p25"],
|
|
3057
|
+
x2=self.steps[n_step]["Bands"][h][lst_aux[i]]["p75"],
|
|
3058
|
+
color="tab:grey",
|
|
3059
|
+
alpha=0.5,
|
|
3060
|
+
zorder=2,
|
|
3061
|
+
)
|
|
3062
|
+
plt.fill_betweenx(
|
|
3063
|
+
[-1, 2],
|
|
3064
|
+
x1=self.steps[n_step]["Bands"][h][lst_aux[i]]["p05"],
|
|
3065
|
+
x2=self.steps[n_step]["Bands"][h][lst_aux[i]]["p95"],
|
|
3066
|
+
color="tab:grey",
|
|
3067
|
+
alpha=0.3,
|
|
3068
|
+
zorder=1,
|
|
3069
|
+
)
|
|
3070
|
+
plt.ylim(-0.05, 1.05)
|
|
3071
|
+
plt.xlim(n_min, n_max)
|
|
3072
|
+
plt.ylabel("$P({})$".format(h))
|
|
3073
|
+
plt.xlabel("${}$".format(h))
|
|
3074
|
+
# plt.gca().set_aspect(1 / (1.5))
|
|
3075
|
+
|
|
3076
|
+
# show or save
|
|
3077
|
+
if show:
|
|
3078
|
+
plt.show()
|
|
3079
|
+
else:
|
|
3080
|
+
plt.savefig(
|
|
3081
|
+
"{}/{}_{}_step{}.png".format(folder, filename, h, n_step), dpi=dpi
|
|
3082
|
+
)
|
|
3083
|
+
|
|
3084
|
+
# return
|
|
3085
|
+
return None
|
|
3086
|
+
|
|
3087
|
+
|
|
3088
|
+
# ----- DEPRECATED CLASSES -----
|
|
3089
|
+
|
|
3090
|
+
|
|
3091
|
+
if __name__ == "__main__":
|
|
3092
|
+
print("Hello world!")
|