plans 1.1.0.post1.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
plans/analyst.py ADDED
@@ -0,0 +1,3092 @@
1
+ # SPDX-License-Identifier: GPL-3.0-or-later
2
+ #
3
+ # Copyright (C) 2025 The Project Authors
4
+ # See pyproject.toml for authors/maintainers.
5
+ # See LICENSE for license details.
6
+ """
7
+ Classes designed to handle statistical analysis.
8
+
9
+ Overview
10
+ --------
11
+
12
+ # todo [major docstring improvement] -- overview
13
+ Mauris gravida ex quam, in porttitor lacus lobortis vitae.
14
+ In a lacinia nisl. Pellentesque habitant morbi tristique senectus
15
+ et netus et malesuada fames ac turpis egestas.
16
+
17
+ Example
18
+ -------
19
+
20
+ # todo [major docstring improvement] -- examples
21
+ Lorem ipsum dolor sit amet, consectetur adipiscing elit.
22
+ Nulla mollis tincidunt erat eget iaculis. Mauris gravida ex quam,
23
+ in porttitor lacus lobortis vitae. In a lacinia nisl.
24
+
25
+ .. code-block:: python
26
+
27
+ import numpy as np
28
+ from plans import analyst
29
+
30
+ # get data to a vector
31
+ data_vector = np.random.rand(1000)
32
+
33
+ # instantiate the Univar object
34
+ uni = analyst.Univar(sample=data_vector, name="my_data")
35
+
36
+ # view sample
37
+ uni.view()
38
+
39
+ Mauris gravida ex quam, in porttitor lacus lobortis vitae.
40
+ In a lacinia nisl.
41
+
42
+ """
43
+
44
+ # IMPORTS
45
+ # ***********************************************************************
46
+ # import modules from other libs
47
+
48
+ # Native imports
49
+ # =======================================================================
50
+ import os
51
+
52
+ # External imports
53
+ # =======================================================================
54
+ import numpy as np
55
+ import pandas as pd
56
+ import matplotlib as mpl
57
+ import matplotlib.pyplot as plt
58
+ import matplotlib.ticker as mticker
59
+ import matplotlib.colors as mcolors
60
+
61
+ # Project-level imports
62
+ # =======================================================================
63
+ from plans import viewer
64
+ from plans.root import DataSet
65
+
66
+
67
+ # FUNCTIONS
68
+ # ***********************************************************************
69
+
70
+
71
+ def linear(x, c0, c1):
72
+ """Linear function f(x) = c0 + c1 * x
73
+
74
+ :param x: function inputs
75
+ :type x: float | :class:`numpy.ndarray`
76
+ :param c0: translational parameter
77
+ :type c0: float
78
+ :param c1: scaling parameter
79
+ :type c1: float
80
+ :return: function output
81
+ :rtype: float | :class:`numpy.ndarray`
82
+ """
83
+ return c0 + (x * c1)
84
+
85
+
86
+ def power(x, c0, c1, c2):
87
+ """Power function f(x) = c2 * ((x + c0)^c1)
88
+
89
+ :param x: function inputs
90
+ :type x: float | :class:`numpy.ndarray`
91
+ :param c0: translational parameter
92
+ :type c0: float
93
+ :param c1: exponent parameter
94
+ :type c1: float
95
+ :param c2: scaling parameter
96
+ :type c2: float
97
+ :return: function output
98
+ :rtype: float | :class:`numpy.ndarray`
99
+ """
100
+ return c2 * (np.power((x + c0), c1))
101
+
102
+
103
+ def power_zero(x, c0, c1):
104
+ """Power function with root in zero f(x) = c1 * ((x)^c0)
105
+
106
+ :param x: function inputs
107
+ :type x: float | :class:`numpy.ndarray`
108
+ :param c0: exponent parameter
109
+ :type c0: float
110
+ :param c1: scaling parameter
111
+ :type c1: float
112
+ :return: function output
113
+ :rtype: float | :class:`numpy.ndarray`
114
+ """
115
+ return c1 * (np.power((x), c0))
116
+
117
+
118
+ # CLASSES
119
+ # ***********************************************************************
120
+
121
+
122
+ class Univar(DataSet):
123
+ """
124
+ The Univariate object
125
+
126
+ """
127
+
128
+ def __init__(self, name="MyUnivar", alias="Uv0"):
129
+ # ------------ set defaults ----------- #
130
+ self.varfield = "V"
131
+ self.varalias = "Var"
132
+ self.varname = "Variable"
133
+ self.units = "units"
134
+
135
+ # ------------ call super ----------- #
136
+ super().__init__(name=name, alias=alias)
137
+
138
+ # overwriters
139
+
140
+ # attributes
141
+ self.stats_df = None
142
+ self.freq_df = None
143
+ self.weibull_df = None
144
+
145
+ def load_data(self, file_data):
146
+ """
147
+ Load data from file. Expected to overwrite superior methods.
148
+
149
+ :param file_data: file path to data.
150
+ :type file_data: str
151
+ :return: None
152
+ :rtype: None
153
+ """
154
+
155
+ # -------------- overwrite relative path inputs -------------- #
156
+ self.file_data = os.path.abspath(file_data)
157
+
158
+ # -------------- implement loading logic -------------- #
159
+ default_columns = {
160
+ #'DateTime': 'datetime64[1s]',
161
+ self.varfield: float,
162
+ }
163
+
164
+ # -------------- call loading function -------------- #
165
+ self.data = pd.read_csv(
166
+ self.file_data,
167
+ sep=self.file_csv_sep,
168
+ dtype=default_columns,
169
+ usecols=list(default_columns.keys()),
170
+ )
171
+
172
+ # -------------- post-loading logic -------------- #
173
+ self.data.dropna(inplace=True)
174
+
175
+ # -------------- update other mutables -------------- #
176
+ self.update()
177
+
178
+ # ... continues in downstream objects ... #
179
+
180
+ return None
181
+
182
+ def set_array(self, array):
183
+ """
184
+ Set array to data
185
+
186
+ :param array: Numpy array
187
+ :type array: :class:`numpy.ndarray`
188
+ :return: None
189
+ :rtype: None
190
+ """
191
+ self.data = pd.DataFrame({self.varfield: array})
192
+ self.update()
193
+ return None
194
+
195
+ def update(self):
196
+ # todo [docstring]
197
+ # ------------ call super ----------- #
198
+ super().update()
199
+
200
+ # update stats
201
+ if self.data is not None:
202
+ self.stats_df = self.get_basic_stats()
203
+ self.freq_df = self.get_frequency()
204
+ self.weibull_df = self.get_cdf_weibull()
205
+
206
+ # ... continues in downstream objects ... #
207
+ return None
208
+
209
+ # GET METHODS
210
+ # ===================================================================
211
+
212
+ def get_normality(self, clevel=0.95):
213
+ """
214
+ Get normality analysis using standard tests
215
+
216
+ :return: dataframe of assessment results
217
+ :rtype: :class:`pandas.DataFrame`
218
+ """
219
+ df_clean = self.data.dropna()
220
+ data = df_clean[self.varfield].values
221
+
222
+ # run tests
223
+ lst_tests = []
224
+ lst_tests.append(Univar.test_normality_ks(data, clevel=clevel))
225
+ lst_tests.append(Univar.test_normality_sw(data, clevel=clevel))
226
+ lst_tests.append(Univar.test_normality_dp(data, clevel=clevel))
227
+ # create dataframe
228
+ lst_names = []
229
+ lst_stats = []
230
+ lst_p = []
231
+ lst_clvl = []
232
+ lst_is = []
233
+ for e in lst_tests:
234
+ lst_names.append(e["Test"])
235
+ lst_stats.append(e["Statistic"])
236
+ lst_p.append(e["p-value"])
237
+ lst_clvl.append(e["Confidence"])
238
+ lst_is.append(e["is_normal"])
239
+ df_result = pd.DataFrame(
240
+ {
241
+ "Test": lst_names,
242
+ "Statistic": lst_stats,
243
+ "p-value": lst_p,
244
+ "is_normal": lst_is,
245
+ "Confidence": lst_clvl,
246
+ }
247
+ )
248
+ return df_result
249
+
250
+ def get_frequency(self):
251
+ """
252
+ Get data frequencies
253
+
254
+ :return: result dataframe
255
+ :rtype: :class:`pandas.DataFrame`
256
+ """
257
+
258
+ df_clean = self.data.dropna()
259
+ data = df_clean[self.varfield].values
260
+
261
+ # compute percentiles intevarls
262
+ vct_percentiles = np.arange(0, 100)
263
+
264
+ # get CFC values
265
+ vct_cfc = np.percentile(data, vct_percentiles)
266
+
267
+ # reverse to get exceedance
268
+ vct_exeed = 100 - vct_percentiles
269
+
270
+ # get count
271
+ vct_count = np.histogram(data, bins=len(vct_percentiles))[0]
272
+
273
+ # get empirical prob in the histogram
274
+ vct_empprob = vct_count / np.sum(vct_count)
275
+
276
+ df_result = pd.DataFrame(
277
+ {
278
+ "Percentiles": vct_percentiles,
279
+ "Exceedance": vct_exeed,
280
+ "Frequency": vct_count,
281
+ "Empirical Probability": vct_empprob,
282
+ "Values": vct_cfc,
283
+ }
284
+ )
285
+ return df_result
286
+
287
+ def get_basic_stats(self):
288
+ """
289
+ Get basic statistics data of the variable field.
290
+
291
+ :return: DataFrame with statistics and their values.
292
+ :rtype: :class:`pandas.DataFrame`
293
+ """
294
+
295
+ def _compute(data):
296
+ return {
297
+ "count": len(data),
298
+ "sum": np.sum(data),
299
+ "mean": np.mean(data),
300
+ "sd": np.std(data),
301
+ "min": np.min(data),
302
+ "p01": np.percentile(data, 1),
303
+ "p05": np.percentile(data, 5),
304
+ "p10": np.percentile(data, 10),
305
+ "p20": np.percentile(data, 20),
306
+ "p25": np.percentile(data, 25),
307
+ "p40": np.percentile(data, 40),
308
+ "p50": np.percentile(data, 50),
309
+ "p60": np.percentile(data, 60),
310
+ "p75": np.percentile(data, 75),
311
+ "p80": np.percentile(data, 80),
312
+ "p90": np.percentile(data, 90),
313
+ "p95": np.percentile(data, 95),
314
+ "p99": np.percentile(data, 99),
315
+ "max": np.max(data),
316
+ }
317
+
318
+ # handle data
319
+ df_clean = self.data.dropna()
320
+ # get data
321
+ data = df_clean[self.varfield].values
322
+
323
+ # handle void case
324
+ if len(df_clean) == 0:
325
+ dct = _compute(data=[0]) # feed dummy values
326
+ ls_values = [np.nan for key in dct]
327
+ ls_values[0] = 0 # fix for count
328
+ df_result = pd.DataFrame(
329
+ {"statistic": list(dct.keys()), "value": ls_values}
330
+ )
331
+ else:
332
+ dct = _compute(data=data)
333
+ ls_values = [dct[key] for key in dct]
334
+ df_result = pd.DataFrame({"statistic": list(dct.keys()), "value": ls_values})
335
+ return df_result
336
+
337
+ def get_cdf_weibull(self):
338
+ """
339
+ Get the Weibull model for the data
340
+
341
+ :param x: function inputs
342
+ :type x: :class:`numpy.ndarray`
343
+ :return: model dataframe
344
+ :rtype: :class:`pandas.DataFrame`
345
+ """
346
+ df_clean = self.data.dropna()
347
+ x = df_clean[self.varfield].values
348
+ v = np.sort(x)
349
+ vr = v[::-1]
350
+ r = np.arange(0, len(x)) + 1
351
+ px = Univar.weibull_px(ranks=r)
352
+
353
+ result_df = pd.DataFrame({"Data": vr, "P(X)": px, "F(X)": 1 - px})
354
+
355
+ return result_df
356
+
357
+ def get_cdf_gumbel(self):
358
+ """
359
+ Get the Gumbel CDF for the data (it assumes that is a maxima dataset)
360
+
361
+ :return: multiple results from the assessment
362
+ :rtype: dict
363
+ """
364
+ from scipy import stats
365
+
366
+ # clean data
367
+ df = self.data.dropna()
368
+
369
+ # get extra values
370
+ df = df.sort_values(by=self.varfield, ascending=False).reset_index(drop=True)
371
+ df["Rank"] = df.index + 1
372
+ df["P(X)_Empirical"] = Univar.empirical_px(ranks=df["Rank"].values)
373
+ df["P(X)_Weibull"] = Univar.weibull_px(ranks=df["Rank"].values)
374
+ df["T(X)_Weibull"] = 1 / df["P(X)_Weibull"]
375
+ df["P(X)_Gringorten"] = Univar.gringorten_px(ranks=df["Rank"].values)
376
+
377
+ # ---- model Gumbel using the method of moments ----
378
+
379
+ # using scipy
380
+ # Fit a Gumbel distribution to the data
381
+ params = stats.gumbel_r.fit(df[self.varfield].values, method="MM") # MLM or MM
382
+ gumbel_a = params[0]
383
+ gumbel_b = params[1]
384
+
385
+ # compute fit gumbel values
386
+ df["P(X)_Gumbel"] = 1 - Univar.gumbel_fx(
387
+ x=df[self.varfield].values, a=gumbel_a, b=gumbel_b
388
+ )
389
+ # return period
390
+ df["T(X)_Gumbel"] = 1 / df["P(X)_Gumbel"]
391
+
392
+ # compute uncertainty 90% bands for T(X) in the Annual Series
393
+ t = stats.t.ppf((1 + 0.9) / 2, df=len(df) - 1)
394
+ gumbel_se = Univar.gumbel_se(
395
+ std_sample=np.std(a=df[self.varfield].values),
396
+ n_sample=len(df),
397
+ tx=df["T(X)_Gumbel"],
398
+ )
399
+ # Standar error analysis
400
+ df["T(X)_Gumbel_SE90"] = t * gumbel_se
401
+ h_max = df[self.varfield] + df["T(X)_Gumbel_SE90"]
402
+ h_min = df[self.varfield] - df["T(X)_Gumbel_SE90"]
403
+ df["T(X)_Gumbel_P05"] = Univar.gumbel_tx(x=h_min, a=gumbel_a, b=gumbel_b)
404
+ df["T(X)_Gumbel_P95"] = Univar.gumbel_tx(x=h_max, a=gumbel_a, b=gumbel_b)
405
+
406
+ # -------- Return Period analysis
407
+
408
+ # evely spaced
409
+ txs = np.arange(2, 1001, step=1)
410
+ k = Univar.gumbel_freqfactor(txs)
411
+
412
+ # apply reverse formula
413
+ stages = np.mean(a=df[self.varfield]) + k * np.std(a=df[self.varfield])
414
+
415
+ # get standard error
416
+ gumbel_se = Univar.gumbel_se(
417
+ std_sample=np.std(a=df[self.varfield]), n_sample=len(df), tx=txs
418
+ )
419
+ # compute 50% uncertainty bands by Student's t distribution
420
+ t_50 = stats.t.ppf((1 + 0.5) / 2, df=len(df) - 1)
421
+ h_range_50 = t_50 * gumbel_se
422
+
423
+ # compute 90% uncertainty bands by Student's t distribution
424
+ t_90 = stats.t.ppf((1 + 0.9) / 2, df=len(df) - 1)
425
+ h_range_90 = t_90 * gumbel_se
426
+
427
+ # Set TX dataframe
428
+ df_tx = pd.DataFrame(
429
+ {
430
+ "T(X)_Gumbel": txs,
431
+ f"{self.varfield}": stages,
432
+ f"{self.varfield}_P25": stages - h_range_50,
433
+ f"{self.varfield}_P75": stages + h_range_50,
434
+ f"{self.varfield}_P05": stages - h_range_90,
435
+ f"{self.varfield}_P95": stages + h_range_90,
436
+ }
437
+ )
438
+
439
+ # -------- QQ plots
440
+ qq, qq_params = stats.probplot(
441
+ x=df[self.varfield].values, dist="gumbel_r", sparams=params
442
+ )
443
+ # Set QQ dataframe
444
+ df_qq = pd.DataFrame({self.varfield: qq[1], "T-Q": qq[0]})
445
+
446
+ # -------- Goodness of fit tests
447
+ ks_stat, ks_p_value = stats.kstest(
448
+ df[self.varfield].values, cdf="gumbel_r", args=params
449
+ )
450
+ # accept Null Hypothesis
451
+ is_gumbel = True
452
+ if ks_p_value <= 0.05:
453
+ is_gumbel = False
454
+
455
+ # -------- Metadata dataframe
456
+ df_meta = pd.DataFrame(
457
+ {
458
+ "Metadata": [
459
+ "N",
460
+ "Gumbel a",
461
+ "Gumbel b",
462
+ "KS-test s-value",
463
+ "KS-test p-value",
464
+ "KS-test Is Gumbel (NullH)",
465
+ "QQ-plot c0",
466
+ "QQ plot c1",
467
+ "QQ-plot r2",
468
+ ],
469
+ "Value": [
470
+ len(df),
471
+ gumbel_a,
472
+ gumbel_b,
473
+ ks_stat,
474
+ ks_p_value,
475
+ is_gumbel,
476
+ qq_params[1],
477
+ qq_params[0],
478
+ qq_params[2],
479
+ ],
480
+ }
481
+ )
482
+
483
+ return {"Data": df, "Data_T(X)": df_tx, "Data_QQ": df_qq, "Metadata": df_meta}
484
+
485
+ # PLOT METHODS
486
+ # ===================================================================
487
+ # Plotting methods
488
+ # todo [DRY] evaluate to discard or refactor for viewer styles
489
+ def plot_hist(
490
+ self,
491
+ bins=100,
492
+ colored=False,
493
+ annotated=False,
494
+ rule=None,
495
+ show=False,
496
+ folder="C:/sample",
497
+ filename="histogram",
498
+ specs=None,
499
+ dpi=300,
500
+ ):
501
+ """
502
+ Plot histogram of sample
503
+
504
+ :param bins: number of bins
505
+ :type bins: int
506
+ :param colored: Boolean to quantile-colored histogram
507
+ :type colored: bool
508
+ :param annotated: Boolean to plot stats texts over histogram
509
+ :type annotated: bool
510
+ :param rule: name of rule to compute bins
511
+ :type rule: str
512
+ :param show: Boolean to show instead of saving
513
+ :type show: bool
514
+ :param folder: output folder
515
+ :type folder: str
516
+ :param filename: image file name
517
+ :type filename: str
518
+ :param specs: specification dictionary
519
+ :type specs: dict
520
+ :param dpi: image resolution (default = 96)
521
+ :type dpi: int
522
+ """
523
+
524
+ # get bins
525
+ if rule is None:
526
+ pass
527
+ else:
528
+ bins = self.get_nbins_by_rule(rule=rule)
529
+
530
+ # get specs
531
+ default_specs = {
532
+ "color": "tab:grey",
533
+ "title": "Histogram of {}".format(self.name),
534
+ "width": 4 * 1.618,
535
+ "height": 4,
536
+ "xlabel": "value",
537
+ "range": (0, 0.1),
538
+ "range_x": (
539
+ 0.95 * np.min(self.data[self.varfield]),
540
+ 1.05 * np.max(self.data[self.varfield]),
541
+ ),
542
+ "subtitle": None,
543
+ "cmap": "viridis",
544
+ "colors": None,
545
+ "n_classes": 5,
546
+ "grid": False,
547
+ "quantiles": True,
548
+ }
549
+ # handle inputs specs
550
+ if specs is None:
551
+ pass
552
+ else: # override default
553
+ for k in specs:
554
+ default_specs[k] = specs[k]
555
+ specs = default_specs
556
+
557
+ # start plot
558
+ fig = plt.figure(figsize=(specs["width"], specs["height"])) # Width, Height
559
+ ax = plt.gca()
560
+ ax.set_position([0.15, 0.15, 0.75, 0.75])
561
+ if specs["subtitle"] is None:
562
+ plt.title(specs["title"])
563
+ else:
564
+ plt.title("{} | {}".format(specs["title"], specs["subtitle"]))
565
+
566
+ # Calculate quantiles
567
+ data = pd.Series(self.data[self.varfield])
568
+ # filter data to range x
569
+ data = data[data.between(specs["range_x"][0], specs["range_x"][1])]
570
+ if specs["quantiles"]:
571
+ quantiles = data.quantile(np.linspace(0, 1, specs["n_classes"] + 1))
572
+ else:
573
+ quantiles = pd.Series(
574
+ np.linspace(
575
+ start=data.min(), stop=data.max(), num=specs["n_classes"] + 1
576
+ )
577
+ )
578
+
579
+ # Determine the overall bin width you desire
580
+ bin_width = (
581
+ data.max() - data.min()
582
+ ) / bins # for example, 40 equal width bins across the data range
583
+ # handle colored plots
584
+ if colored:
585
+ if specs["colors"] is None:
586
+ cmap = mpl.colormaps[specs["cmap"]]
587
+ colors = [cmap(i) for i in np.linspace(0, 1, len(quantiles) - 1)]
588
+ else:
589
+ colors = specs["colors"]
590
+
591
+ # Create the bins
592
+ binsv = np.arange(start=data.min(), stop=data.max(), step=bin_width)
593
+ binsv = np.append(
594
+ binsv, data.max()
595
+ ) # Ensure the last bin includes the max value
596
+
597
+ # Plot the histogram
598
+ n, bins, patches = plt.hist(
599
+ self.data,
600
+ bins=binsv,
601
+ linewidth=0,
602
+ weights=np.ones(len(self.data)) / len(self.data),
603
+ )
604
+
605
+ # Color the bars based on the quantile
606
+ for patch, edge in zip(patches, bins[:-1]):
607
+ if edge < quantiles.values[1]:
608
+ plt.setp(patch, "facecolor", colors[0])
609
+ elif edge < quantiles.values[2]:
610
+ plt.setp(patch, "facecolor", colors[1])
611
+ elif edge < quantiles.values[3]:
612
+ plt.setp(patch, "facecolor", colors[2])
613
+ elif edge < quantiles.values[4]:
614
+ plt.setp(patch, "facecolor", colors[3])
615
+ else:
616
+ plt.setp(patch, "facecolor", colors[4])
617
+ else:
618
+ plt.hist(
619
+ self.data,
620
+ bins=bins,
621
+ weights=np.ones(len(self.data)) / len(self.data),
622
+ color=specs["color"],
623
+ )
624
+
625
+ plt.xlabel(specs["xlabel"])
626
+ plt.ylim(specs["range"])
627
+ plt.xlim(specs["range_x"])
628
+ plt.grid(specs["grid"])
629
+ # Optionally, add vertical lines for each quantile and annotate them
630
+ mu = np.mean(self.data)
631
+ plt.axvline(mu, color="r", linestyle="dashed", linewidth=1)
632
+ plt.text(
633
+ mu,
634
+ plt.gca().get_ylim()[1] * 0.92,
635
+ rf"$\mu$ = {mu:.1f}",
636
+ ha="left",
637
+ va="bottom",
638
+ fontsize=10,
639
+ rotation=0,
640
+ color="red",
641
+ )
642
+ if annotated:
643
+ aux = 0
644
+ for q in quantiles:
645
+ plt.axvline(q, color="k", linestyle="dashed", linewidth=1)
646
+ plt.text(
647
+ q,
648
+ plt.gca().get_ylim()[1] * (0.9 - aux),
649
+ f"{q:.1f}",
650
+ ha="right",
651
+ va="bottom",
652
+ fontsize=10,
653
+ rotation=0,
654
+ )
655
+ aux = aux + 0.05
656
+ # plt.tight_layout()
657
+ # show or save
658
+ if show:
659
+ plt.show()
660
+ else:
661
+ plt.savefig(
662
+ "{}/{}_{}.png".format(folder, self.name, filename),
663
+ dpi=dpi,
664
+ bbox_inches="tight",
665
+ )
666
+
667
+ # todo [evaluate]
668
+ def plot_qqplot(
669
+ self, show=True, folder="C:/sample", filename="qqplot", specs=None, dpi=300
670
+ ):
671
+ """
672
+ Plot Q-Q Plot on Normal distribution
673
+
674
+ :param show: Boolean to show instead of saving
675
+ :type show: bool
676
+ :param folder: output folder
677
+ :type folder: str
678
+ :param filename: image file name
679
+ :type filename: str
680
+ :param specs: specification dictionary
681
+ :type specs: dct
682
+ :param dpi: image resolution (default = 300)
683
+ :type dpi: int
684
+ :return: None
685
+ :rtype: None
686
+ """
687
+ df_clean = self.data.dropna()
688
+ data = df_clean[self.varfield]
689
+ # get specs
690
+ default_specs = {
691
+ "color": "tab:grey",
692
+ "title": "Q-Q Plot of {}".format(self.name),
693
+ "width": 4 * 1.618,
694
+ "height": 4,
695
+ "xlabel": "value",
696
+ "range": (0.95 * np.min(data), 1.05 * np.max(data)),
697
+ "range_x": (-3, 3),
698
+ "subtitle": None,
699
+ }
700
+ # handle inputs specs
701
+ if specs is None:
702
+ pass
703
+ else: # override default
704
+ for k in specs:
705
+ default_specs[k] = specs[k]
706
+ specs = default_specs
707
+
708
+ # process quantiles
709
+ _df = Univar.qqplot(data=data)
710
+ # plot
711
+ fig = plt.figure(figsize=(specs["width"], specs["height"])) # Width, Height
712
+ # grid
713
+ gs = mpl.gridspec.GridSpec(
714
+ 1, 1, wspace=0.4, hspace=0.5, left=0.1, bottom=0.15, top=0.9, right=0.95
715
+ ) # nrows, ncols
716
+ # scatter plot
717
+ ax = fig.add_subplot(gs[0, 0])
718
+ plt.title(specs["title"])
719
+ plt.scatter(_df["T-Quantiles"], _df["Data"], marker=".", color="tab:grey")
720
+ plt.ylim([_df["Data"].min(), _df["Data"].max()])
721
+ plt.xlim(specs["range_x"])
722
+ plt.xlabel("Normal Theoretical Quantiles")
723
+ plt.ylabel("Data Empirical Quantiles")
724
+ plt.gca().set_aspect(
725
+ (specs["range_x"][1] - specs["range_x"][0])
726
+ / (_df["Data"].max() - _df["Data"].min())
727
+ )
728
+
729
+ # show or save
730
+ if show:
731
+ plt.show()
732
+ else:
733
+ plt.savefig("{}/{}_{}.png".format(folder, self.name, filename), dpi=dpi)
734
+
735
+ def _plot(self, fig, gs, specs):
736
+ # todo [docstring]
737
+
738
+ # from matplotlib.ticker import PercentFormatter # Import PercentFormatter
739
+ formatter = mticker.ScalarFormatter(useOffset=False, useMathText=False)
740
+ formatter.set_scientific(False) # Ensure scientific notation is off
741
+
742
+ # AUX DATA
743
+ # -------------------------------------------------------------------
744
+ if self.stats_df is None:
745
+ self.stats_df = self.get_basic_stats()
746
+ if self.weibull_df is None:
747
+ self.weibull_df = self.get_cdf_weibull()
748
+ # get mean
749
+ plot_mean = specs["plot_mean"]
750
+ y_mu = np.nanmean(self.data[self.varfield])
751
+
752
+ # RANGE
753
+ # -------------------------------------------------------------------
754
+ range_data = specs["range"]
755
+ # handle if none
756
+ if specs["range"] is None:
757
+ p_low = self.stats_df.loc[
758
+ self.stats_df["statistic"] == "p01", "value"
759
+ ].values[0]
760
+ p_hi = self.stats_df.loc[
761
+ self.stats_df["statistic"] == "p99", "value"
762
+ ].values[0]
763
+ rng = p_hi - p_low
764
+ rng_margin = rng * 0.1
765
+ range_data = (p_low - rng_margin, p_hi + rng_margin)
766
+ specs["range"] = range_data
767
+
768
+ # TITLE
769
+ # -------------------------------------------------------------------
770
+ if specs["title"] is not None:
771
+ plt.suptitle(specs["title"], fontsize=9)
772
+
773
+ # AXES
774
+ # -------------------------------------------------------------------
775
+ fig = self._build_axes(fig=fig, gs=gs, specs=specs)
776
+ all_axes = fig.get_axes()
777
+
778
+ # DATA
779
+ ax_data = False
780
+ if specs["ax_data"]:
781
+ ax_data = all_axes[specs["ax_data"]]
782
+
783
+ # HIST
784
+ ax_histh = False
785
+ if specs["ax_histh"]:
786
+ ax_histh = all_axes[specs["ax_histh"]]
787
+
788
+ # CDF
789
+ ax_cdf = False
790
+ if specs["ax_cdf"]:
791
+ ax_cdf = all_axes[specs["ax_cdf"]]
792
+
793
+ # CLEANUP
794
+ # -------------------------------------------------------------------
795
+ df_clean = self.data.dropna()
796
+ data = df_clean[self.varfield].values
797
+
798
+ # PLOT DATA
799
+ # -------------------------------------------------------------------
800
+ Univar.plot_scatter(
801
+ data=data,
802
+ ax=ax_data,
803
+ specs=specs,
804
+ formatter=formatter,
805
+ x_factor=2,
806
+ )
807
+
808
+ # PLOT HIST
809
+ # -------------------------------------------------------------------
810
+ Univar.plot_histh(
811
+ data=data,
812
+ ax=ax_histh,
813
+ specs=specs,
814
+ formatter=formatter,
815
+ )
816
+
817
+ # PLOT CDF
818
+ # -------------------------------------------------------------------
819
+ Univar.plot_cdf(
820
+ cdf_df=self.weibull_df,
821
+ ax=ax_cdf,
822
+ specs=specs,
823
+ formatter=formatter,
824
+ )
825
+
826
+ # PLOT STATS
827
+ # -------------------------------------------------------------------
828
+ if specs["mode"] == "full":
829
+ x_stats = specs["x_stats"]
830
+ x_stats_sep = specs["x_stats_sep"]
831
+ y_stats = specs["y_stats"]
832
+ fig = Univar.plot_stats(
833
+ fig=fig, stats_df=self.stats_df, x=x_stats, y=y_stats, sep=x_stats_sep
834
+ )
835
+
836
+ # RETURN FIG
837
+ # -------------------------------------------------------------------
838
+ return fig
839
+
840
+ def _build_axes(self, fig, gs, specs):
841
+ # todo [docstring]
842
+ # ------------ setup axes ------------
843
+ if specs["mode"] == "full":
844
+ fig.add_subplot(gs[2:10, 2:10])
845
+ fig.add_subplot(gs[2:10, 13:18])
846
+ fig.add_subplot(gs[2:10, 19:])
847
+ elif specs["mode"] == "mini":
848
+ fig.add_subplot(gs[2:10, 2:10])
849
+ fig.add_subplot(gs[2:10, 13:18])
850
+ fig.add_subplot(gs[2:10, 19:])
851
+ else:
852
+ fig.add_subplot(gs[2:10, 2:10])
853
+ fig.add_subplot(gs[2:10, 13:18])
854
+ fig.add_subplot(gs[2:10, 19:])
855
+ return fig
856
+
857
+ def _get_fig_specs(self):
858
+ # todo [docstring]
859
+ # handle specs
860
+ specs = self.view_specs.copy()
861
+
862
+ # handle mode
863
+ mode = specs["mode"]
864
+ if mode == "full":
865
+ specs_aux = {
866
+ "ncols": 24,
867
+ "nrows": 16,
868
+ "width": viewer.FIG_SIZES["M"]["w"],
869
+ "height": viewer.FIG_SIZES["M"]["h"],
870
+ }
871
+ else:
872
+ specs_aux = {
873
+ "ncols": 24,
874
+ "nrows": 12,
875
+ "width": viewer.FIG_SIZES["M2"]["w"],
876
+ "height": viewer.FIG_SIZES["M2"]["h"],
877
+ }
878
+ # update sizes
879
+ specs.update(specs_aux)
880
+
881
+ # update gridspecs
882
+ specs.update(viewer.GRID_SPECS)
883
+
884
+ return specs
885
+
886
+ def _set_view_specs(self):
887
+ """
888
+ Set view specifications. Expected to overwrite superior methods.
889
+
890
+ :return: None
891
+ :rtype: None
892
+ """
893
+ super()._set_view_specs()
894
+ self.view_specs.update(
895
+ {
896
+ # layout
897
+ "style": "wien",
898
+ "mode": "full",
899
+ "width": viewer.FIG_SIZES["M"]["w"],
900
+ "height": viewer.FIG_SIZES["M"]["h"],
901
+ # titles
902
+ "subtitle_data": "Distribution",
903
+ "subtitle_hist": "Histogram",
904
+ "subtitle_cdf": "CDF",
905
+ # fields
906
+ "yvar_field": self.varfield,
907
+ "xvar": "i",
908
+ "yvar": self.varname,
909
+ # axis labels
910
+ "xlabel": self.varname,
911
+ "xlabel_cdf": "P(X)",
912
+ "ylabel": self.units,
913
+ # data labels
914
+ "data_label": self.name,
915
+ "data_legend": False,
916
+ # colors
917
+ "color_hist": "tab:grey",
918
+ "color_cdf": "black",
919
+ "color_mean": "red",
920
+ "color_mean_text": None,
921
+ # cmaps
922
+ "scheme_cmap": None,
923
+ "cmap": "viridis",
924
+ "colorize_scatter": False,
925
+ # alphas
926
+ "alpha": 0.4,
927
+ "alpha_hist": 1,
928
+ "alpha_cdf": 1,
929
+ # ranges
930
+ "range": None,
931
+ "range_x": None,
932
+ # mean
933
+ "plot_mean": True,
934
+ "linestyle_mean": "solid",
935
+ "pad_mean": 1,
936
+ # histogram
937
+ "hist_density": True, # todo check this may be deprecated
938
+ "bins": 20,
939
+ "bins_density": False,
940
+ # axes
941
+ "ax_data": 0,
942
+ "ax_histh": 1,
943
+ "ax_cdf": 2,
944
+ # stats locators
945
+ "x_stats": 0.01,
946
+ "x_stats_sep": 0.2,
947
+ "y_stats": 0.3,
948
+ # zorder
949
+ "zorder_data": 1,
950
+ "zorder_histh": 1,
951
+ "zorder_cdf": 1,
952
+ }
953
+ )
954
+ return None
955
+
956
+ def view(self, show=True, return_fig=False):
957
+ # todo [docstring]
958
+
959
+ # SPECS
960
+ # -------------------------------------------------------------------
961
+ specs = self._get_fig_specs()
962
+
963
+ # BUILD
964
+ # -------------------------------------------------------------------
965
+ # build from specs
966
+ fig, gs = viewer.build_fig(specs=specs)
967
+
968
+ # PLOT
969
+ # -------------------------------------------------------------------
970
+ fig = self._plot(fig=fig, gs=gs, specs=specs)
971
+
972
+ # SHIP
973
+ # -------------------------------------------------------------------
974
+ # create path
975
+ file_path = "{}/{}.{}".format(
976
+ specs["folder"], specs["filename"], specs["fig_format"]
977
+ )
978
+ if return_fig:
979
+ return fig
980
+ else:
981
+ viewer.ship_fig(fig=fig, show=show, file_output=file_path, dpi=specs["dpi"])
982
+
983
+ # STATIC METHODS
984
+ # ===================================================================
985
+
986
+ @staticmethod
987
+ def plot_mean(ax, y_mu, xmin, xmax, specs):
988
+ # todo [docstring]
989
+ if not ax:
990
+ return None
991
+
992
+ ax.hlines(
993
+ y=y_mu,
994
+ xmin=xmin,
995
+ xmax=xmax,
996
+ colors=specs["color_mean"],
997
+ linestyle=specs["linestyle_mean"],
998
+ )
999
+
1000
+ if specs["color_mean_text"] is None:
1001
+ specs["color_mean_text"] = "black"
1002
+
1003
+ pad = specs["pad_mean"] * " "
1004
+ s = r"{}$\mu$ = {}".format(pad, round(y_mu, 2))
1005
+ ax.annotate(
1006
+ s,
1007
+ xy=(0, y_mu),
1008
+ xytext=(1, 3),
1009
+ textcoords="offset points",
1010
+ color=specs["color_mean_text"],
1011
+ )
1012
+ return None
1013
+
1014
+ @staticmethod
1015
+ def plot_scatter(data, ax, specs, formatter=None, x_factor=4):
1016
+ # todo [docstring]
1017
+ if not ax:
1018
+ return None
1019
+ x_values = np.arange(len(data))
1020
+ # handle scheme
1021
+ if specs["colorize_scatter"]:
1022
+ color_data = Univar.classify(
1023
+ data=data, n_classes=5, scheme=specs["scheme_cmap"]
1024
+ )
1025
+ color_cmap = viewer.get_discrete_cmap(
1026
+ n_classes=5, base_cmap_name=specs["cmap"]
1027
+ )
1028
+ # plot
1029
+ ax.scatter(
1030
+ x_values,
1031
+ data,
1032
+ marker=".",
1033
+ c=color_data,
1034
+ cmap=color_cmap,
1035
+ alpha=specs["alpha"],
1036
+ s=0.8 * viewer.MM_TO_PT,
1037
+ zorder=specs["zorder_data"],
1038
+ )
1039
+ else:
1040
+ ax.scatter(
1041
+ x_values,
1042
+ data,
1043
+ marker=".",
1044
+ color=specs["color"],
1045
+ alpha=specs["alpha"],
1046
+ s=0.8 * viewer.MM_TO_PT,
1047
+ zorder=specs["zorder_data"],
1048
+ )
1049
+
1050
+ if specs["subtitle_data"] is not None:
1051
+ ax.set_title(specs["subtitle_data"], loc="left")
1052
+
1053
+ ax.set_ylim(specs["range"])
1054
+ ax.set_xlabel(specs["xlabel"])
1055
+ ax.set_ylabel(specs["ylabel"])
1056
+ ax.grid(axis="x", visible=False)
1057
+
1058
+ if formatter is not None:
1059
+ ax.yaxis.set_major_formatter(formatter)
1060
+
1061
+ # handle x axis
1062
+ n_size = len(data)
1063
+ x_low = -x_factor * n_size
1064
+ x_hi = n_size + (x_factor * n_size)
1065
+ ax.set_xlim(x_low, x_hi)
1066
+ ax.set_xticks([])
1067
+
1068
+ # handle cbars
1069
+ # ------------ cbar plot ------------
1070
+ if specs["colorize_scatter"]:
1071
+ Univar.plot_cbar(
1072
+ data=data,
1073
+ ax=ax,
1074
+ scheme=specs["scheme_cmap"],
1075
+ cmap=specs["cmap"],
1076
+ side="right",
1077
+ )
1078
+
1079
+ # PLOT MEAN LINE
1080
+ # -------------------------------------------------------------------
1081
+ if specs["plot_mean"]:
1082
+ # data
1083
+ Univar.plot_mean(
1084
+ ax,
1085
+ y_mu=np.nanmean(data),
1086
+ xmin=x_low,
1087
+ xmax=x_hi,
1088
+ specs=specs,
1089
+ )
1090
+
1091
+ return (x_low, x_hi)
1092
+
1093
+ @staticmethod
1094
+ def plot_histh(data, ax, specs, formatter=None):
1095
+ # todo [docstring]
1096
+ # todo dry --- evaluate to remove the range_data parameter
1097
+ # ------- local functions --------
1098
+ def _get_max(std_values, data):
1099
+ data_max = np.max(data)
1100
+ closest_max_index = np.argmin(np.abs(np.array(std_values) - data_max))
1101
+ _id = closest_max_index + 1
1102
+ if _id >= len(std_values):
1103
+ _id = len(std_values) - 1
1104
+ return std_values[_id]
1105
+
1106
+ h = None
1107
+
1108
+ if not ax:
1109
+ return None
1110
+
1111
+ # Standard x-axis maximum values for hist
1112
+ sdmax1 = list(np.linspace(1, 9, 9) / 100)
1113
+ sdmax2 = list(np.linspace(10, 100, 10) / 100)
1114
+ standard_max_values = sdmax1 + sdmax2
1115
+
1116
+ # handle data range
1117
+ data_range = specs["range"]
1118
+ if data_range is None:
1119
+ data_range = [np.nanmin(data), np.nanmax(data)]
1120
+
1121
+ # handle bins
1122
+ n_bins = specs["bins"]
1123
+ if specs["bins_density"]:
1124
+ n_units = n_bins
1125
+ d_range = [np.nanmin(data), np.nanmax(data)]
1126
+ n_bins = Univar.get_nbins_density(n_units=n_units, data_range=d_range)
1127
+
1128
+ # plot horizontal hist
1129
+ h = ax.hist(
1130
+ data,
1131
+ bins=n_bins,
1132
+ color=specs["color_hist"],
1133
+ alpha=specs["alpha_hist"],
1134
+ orientation="horizontal",
1135
+ weights=np.ones(len(data)) / len(data),
1136
+ zorder=specs["zorder_histh"],
1137
+ )
1138
+ ax.xaxis.set_major_formatter(mticker.PercentFormatter(1))
1139
+ if specs["subtitle_hist"] is not None:
1140
+ ax.set_title(specs["subtitle_hist"], loc="left")
1141
+
1142
+ # Get the maximum value from the data
1143
+ xmax = _get_max(standard_max_values, h[0]) * 1.1
1144
+ ax.set_xlim(0, xmax)
1145
+ ax.set_ylim(data_range)
1146
+ ax.set_xlabel("%")
1147
+ ax.set_ylabel(specs["ylabel"])
1148
+ if formatter is not None:
1149
+ ax.yaxis.set_major_formatter(formatter)
1150
+
1151
+ # PLOT MEAN
1152
+ # --------------------------------------------------
1153
+ if specs["plot_mean"]:
1154
+ # hist
1155
+ Univar.plot_mean(
1156
+ ax,
1157
+ y_mu=np.nanmean(data),
1158
+ xmin=0,
1159
+ xmax=1,
1160
+ specs=specs,
1161
+ )
1162
+
1163
+ return h
1164
+
1165
+ @staticmethod
1166
+ def plot_cdf(cdf_df, ax, specs, formatter=None, px_field="P(X)", var_field="Data"):
1167
+ # todo [docstring]
1168
+
1169
+ if not ax:
1170
+ return None
1171
+
1172
+ # PLOT CDF
1173
+ # --------------------------------------------------
1174
+ ax.plot(
1175
+ cdf_df[px_field],
1176
+ cdf_df[var_field],
1177
+ color=specs["color_cdf"],
1178
+ alpha=specs["alpha_cdf"],
1179
+ zorder=specs["zorder_cdf"],
1180
+ )
1181
+
1182
+ if specs["subtitle_cdf"] is not None:
1183
+ ax.set_title(specs["subtitle_cdf"], loc="left")
1184
+
1185
+ ax.set_xlabel(specs["xlabel_cdf"])
1186
+ ax.set_xlim(-0.05, 1.05)
1187
+ ax.set_ylim(specs["range"])
1188
+
1189
+ if formatter is not None:
1190
+ ax.yaxis.set_major_formatter(formatter)
1191
+ ax.set_yticklabels([])
1192
+ ax.xaxis.set_major_formatter(mticker.PercentFormatter(1))
1193
+
1194
+ # PLOT MEAN
1195
+ # --------------------------------------------------
1196
+ if specs["plot_mean"]:
1197
+ Univar.plot_mean(
1198
+ ax,
1199
+ y_mu=cdf_df[var_field].mean(),
1200
+ xmin=-0.05,
1201
+ xmax=1.05,
1202
+ specs=specs,
1203
+ )
1204
+
1205
+ return None
1206
+
1207
+ @staticmethod
1208
+ def plot_stats(fig, stats_df, x=0.5, y=0.4, sep=0.2):
1209
+ # todo [docstring]
1210
+
1211
+ # Helper function to plot a single column of stats
1212
+ def _plot_column(x_position, stats_list, start_y):
1213
+ current_y = start_y
1214
+ for stat in stats_list:
1215
+ # Handle "Median" special case if it's coming from ls_stats that uses it
1216
+ display_stat = "p50" if stat == "median" else stat
1217
+
1218
+ # Ensure the stat exists in the DataFrame before attempting to access
1219
+ stat_row = stats_df.loc[stats_df["statistic"] == display_stat]
1220
+ if not stat_row.empty:
1221
+ s_value = stat_row["value"].values[0]
1222
+ s_head = stat # Use the original name for display if needed, or display_stat
1223
+
1224
+ # Determine formatting based on value magnitude
1225
+ if abs(s_value) > 1_000_000:
1226
+ s_line = "{:>10}: {:<10.2e}".format(s_head, s_value)
1227
+ elif stat == "count":
1228
+ s_line = "{:>10}: {:<10}".format(s_head, int(s_value))
1229
+ else:
1230
+ s_line = "{:>10}: {:<10.2f}".format(s_head, s_value)
1231
+
1232
+ current_y -= n_step
1233
+ plt.text(
1234
+ x=x_position,
1235
+ y=current_y,
1236
+ s=s_line,
1237
+ fontdict={"family": "monospace"},
1238
+ transform=fig.transFigure,
1239
+ )
1240
+ else:
1241
+ # Optional: Handle cases where a stat might not be found
1242
+ # print(f"Warning: Statistic '{display_stat}' not found in DataFrame.")
1243
+ pass
1244
+
1245
+ n_step = 0.03
1246
+
1247
+ # First column of statistics
1248
+ ls_stats_col1 = [
1249
+ "count",
1250
+ "sum",
1251
+ "mean",
1252
+ "median", # This will be mapped to "P50" internally by the helper
1253
+ "sd",
1254
+ "min",
1255
+ "max",
1256
+ ]
1257
+ _plot_column(x, ls_stats_col1, y - 0.01)
1258
+
1259
+ # Second column of statistics
1260
+ ls_stats_col2 = [
1261
+ "p05",
1262
+ "p10",
1263
+ "p25",
1264
+ "p50",
1265
+ "p75",
1266
+ "p90",
1267
+ "p95",
1268
+ ]
1269
+ _plot_column(x + sep, ls_stats_col2, y - 0.01)
1270
+
1271
+ return fig
1272
+
1273
+ @staticmethod
1274
+ def plot_cbar(
1275
+ data, ax, scheme, cmap="viridis", n_classes=5, side="right", width_factor=16
1276
+ ):
1277
+ # todo [docstring]
1278
+
1279
+ # handle bins
1280
+ bins = Univar.get_bins(data=data, n_bins=n_classes, scheme=scheme)
1281
+
1282
+ # handle egde cases
1283
+ n_bins = len(bins)
1284
+ if n_bins <= n_classes:
1285
+ n_classes = n_bins - 1
1286
+
1287
+ # handle colors
1288
+ custom_cmap = viewer.get_discrete_cmap(n_classes=n_classes, base_cmap_name=cmap)
1289
+ colors = []
1290
+ for i in range(n_classes):
1291
+ colors.append(custom_cmap.colors[i])
1292
+
1293
+ # handle position
1294
+ xlims = ax.get_xlim()
1295
+ bar_w = abs(xlims[1] - xlims[0]) / width_factor
1296
+ if side == "left":
1297
+ vrange = [xlims[0], xlims[0] + bar_w]
1298
+ elif side == "right":
1299
+ vrange = [xlims[1] - bar_w, xlims[1]]
1300
+ else:
1301
+ vrange = [xlims[0], xlims[0] + bar_w]
1302
+
1303
+ # plotting loop
1304
+ n_classes = len(colors)
1305
+ for i in range(n_classes):
1306
+ lower_bound = bins[i]
1307
+ upper_bound = bins[i + 1]
1308
+ color = colors[i]
1309
+ ax.fill_between(
1310
+ vrange, lower_bound, upper_bound, color=color, edgecolor="none"
1311
+ )
1312
+
1313
+ return None
1314
+
1315
+ @staticmethod
1316
+ def test_distribution(test_name, stat, p, clevel=0.95, distr="normal"):
1317
+ """
1318
+ Util function for statistical testing
1319
+
1320
+ :param test_name: name of test
1321
+ :type test_name: str
1322
+ :param stat: statistic
1323
+ :type stat: float
1324
+ :param p: p-value
1325
+ :type p: float
1326
+ :param clevel: confidence level
1327
+ :type clevel: float
1328
+ :param distr: name of distribution
1329
+ :type distr: str
1330
+ :return: summary of test
1331
+ :rtype: dict
1332
+ """
1333
+ # built output dict
1334
+ dct_out = {
1335
+ "Test": test_name,
1336
+ "Statistic": stat,
1337
+ "p-value": p,
1338
+ "Confidence": clevel,
1339
+ }
1340
+
1341
+ if p > (1 - clevel):
1342
+ dct_out["is_{}".format(distr.lower())] = True
1343
+ else:
1344
+ dct_out["is_{}".format(distr.lower())] = False
1345
+
1346
+ return dct_out
1347
+
1348
+ @staticmethod
1349
+ def test_normality_ks(data, clevel=0.95):
1350
+ """
1351
+ Test for normality using the Kolmogorov-Smirnov test
1352
+
1353
+ Kolmogorov-Smirnov Test: This test compares the observed distribution with
1354
+ the expected normal distribution using a test statistic and a p-value.
1355
+ A p-value less than 0.05 indicates that the null hypothesis should be rejected.
1356
+
1357
+ :param data: vector of data without nan values
1358
+ :type data: :class:`numpy.ndarray`
1359
+ :return: test result dictionary. Keys: Statistic, p-value and Is normal
1360
+ :rtype: dct
1361
+ """
1362
+ from scipy.stats import kstest
1363
+
1364
+ vct_norm = (data - np.mean(data)) / np.std(data)
1365
+
1366
+ # test
1367
+ result = kstest(vct_norm, "norm")
1368
+ return Univar.test_distribution(
1369
+ test_name="Kolmogorov-Smirnov",
1370
+ stat=result.statistic,
1371
+ p=result.pvalue,
1372
+ clevel=clevel,
1373
+ distr="normal",
1374
+ )
1375
+
1376
+ @staticmethod
1377
+ def test_normality_sw(data, clevel=0.95):
1378
+ """
1379
+ Test for normality using the Shapiro-Wilk test.
1380
+
1381
+ :param data: vector of data without nan values
1382
+ :type data: :class:`numpy.ndarray`
1383
+ :return: test result dictionary. Keys: Statistic, p-value and Is normal
1384
+ :rtype: dct
1385
+ """
1386
+ from scipy.stats import shapiro
1387
+
1388
+ # test
1389
+ stat, p = shapiro(data)
1390
+
1391
+ return Univar.test_distribution(
1392
+ test_name="Shapiro-Wilk", stat=stat, p=p, clevel=clevel, distr="normal"
1393
+ )
1394
+
1395
+ @staticmethod
1396
+ def test_normality_dp(data, clevel=0.95):
1397
+ """
1398
+ Test for normality using the D'Agostino-Pearson test.
1399
+
1400
+ :param data: vector of data without nan values
1401
+ :type data: :class:`numpy.ndarray`
1402
+ :return: test result dictionary. Keys: Statistic, p-value and Is normal
1403
+ :rtype: dct
1404
+ """
1405
+ from scipy.stats import normaltest
1406
+
1407
+ # test
1408
+ stat, p = normaltest(data)
1409
+
1410
+ return Univar.test_distribution(
1411
+ test_name="D'Agostino-Pearson",
1412
+ stat=stat,
1413
+ p=p,
1414
+ clevel=clevel,
1415
+ distr="normal",
1416
+ )
1417
+
1418
+ @staticmethod
1419
+ def get_tx(fx):
1420
+ """
1421
+ Simple function for computing the Return Period from the CDF
1422
+
1423
+ :param fx: CDF (FX)
1424
+ :type fx: float | :class:`numpy.ndarray`
1425
+ :return: Return Period
1426
+ :rtype: float | :class:`numpy.ndarray`
1427
+ """
1428
+ return 1 / (1 - fx)
1429
+
1430
+ @staticmethod
1431
+ def gumbel_fx(x, a, b):
1432
+ """
1433
+ Gumbel probability distribution F(X)
1434
+
1435
+ :param x: function inputs
1436
+ :type x: float | :class:`numpy.ndarray`
1437
+ :param a: distribution parameter a
1438
+ :type a: float
1439
+ :param b: distribution parameter b
1440
+ :type b: float
1441
+ :return: function output
1442
+ :rtype: float | :class:`numpy.ndarray`
1443
+ """
1444
+ aux_1 = (x - a) / b
1445
+ aux_2 = np.exp(-aux_1)
1446
+ g_fx = np.exp(-aux_2)
1447
+ return g_fx
1448
+
1449
+ @staticmethod
1450
+ def gumbel_tx(x, a, b):
1451
+ """
1452
+ Gumbel return period distribution T(X)
1453
+
1454
+ :param x: function inputs
1455
+ :type x: float | :class:`numpy.ndarray`
1456
+ :param a: distribution parameter a
1457
+ :type a: float
1458
+ :param b: distribution parameter b
1459
+ :type b: float
1460
+ :return: function output
1461
+ :rtype: float | :class:`numpy.ndarray`
1462
+ """
1463
+ g_fx = Univar.gumbel_fx(x, a, b)
1464
+ return Univar.get_tx(fx=g_fx)
1465
+
1466
+ @staticmethod
1467
+ def gumbel_freqfactor(tx=2):
1468
+ """
1469
+ Gumbel Frequency Factor K(T)
1470
+
1471
+ :param tx: return period T
1472
+ :type tx: float | :class:`numpy.ndarray`
1473
+ :return: function output
1474
+ :rtype: float | :class:`numpy.ndarray`
1475
+ """
1476
+ aux1 = tx / (tx - 1)
1477
+ aux2 = np.log(aux1)
1478
+ aux3 = np.log(aux2)
1479
+ aux4 = 0.5772 + aux3
1480
+ aux5 = -np.sqrt(6) * aux4 / np.pi
1481
+ return aux5
1482
+
1483
+ @staticmethod
1484
+ def gumbel_se(std_sample, n_sample, tx):
1485
+ """
1486
+ Gumbel Standard Error for the MM fitted Gumbel function
1487
+
1488
+ :param std_sample: sample standard deviation
1489
+ :type std_sample: float
1490
+ :param n_sample: sample size
1491
+ :type n_sample: int
1492
+ :param tx: return period T
1493
+ :type tx: float | :class:`numpy.ndarray`
1494
+ :return: function output
1495
+ :rtype: float | :class:`numpy.ndarray`
1496
+ """
1497
+ aux1 = std_sample / np.sqrt(n_sample)
1498
+ k = Univar.gumbel_freqfactor(tx=tx)
1499
+ aux2 = 1 + (1.14 * k) + (1.1 * np.square(k))
1500
+ aux3 = np.sqrt(aux2)
1501
+ return aux1 * aux3
1502
+
1503
+ @staticmethod
1504
+ def empirical_px(ranks):
1505
+ """
1506
+ Get the empirical exceedance probability P(X)
1507
+
1508
+ :param ranks: vector of ranks
1509
+ :type ranks: class:`numpy.array`
1510
+ :return: empirical exceedance probability P(X)
1511
+ :rtype: class:`numpy.array`
1512
+ """
1513
+ return ranks / len(ranks)
1514
+
1515
+ @staticmethod
1516
+ def weibull_px(ranks):
1517
+ """
1518
+ Get the Weibull exceedance probability P(X)
1519
+
1520
+ :param ranks: vector of ranks
1521
+ :type ranks: class:`numpy.array`
1522
+ :return: Weibull exceedance probability P(X)
1523
+ :rtype: class:`numpy.array`
1524
+ """
1525
+ return ranks / (len(ranks) + 1)
1526
+
1527
+ @staticmethod
1528
+ def gringorten_px(ranks):
1529
+ """
1530
+ Get the Gringorten exceedance probability P(X)
1531
+
1532
+ :param ranks: vector of ranks
1533
+ :type ranks: class:`numpy.array`
1534
+ :return: Gringorten exceedance probability P(X)
1535
+ :rtype: class:`numpy.array`
1536
+ """
1537
+ return (ranks - 0.44) / (len(ranks) + 0.12)
1538
+
1539
+ @staticmethod
1540
+ def sample_gamma(size, shape=3, scale=1, shape_mode=None, n_min=None, n_max=None):
1541
+ """
1542
+ Sample Gamma distribution
1543
+
1544
+ :param size: Sample size
1545
+ :type size: int
1546
+ :param shape: Shape parameter
1547
+ :type shape: float
1548
+ :param scale: Scale parameter
1549
+ :type scale: float
1550
+ :param shape_mode: Shape mode (overwrites shape value). See dict for options
1551
+ :type shape_mode: str or None
1552
+ :param n_min: Normalization lower value
1553
+ :type n_min: float or None
1554
+ :param n_max: Normalization upper value
1555
+ :type n_max: float or None
1556
+ :return: Sampled array
1557
+ :rtype: :class:`numpy.ndarray`
1558
+ """
1559
+ from plans.geo import normalize
1560
+
1561
+ # set gamma shapes
1562
+ dc_gamma_shapes = {
1563
+ "decay": 1,
1564
+ "asymmetrical_extra": 2,
1565
+ "asymmetrical": 3,
1566
+ "symmetrical": 20,
1567
+ }
1568
+ # handle shapemode
1569
+ if shape_mode is not None:
1570
+ shape = dc_gamma_shapes[shape_mode]
1571
+
1572
+ # sample
1573
+ sample = np.random.gamma(shape=shape, scale=scale, size=size)
1574
+
1575
+ # handle normalization
1576
+ if n_min is None:
1577
+ n_min = np.min(sample)
1578
+ if n_max is None:
1579
+ n_max = np.max(sample)
1580
+
1581
+ # normalize
1582
+ sample_norm = normalize(array=sample, min_value=n_min, max_value=n_max)
1583
+
1584
+ return sample_norm
1585
+
1586
+ @staticmethod
1587
+ def get_nbins_fd(data):
1588
+ """
1589
+ This function computes the number of bins for histograms using the Freedman-Diaconis rule, which takes into
1590
+ account the interquartile range (IQR) of the sample, in addition to its range.
1591
+
1592
+ :param data: vector of data without nan values
1593
+ :type data: :class:`numpy.ndarray`
1594
+ :return: number of bins for histogram using the Freedman-Diaconis rule
1595
+ :rtype: int
1596
+ """
1597
+ iqr = np.subtract(*np.percentile(data, [75, 25]))
1598
+ binsize = 2 * iqr * len(data) ** (-1 / 3)
1599
+ # hack for non infinite values
1600
+ if binsize == 0:
1601
+ binsize = 100
1602
+ return int(np.ceil((max(data) - min(data)) / binsize))
1603
+
1604
+ @staticmethod
1605
+ def get_nbins_sturges(data):
1606
+ """
1607
+ This function computes the number of bins using the Sturges rule,
1608
+ which assumes that the data follows a normal distribution and computes
1609
+ the number of bins based on its data runsize.
1610
+
1611
+ :param data: vector of data without nan values
1612
+ :type data: :class:`numpy.ndarray`
1613
+ :return: number of bins using the Sturges rule
1614
+ :rtype: int
1615
+ """
1616
+ return int(np.ceil(np.log2(len(data)) + 1))
1617
+
1618
+ @staticmethod
1619
+ def get_nbins_scott(data):
1620
+ """
1621
+ This function computes the number of bins using the Scott rule,
1622
+ which is similar to the Freedman-Diaconis rule, but uses the standard deviation
1623
+ of the data to compute the bin runsize.
1624
+
1625
+ :param data: vector of data without nan values
1626
+ :type data: :class:`numpy.ndarray`
1627
+ :return: number of bins using the Scott rule
1628
+ :rtype: int
1629
+ """
1630
+ binsize = 3.5 * np.nanstd(data) * len(data) ** (-1 / 3)
1631
+ return int(np.ceil((max(data) - min(data)) / binsize))
1632
+
1633
+ @staticmethod
1634
+ def get_nbins_density(n_units, data_range):
1635
+ # todo docstring
1636
+ n_range = data_range[1] - data_range[0]
1637
+ bins = int(n_range / n_units)
1638
+ if bins < 1:
1639
+ return 1
1640
+ else:
1641
+ return bins
1642
+
1643
+ @staticmethod
1644
+ def get_nbins_by_rule(data, rule=None):
1645
+ """
1646
+ Util function for rule-based nbins computation
1647
+
1648
+ :param data: vector of data without nan values
1649
+ :type data: :class:`numpy.ndarray`
1650
+ :param rule: rule code (sturges, fd, scott)
1651
+ :type rule: str
1652
+ :return: number of bins for histogram
1653
+ :rtype: int
1654
+ """
1655
+ bins = 10
1656
+ if rule is None:
1657
+ pass
1658
+ else:
1659
+ if rule.lower() == "sturges":
1660
+ bins = Univar.get_nbins_sturges()
1661
+ elif rule.lower() == "fd":
1662
+ bins = Univar.get_nbins_fd()
1663
+ elif rule.lower() == "scott":
1664
+ bins = Univar.get_nbins_scott()
1665
+ else:
1666
+ pass
1667
+ return bins
1668
+
1669
+ @staticmethod
1670
+ def get_histogram(data, bins=100, rule=None):
1671
+ """
1672
+ Compute the histogram of the sample
1673
+
1674
+ :param data: vector of data without nan values
1675
+ :type data: :class:`numpy.ndarray`
1676
+ :param bins: number of bins
1677
+ :type bins: int
1678
+ :param rule: rule to define the number of bins. If 'none', uses bins parameters.
1679
+ :type rule: str
1680
+ :return: dataframe of histogram
1681
+ :rtype: :class:`pandas.DataFrame`
1682
+ """
1683
+
1684
+ if bins is None:
1685
+ if rule is None:
1686
+ pass
1687
+ else:
1688
+ bins = Univar.get_nbins_by_rule(rule=rule)
1689
+ # compute histogram
1690
+ vct_hist, vct_bins = np.histogram(data, bins=bins)
1691
+ # get dataframe
1692
+ df_hist = pd.DataFrame({"Upper value": vct_bins[1:], "Count": vct_hist})
1693
+ return df_hist
1694
+
1695
+ @staticmethod
1696
+ def qqplot(data):
1697
+ """
1698
+ Calculate the QQ-plot of data against normal distribution
1699
+
1700
+ :param data: vector of data without nan values
1701
+ :type data: :class:`numpy.ndarray`
1702
+ :return: dataframe of QQ plot
1703
+ :rtype: :class:`pandas.DataFrame`
1704
+ """
1705
+ from scipy.stats import norm
1706
+
1707
+ # process quantiles
1708
+ _df = pd.DataFrame(
1709
+ {
1710
+ "Data": np.sort(data),
1711
+ "E-Quantiles": (np.arange(1, len(data) + 1) - 0.5) / len(data),
1712
+ }
1713
+ )
1714
+ # get theoretical
1715
+ _df["T-Quantiles"] = norm.ppf(_df["E-Quantiles"])
1716
+ return _df
1717
+
1718
+ @staticmethod
1719
+ def trace_variance(data):
1720
+ """
1721
+ Trace the mean variance from sample
1722
+
1723
+ :param data: vector of data without nan values
1724
+ :type data: :class:`numpy.ndarray`
1725
+ :return: dataframe of accumulated variance
1726
+ :rtype: :class:`pandas.DataFrame`
1727
+ """
1728
+ vct_variance_mean = np.zeros(len(data))
1729
+ for i in range(2, len(data)):
1730
+ vct_variance_mean[i] = np.var(data[:i])
1731
+
1732
+ # process quantiles
1733
+ _df = pd.DataFrame(
1734
+ {
1735
+ "Data": data,
1736
+ "Variance": vct_variance_mean,
1737
+ }
1738
+ )
1739
+
1740
+ return _df
1741
+
1742
+ @staticmethod
1743
+ def get_bins(data, n_bins=5, scheme="equal"):
1744
+ # todo [docstring]
1745
+ dc = {
1746
+ "linear": Univar.bins_equal,
1747
+ "equal": Univar.bins_equal,
1748
+ "quantiles": Univar.bins_quantiles,
1749
+ "quantile": Univar.bins_quantiles,
1750
+ }
1751
+ v = dc[scheme](data=data, n_bins=n_bins)
1752
+ return v
1753
+
1754
+ @staticmethod
1755
+ def bins_equal(data, n_bins):
1756
+ """
1757
+ Calculates equally spaced (linear) bins for a 1D NumPy array.
1758
+
1759
+ :param data: The 1D NumPy array to analyze.
1760
+ :type data: numpy.ndarray
1761
+ :param n_bins: The desired number of bins/intervals.
1762
+ :type n_bins: int
1763
+ :returns: A NumPy array containing the linear bin boundary values.
1764
+ :rtype: numpy.ndarray
1765
+ """
1766
+ min_val = np.min(data)
1767
+ max_val = np.max(data)
1768
+
1769
+ # np.linspace creates 'n_classes + 1' points to define 'n_classes' intervals
1770
+ linear_bins = np.linspace(min_val, max_val, n_bins + 1)
1771
+ return linear_bins
1772
+
1773
+ @staticmethod
1774
+ def bins_quantiles(data, n_bins=5):
1775
+ """
1776
+ Calculates the numerical quantile boundaries (bins) for a 1D NumPy array.
1777
+
1778
+ :param data: The 1D NumPy array to analyze.
1779
+ :type data: numpy.ndarray
1780
+ :param n_bins: The number of quantile classes.
1781
+ :type n_bins: int
1782
+ :returns: A NumPy array containing the quantile boundary values.
1783
+ :rtype: numpy.ndarray
1784
+ """
1785
+ # pd.qcut with retbins=True returns (labels, bins)
1786
+ _, bins = pd.qcut(
1787
+ data,
1788
+ q=n_bins,
1789
+ retbins=True,
1790
+ duplicates="drop", # Essential for handling non-unique quantiles
1791
+ )
1792
+ return bins
1793
+
1794
+ @staticmethod
1795
+ def classify(data, bins=None, n_classes=5, scheme="equal"):
1796
+ """
1797
+ Classifies a 1D NumPy array into 0-indexed bins based on provided bin boundaries.
1798
+
1799
+ Values `x` are assigned to class `i` if `bins[i] < x <= bins[i+1]`.
1800
+ Values less than or equal to `bins[0]` are assigned to class 0.
1801
+ Values greater than `bins[-1]` are assigned to the last class (`len(bins) - 2`).
1802
+
1803
+ :param data: The 1D NumPy array to classify.
1804
+ :type data: numpy.ndarray
1805
+ :param bins: A 1D NumPy array specifying the sorted bin edges.
1806
+ E.g., `[min_val, boundary1, boundary2, ..., max_val]`.
1807
+ Must contain at least two elements.
1808
+ :type bins: numpy.ndarray
1809
+ :returns: A 1D NumPy array of integers representing the 0-indexed class for
1810
+ each element in `data_array`.
1811
+ :rtype: numpy.ndarray
1812
+ """
1813
+ # handle bins
1814
+ if bins is None:
1815
+ bins = Univar.get_bins(data=data, n_bins=n_classes, scheme=scheme)
1816
+
1817
+ # Number of intervals (classes) will be one less than the number of bins
1818
+ num_classes = len(bins) - 1
1819
+
1820
+ # np.searchsorted(a, v, side='right') returns index 'i' such that all v in a[0:i] <= v
1821
+ # This means `bins[i-1] < x <= bins[i]`.
1822
+ # For example, if bins = [0, 10, 20]:
1823
+ # x=5 => searchsorted returns 1 (meaning it's after bins[0])
1824
+ # x=10 => searchsorted returns 1 (meaning it's after bins[0])
1825
+ # x=15 => searchsorted returns 2 (meaning it's after bins[1])
1826
+ # x=20 => searchsorted returns 2 (meaning it's after bins[1])
1827
+ # Subtracting 1 yields the 0-indexed class for [bins[i-1], bins[i]] like intervals.
1828
+
1829
+ # `classes` will be 0 for x <= bins[0], 1 for x in (bins[0], bins[1]], etc.
1830
+ # For x > bins[-1], it will be `num_classes` (len(bins) - 1).
1831
+ classes = np.searchsorted(bins, data, side="right") - 1
1832
+
1833
+ # Clamp the results to ensure they fall within the valid 0 to num_classes-1 range.
1834
+ # This handles values outside the `bins` range by assigning them to the
1835
+ # first (0) or last (num_classes-1) available class.
1836
+ classified_array = np.clip(classes, 0, num_classes - 1)
1837
+
1838
+ return classified_array
1839
+
1840
+ @staticmethod
1841
+ def quantiles_classify(data, n_classes=5):
1842
+ """
1843
+ Classifies a 1D NumPy array into quantile classes.
1844
+
1845
+ :param data: The 1D NumPy array to classify.
1846
+ :type data: numpy.ndarray
1847
+ :param n_classes: The number of quantile classes.
1848
+ :type n_classes: int
1849
+ :returns: A 1D NumPy array containing the quantile class (0 to n_classes-1) for each element.
1850
+ :rtype: numpy.ndarray
1851
+ """
1852
+
1853
+ bins = Univar.get_bins(data=data, n_bins=n_classes, scheme="quantiles")
1854
+
1855
+ values = Univar.classify(data, bins=bins)
1856
+
1857
+ """
1858
+ # Use pandas qcut, which is efficient and handles duplicates well
1859
+ # labels=False ensures integer class labels (0 to n_classes-1)
1860
+ # duplicates='drop' handles cases where fewer unique quantiles than n_classes exist
1861
+ quantile_classes = pd.qcut(
1862
+ data,
1863
+ q=n_classes,
1864
+ labels=False,
1865
+ duplicates='drop'
1866
+ )#.to_numpy() # Convert the pandas Series back to a NumPy array
1867
+ """
1868
+
1869
+ return quantile_classes
1870
+
1871
+
1872
+ class GeoUnivar(Univar):
1873
+
1874
+ def __init__(self, name="MyGeoUnivar", alias="GV0"):
1875
+ # call super
1876
+ super().__init__(name=name, alias=alias)
1877
+ self.layer_name = None
1878
+
1879
+ def _set_fields(self):
1880
+ # ------------ call super ----------- #
1881
+ super()._set_fields()
1882
+ # Attribute fields
1883
+ self.field_geometry = "geometry"
1884
+ # ... continues in downstream objects ... #
1885
+
1886
+ def _set_view_specs(self):
1887
+ super()._set_view_specs()
1888
+ self.view_specs.update(
1889
+ {
1890
+ "subtitle_map": "Map",
1891
+ "ax_map": 0,
1892
+ "ax_data": 1,
1893
+ "ax_histh": 2,
1894
+ "ax_cdf": 3,
1895
+ "x_stats": 0.55,
1896
+ "zorder_map": 0,
1897
+ "empty_map": False,
1898
+ "grid_map": False,
1899
+ "marker_map": "o",
1900
+ "markersize_map": 10,
1901
+ "edgecolor_map": "none",
1902
+ "colorize_map": True,
1903
+ "cbar_scatter": True,
1904
+ "cbar_histh": True,
1905
+ }
1906
+ )
1907
+ return None
1908
+
1909
+ def load_data(self, file_data, layer_name):
1910
+ """
1911
+ Load data from file. Expected to overwrite superior methods.
1912
+
1913
+ :param file_data: file path to data.
1914
+ :type file_data: str
1915
+ :param layer_name:
1916
+ :type layer_name: str
1917
+ :return: None
1918
+ :rtype: None
1919
+ """
1920
+ # include geopandas
1921
+ import geopandas as gpd
1922
+
1923
+ # -------------- overwrite relative path inputs -------------- #
1924
+ self.file_data = os.path.abspath(file_data)
1925
+ self.layer_name = layer_name
1926
+
1927
+ # -------------- implement loading logic -------------- #
1928
+ default_columns = {
1929
+ #'DateTime': 'datetime64[1s]',
1930
+ self.varfield: float,
1931
+ }
1932
+ ls_cols = [self.varfield, self.field_geometry]
1933
+ # -------------- call loading function -------------- #
1934
+
1935
+ # handle data formats
1936
+ self.data = gpd.read_file(self.file_data, layer=layer_name, columns=ls_cols)
1937
+
1938
+ # -------------- post-loading logic -------------- #
1939
+ self.data.dropna(inplace=True)
1940
+
1941
+ # -------------- update other mutables -------------- #
1942
+ self.update()
1943
+
1944
+ # ... continues in downstream objects ... #
1945
+
1946
+ return None
1947
+
1948
+ # todo [dev] -- export method
1949
+
1950
+ def _build_axes(self, fig, gs, specs):
1951
+ # todo [docstring]
1952
+ # ------------ setup axes ------------
1953
+ if specs["mode"] == "full":
1954
+ fig.add_subplot(gs[2:15, 1:14])
1955
+ fig.add_subplot(gs[2:10, 16:21])
1956
+ fig.add_subplot(gs[2:10, 23:28])
1957
+ fig.add_subplot(gs[2:10, 29:])
1958
+ elif specs["mode"] == "mini":
1959
+ fig.add_subplot(gs[2:10, 2:10])
1960
+ fig.add_subplot(gs[2:10, 16:21])
1961
+ fig.add_subplot(gs[2:10, 23:28])
1962
+ fig.add_subplot(gs[2:10, 29:])
1963
+ return fig
1964
+
1965
+ def _get_fig_specs(self):
1966
+ # todo [docstring]
1967
+ # handle specs
1968
+ specs = self.view_specs.copy()
1969
+
1970
+ # handle mode
1971
+ mode = specs["mode"]
1972
+ if mode == "full":
1973
+ specs_aux = {
1974
+ "ncols": 34,
1975
+ "nrows": 16,
1976
+ "width": viewer.FIG_SIZES["L"]["w"],
1977
+ "height": viewer.FIG_SIZES["L"]["h"],
1978
+ }
1979
+ else:
1980
+ specs_aux = {
1981
+ "ncols": 34,
1982
+ "nrows": 12,
1983
+ "width": viewer.FIG_SIZES["L2"]["w"],
1984
+ "height": viewer.FIG_SIZES["L2"]["h"],
1985
+ }
1986
+ # update sizes
1987
+ specs.update(specs_aux)
1988
+
1989
+ # update gridspecs
1990
+ specs.update(viewer.GRID_SPECS)
1991
+
1992
+ return specs
1993
+
1994
+ def _plot(self, fig, gs, specs):
1995
+ # todo [docstring]
1996
+ fig = super()._plot(fig=fig, gs=gs, specs=specs)
1997
+
1998
+ all_axes = fig.get_axes()
1999
+
2000
+ # ------------ map plot ------------
2001
+ ax_map = all_axes[specs["ax_map"]]
2002
+
2003
+ GeoUnivar.plot_map(data=self.data, ax=ax_map, column=self.varfield, specs=specs)
2004
+
2005
+ # ------------ scatter cbar plot ------------
2006
+ # handle plotting conflict
2007
+ if specs["colorize_scatter"]:
2008
+ specs["cbar_scatter"] = False
2009
+ if specs["cbar_scatter"]:
2010
+ Univar.plot_cbar(
2011
+ data=self.data[specs["yvar_field"]].values,
2012
+ ax=all_axes[specs["ax_data"]],
2013
+ scheme=specs["scheme_cmap"],
2014
+ cmap=specs["cmap"],
2015
+ )
2016
+
2017
+ # ------------ hist cbar plot ------------
2018
+ if specs["cbar_histh"]:
2019
+ Univar.plot_cbar(
2020
+ data=self.data[specs["yvar_field"]].values,
2021
+ ax=all_axes[specs["ax_histh"]],
2022
+ scheme=specs["scheme_cmap"],
2023
+ cmap=specs["cmap"],
2024
+ )
2025
+
2026
+ return fig
2027
+
2028
+ @staticmethod
2029
+ def plot_map(data, ax, column, specs, legend=False):
2030
+ if not specs["empty_map"]:
2031
+
2032
+ # handle scheme
2033
+ if specs["colorize_map"]:
2034
+ color_data = Univar.classify(
2035
+ data=data[column].values, n_classes=5, scheme=specs["scheme_cmap"]
2036
+ )
2037
+ color_cmap = viewer.get_discrete_cmap(
2038
+ n_classes=5, base_cmap_name=specs["cmap"]
2039
+ )
2040
+ # set new column
2041
+ data[f"{column}_plot"] = color_data
2042
+
2043
+ data.plot(
2044
+ ax=ax,
2045
+ column=f"{column}_plot",
2046
+ zorder=specs["zorder_map"],
2047
+ cmap=color_cmap,
2048
+ legend=legend,
2049
+ marker=specs["marker_map"], # Use a circle marker for dots
2050
+ markersize=specs[
2051
+ "markersize_map"
2052
+ ], # Set the size of the points (adjust as needed)
2053
+ edgecolor=specs["edgecolor_map"],
2054
+ )
2055
+ else:
2056
+ data.plot(
2057
+ ax=ax,
2058
+ column=column,
2059
+ zorder=specs["zorder_map"],
2060
+ cmap=specs["cmap"],
2061
+ legend=legend,
2062
+ marker=specs["marker_map"], # Use a circle marker for dots
2063
+ markersize=specs[
2064
+ "markersize_map"
2065
+ ], # Set the size of the points (adjust as needed)
2066
+ edgecolor=specs["edgecolor_map"],
2067
+ )
2068
+ # Set the Y-axis tick labels orientation
2069
+ ax.tick_params(
2070
+ axis="y", # Apply to the y-axis
2071
+ labelrotation=90, # Rotate labels by 90 degrees (vertical)
2072
+ )
2073
+
2074
+ for label in ax.get_yticklabels():
2075
+ # Option A: Often looks good for vertical Y-labels on the left
2076
+ label.set_horizontalalignment("right")
2077
+ label.set_verticalalignment("center")
2078
+ if specs["subtitle_map"] is not None:
2079
+ ax.set_title(specs["subtitle_map"], loc="left")
2080
+ ax.grid(visible=specs["grid_map"])
2081
+
2082
+ return None
2083
+
2084
+
2085
+ # todo [upgrade] make it a child of Dataset()
2086
+ class Bivar:
2087
+ # todo [docstring]
2088
+ """
2089
+ The Bivariate analyst object
2090
+
2091
+ # [major docstring]
2092
+
2093
+ """
2094
+
2095
+ def __init__(self, df_data, x_name="x", y_name="y", name="myvars"):
2096
+
2097
+ # set inputs attributes
2098
+ self.xname = x_name
2099
+ self.yname = y_name
2100
+ self.name = name
2101
+
2102
+ # set sorted data and reset index
2103
+ df_data = df_data[[x_name, y_name]]
2104
+ df_data = df_data.dropna()
2105
+ self.data = df_data.sort_values(by=self.xname).reset_index(drop=True)
2106
+
2107
+ # models setup
2108
+ self.models = {
2109
+ "Linear": {
2110
+ "Function": linear,
2111
+ "Formula": "f(x) = c0 + c1 * x",
2112
+ "Setup": pd.DataFrame(
2113
+ {"Parameters": ["c_0", "c_1"], "Mean": [0, 1], "SD": [0.1, 0.1]}
2114
+ ),
2115
+ "Data": None,
2116
+ "RMSE": None,
2117
+ },
2118
+ "Power": {
2119
+ "Function": power,
2120
+ "Formula": "f(x) = c2 * ((x + c0)^c1)",
2121
+ "Setup": pd.DataFrame(
2122
+ {
2123
+ "Parameters": ["c_0", "c_1", "c_2"],
2124
+ "Mean": [0, 1, 1],
2125
+ "SD": [0.1, 0.1, 0.1],
2126
+ }
2127
+ ),
2128
+ "Data": None,
2129
+ "RMSE": None,
2130
+ },
2131
+ "Power_zero": {
2132
+ "Function": power_zero,
2133
+ "Formula": "f(x) = c1 * (x^c0)",
2134
+ "Setup": pd.DataFrame(
2135
+ {"Parameters": ["c_0", "c_1"], "Mean": [1, 1], "SD": [0.1, 0.1]}
2136
+ ),
2137
+ "Data": None,
2138
+ "RMSE": None,
2139
+ },
2140
+ }
2141
+
2142
+ """
2143
+ "Gumbel": {
2144
+ "Function": gumbel,
2145
+ "Formula": "f(x) = exp(-exp(-((x-a)/b)))",
2146
+ "Setup": pd.DataFrame(
2147
+ {"Parameters": ["a", "b"], "Mean": [1, 1], "SD": [0.1, 0.1]}
2148
+ ),
2149
+ "Data": None,
2150
+ "RMSE": None,
2151
+ },
2152
+ """
2153
+
2154
+ def fit(self, model_type="Linear"):
2155
+ """
2156
+ Fit model to bivariate object
2157
+
2158
+ :param model_type: model type. options: Linear, Power, Power_zero
2159
+ :type model_type: str
2160
+ :return: None
2161
+ :rtype: None
2162
+ """
2163
+ from scipy.optimize import curve_fit
2164
+
2165
+ popt, pcov = curve_fit(
2166
+ f=self.models[model_type]["Function"],
2167
+ xdata=self.data[self.xname],
2168
+ ydata=self.data[self.yname],
2169
+ p0=self.models[model_type]["Setup"]["Mean"].values,
2170
+ )
2171
+ pstd = np.sqrt(np.diag(pcov))
2172
+ # update model
2173
+ self.update_model(params_mean=popt, params_sd=pstd, model_type=model_type)
2174
+ return None
2175
+
2176
+ def update_model(self, params_mean, params_sd=None, model_type="Linear"):
2177
+ """Update model based on parameters
2178
+
2179
+ :param params_mean: list of mean of parameters
2180
+ :type params_mean: list
2181
+ :param params_sd: list of standard deviation of parameters
2182
+ :type params_sd: list
2183
+ :param model_type: model type. options: Linear, Power, Power_zero
2184
+ :type model_type: str
2185
+ :return: None
2186
+ :rtype: None
2187
+ """
2188
+ # Setup
2189
+ self.models[model_type]["Setup"]["Mean"] = params_mean
2190
+ if params_sd is None:
2191
+ pass
2192
+ else:
2193
+ self.models[model_type]["Setup"]["SD"] = params_sd
2194
+ # update sample
2195
+ self.updata_model_data(model_type=model_type)
2196
+ # update model metrics
2197
+ vct_e = self.models[model_type]["Data"]["e_Mean"].values
2198
+ self.models[model_type]["RMSE"] = np.square(np.mean(np.square(vct_e)))
2199
+ return None
2200
+
2201
+ def updata_model_data(self, model_type="Linear"):
2202
+ """Update only model data output
2203
+
2204
+ :param model_type: model type. options: Linear, Power, Power_zero
2205
+ :type model_type: str
2206
+ :return: None
2207
+ :rtype: None
2208
+ """
2209
+ popt = self.models[model_type]["Setup"]["Mean"].values
2210
+ _df = self.data.copy()
2211
+ s_ymodel = "{}_Mean".format(self.yname)
2212
+ # compute model on sample:
2213
+ _df[s_ymodel] = self.models[model_type]["Function"](
2214
+ self.data[self.xname], *popt
2215
+ )
2216
+ # compute error
2217
+ _df["e_Mean".format(model_type)] = _df[s_ymodel] - _df[self.yname]
2218
+ # set attribute
2219
+ self.models[model_type]["Data"] = _df.copy()
2220
+ del _df
2221
+ return None
2222
+
2223
+ def view(
2224
+ self,
2225
+ show=True,
2226
+ folder="C:/sample",
2227
+ filename="view",
2228
+ specs=None,
2229
+ fig_format="jpg",
2230
+ dpi=300,
2231
+ ):
2232
+ """Plot basic view of Bivar base_object
2233
+
2234
+ :param show: Boolean to show instead of saving
2235
+ :type show: bool
2236
+ :param folder: output folder
2237
+ :type folder: str
2238
+ :param filename: image file name
2239
+ :type filename: str
2240
+ :param specs: specification dictionary
2241
+ :type specs: dict
2242
+ :param dpi: image resolution (default = 300)
2243
+ :type dpi: int
2244
+ :param fig_format: image fig_format (ex: png or jpg). Default jpg
2245
+ :type fig_format: str
2246
+ :return: None
2247
+ :rtype: None
2248
+ """
2249
+
2250
+ # get specs
2251
+ default_specs = {
2252
+ "color": "tab:grey",
2253
+ "color_scatter": "tab:blue",
2254
+ "title": "View of {}".format(self.name),
2255
+ "width": 6,
2256
+ "height": 6,
2257
+ "range_x": [self.data[self.xname].min(), self.data[self.xname].max()],
2258
+ "range": [self.data[self.yname].min(), self.data[self.yname].max()],
2259
+ "xlabel": self.xname,
2260
+ "ylabel": self.yname,
2261
+ }
2262
+ # handle inputs specs
2263
+ if specs is None:
2264
+ pass
2265
+ else: # override default
2266
+ for k in specs:
2267
+ default_specs[k] = specs[k]
2268
+ specs = default_specs
2269
+
2270
+ # start plot
2271
+ fig = plt.figure(figsize=(specs["width"], specs["height"])) # Width, Height
2272
+
2273
+ plt.suptitle(specs["title"])
2274
+
2275
+ # grid
2276
+ gs = mpl.gridspec.GridSpec(
2277
+ 3, 3, wspace=0.3, hspace=0.3, left=0.12, bottom=0.1, top=0.90, right=0.95
2278
+ ) # nrows, ncols
2279
+
2280
+ # scatter plot
2281
+ ax = fig.add_subplot(gs[1:, :2])
2282
+ plt.scatter(
2283
+ self.data[self.xname],
2284
+ self.data[self.yname],
2285
+ marker=".",
2286
+ color=specs["color_scatter"],
2287
+ )
2288
+ plt.xlabel(specs["xlabel"])
2289
+ plt.ylabel(specs["ylabel"])
2290
+ plt.xlim(specs["range_x"])
2291
+ plt.ylim(specs["range"])
2292
+
2293
+ # x hist
2294
+ ax_histx = fig.add_subplot(gs[0, :2], sharex=ax)
2295
+ plt.ylabel("p({})".format(self.xname))
2296
+ xuni = Univar()
2297
+ xuni.data = self.data[self.xname].values
2298
+ plt.hist(
2299
+ self.data[self.xname],
2300
+ bins=xuni.get_nbins_fd(data=xuni.data),
2301
+ color=specs["color"],
2302
+ alpha=1,
2303
+ weights=np.ones(len(self.data)) / len(self.data),
2304
+ )
2305
+ plt.xlim(specs["range_x"])
2306
+
2307
+ # y hist
2308
+ ay_histy = fig.add_subplot(gs[1:, 2], sharey=ax)
2309
+ plt.xlabel("p({})".format(self.yname))
2310
+ yuni = Univar()
2311
+ yuni.data = self.data[self.yname].values
2312
+ plt.hist(
2313
+ yuni.data,
2314
+ bins=yuni.get_nbins_fd(data=yuni.data),
2315
+ color=specs["color"],
2316
+ alpha=1,
2317
+ orientation="horizontal",
2318
+ weights=np.ones(len(self.data)) / len(self.data),
2319
+ )
2320
+ plt.ylim(specs["range"])
2321
+
2322
+ # show or save
2323
+ if show:
2324
+ plt.show()
2325
+ else:
2326
+ if filename is None:
2327
+ filename = self.name
2328
+ plt.savefig("{}/{}.{}".format(folder, filename, fig_format), dpi=dpi)
2329
+ plt.close(fig)
2330
+
2331
+ def view_model(
2332
+ self,
2333
+ model_type="Power",
2334
+ show=True,
2335
+ folder="C:/sample",
2336
+ filename=None,
2337
+ specs=None,
2338
+ dpi=300,
2339
+ fig_format="jpg",
2340
+ ):
2341
+ """Plot pannel for model analysis
2342
+
2343
+ :param show: Boolean to show instead of saving
2344
+ :type show: bool
2345
+ :param folder: output folder
2346
+ :type folder: str
2347
+ :param filename: image file name
2348
+ :type filename: str
2349
+ :param specs: specification dictionary
2350
+ :type specs: dict
2351
+ :param dpi: image resolution (default = 300)
2352
+ :type dpi: int
2353
+ :param fig_format: image fig_format (ex: png or jpg). Default jpg
2354
+ :type fig_format: str
2355
+ :return: None
2356
+ :rtype: None
2357
+ """
2358
+
2359
+ # ensure data is update
2360
+ self.updata_model_data(model_type=model_type)
2361
+
2362
+ # get specs
2363
+ default_specs = {
2364
+ "color": "tab:grey",
2365
+ "color_scatter": "tab:blue",
2366
+ "color_model": "black",
2367
+ "color_variance": "darkred",
2368
+ "title": "View of {} {} Model".format(self.name, model_type),
2369
+ "width": 8,
2370
+ "height": 8,
2371
+ "range_x": [self.data[self.xname].min(), self.data[self.xname].max()],
2372
+ "range": [self.data[self.yname].min(), self.data[self.yname].max()],
2373
+ "elim": [
2374
+ -1.5 * self.models[model_type]["Data"]["e_Mean"].max(),
2375
+ 1.5 * self.models[model_type]["Data"]["e_Mean"].max(),
2376
+ ],
2377
+ "alpha_xy": 0.75,
2378
+ "alpha_e": 0.75,
2379
+ }
2380
+ # handle inputs specs
2381
+ if specs is None:
2382
+ pass
2383
+ else: # override default
2384
+ for k in specs:
2385
+ default_specs[k] = specs[k]
2386
+ specs = default_specs
2387
+
2388
+ # get some ranges
2389
+ e_range = specs["elim"][1] - specs["elim"][0]
2390
+ e_range_10 = e_range / 20
2391
+ x_range = specs["range_x"][1] - specs["range_x"][0]
2392
+ x_range_10 = x_range / 10
2393
+
2394
+ # -------------- start plot
2395
+ fig = plt.figure(figsize=(specs["width"], specs["height"])) # Width, Height
2396
+ plt.suptitle(specs["title"])
2397
+ # grid
2398
+ gs = mpl.gridspec.GridSpec(
2399
+ 4, 4, wspace=0.3, hspace=0.45, left=0.12, bottom=0.1, top=0.90, right=0.95
2400
+ ) # nrows, ncols
2401
+
2402
+ # ----------------- XY plot -----------------
2403
+ ax = fig.add_subplot(gs[:2, :2])
2404
+
2405
+ plt.scatter(
2406
+ self.data[self.xname],
2407
+ self.data[self.yname],
2408
+ marker=".",
2409
+ color=specs["color_scatter"],
2410
+ alpha=specs["alpha_xy"],
2411
+ zorder=1,
2412
+ )
2413
+ plt.plot(
2414
+ self.models[model_type]["Data"][self.xname],
2415
+ self.models[model_type]["Data"]["{}_Mean".format(self.yname)],
2416
+ color=specs["color_model"],
2417
+ zorder=2,
2418
+ linestyle="--",
2419
+ )
2420
+ plt.xlabel(self.xname)
2421
+ plt.ylabel(self.yname)
2422
+ plt.xlim(specs["range_x"])
2423
+ plt.ylim(specs["range"])
2424
+
2425
+ # ----------------- error -----------------
2426
+ e_uni = Univar(data=self.models[model_type]["Data"]["e_Mean"].values)
2427
+ # get mean
2428
+ e_mean = np.mean(e_uni.data)
2429
+
2430
+ ax = fig.add_subplot(gs[2, :2])
2431
+ plt.scatter(
2432
+ self.models[model_type]["Data"][self.xname],
2433
+ self.models[model_type]["Data"]["e_Mean"].values,
2434
+ marker=".",
2435
+ alpha=specs["alpha_e"],
2436
+ color=specs["color"],
2437
+ )
2438
+ # mean line
2439
+ plt.hlines(
2440
+ y=e_mean,
2441
+ xmin=specs["range_x"][0],
2442
+ xmax=specs["range_x"][1],
2443
+ color="tab:red",
2444
+ alpha=0.4,
2445
+ )
2446
+ plt.text(
2447
+ y=e_mean + e_range_10,
2448
+ x=specs["range_x"][1] - (3 * x_range_10),
2449
+ s=r"$\mu$: " + str(round(e_mean, 2)),
2450
+ color="tab:red",
2451
+ )
2452
+ plt.xlabel(self.xname)
2453
+ plt.ylabel(r"$\epsilon$")
2454
+ plt.xlim(specs["range_x"])
2455
+ plt.ylim(specs["elim"])
2456
+
2457
+ # ----------------- error hist -----------------
2458
+ ax = fig.add_subplot(gs[2, 2])
2459
+ plt.hist(
2460
+ e_uni.data,
2461
+ bins=e_uni.get_nbins_fd(),
2462
+ color=specs["color"],
2463
+ alpha=1,
2464
+ orientation="horizontal",
2465
+ weights=np.ones(len(self.data)) / len(self.data),
2466
+ )
2467
+ # mean line
2468
+ plt.hlines(y=e_mean, xmin=0, xmax=0.15, color="tab:red", alpha=0.4)
2469
+ plt.ylim(specs["elim"])
2470
+ plt.xlabel(r"p($\epsilon$)")
2471
+
2472
+ # ----------------- qq plot error -----------------
2473
+ df_qq = e_uni.qqplot()
2474
+ ax = fig.add_subplot(gs[2, 3])
2475
+ plt.scatter(
2476
+ x=df_qq["T-Quantiles"],
2477
+ y=df_qq["Data"],
2478
+ marker=".",
2479
+ alpha=specs["alpha_e"],
2480
+ color=specs["color"],
2481
+ )
2482
+ # mean line
2483
+ plt.hlines(
2484
+ y=e_mean,
2485
+ xmin=df_qq["T-Quantiles"].min(),
2486
+ xmax=df_qq["T-Quantiles"].max(),
2487
+ color="tab:red",
2488
+ alpha=0.4,
2489
+ )
2490
+ plt.ylim(specs["elim"])
2491
+ plt.xlabel("normal quantiles")
2492
+
2493
+ # ----------------- variance tracing -----------------
2494
+ ax = fig.add_subplot(gs[3, :2])
2495
+ vct_var = e_uni.trace_variance()
2496
+ plt.plot(
2497
+ self.models[model_type]["Data"][self.xname],
2498
+ vct_var,
2499
+ color=specs["color_variance"],
2500
+ )
2501
+
2502
+ plt.xlabel(self.xname)
2503
+ plt.ylabel(r"$\sigma^2$")
2504
+ plt.xlim(specs["range_x"])
2505
+ plt.ylim([0, 1.5 * np.max(vct_var)])
2506
+
2507
+ # show or save
2508
+ if show:
2509
+ plt.show()
2510
+ else:
2511
+ if filename is None:
2512
+ filename = self.name
2513
+ plt.savefig("{}/{}.{}".format(folder, filename, fig_format), dpi=dpi)
2514
+ plt.close(fig)
2515
+ return None
2516
+
2517
+ def correlation(self):
2518
+ """Compute the R correlation coefficient of the base_object
2519
+
2520
+ :return: R correlation coefficient
2521
+ :rtype: float
2522
+ """
2523
+ corr_df = self.data.corr().loc[self.xname, self.yname]
2524
+ return corr_df
2525
+
2526
+ def prediction_bands(
2527
+ self, lst_bounds=None, n_sim=100, n_grid=100, n_seed=None, p0=None
2528
+ ):
2529
+ """Run Monte Carlo Simulation to get prediciton bands
2530
+
2531
+ :param lst_bounds: list of prediction bounds [min, max], if None, 3x the date range is used
2532
+ :type lst_bounds: list
2533
+ :param n_sim: number of simulation runs
2534
+ :type n_sim: int
2535
+ :param n_grid: number of prediction intervals
2536
+ :type n_grid: int
2537
+ :param n_seed: number of random seed for reproducibility. Default = None
2538
+ :type n_seed: int
2539
+ :param p0: list of initial values to search. Default: None
2540
+ :type p0: list
2541
+ :return: base_object with result dataframes
2542
+ :rtype: dict
2543
+ """
2544
+ from scipy.optimize import curve_fit
2545
+
2546
+ # todo this seems to be deprecated
2547
+ # handle None bounds
2548
+ if lst_bounds is None:
2549
+ _min = self.data[self.xname].min()
2550
+ _max = self.data[self.xname].max()
2551
+ # extrapolation zone 2x the interpolation zone
2552
+ lst_bounds = [_min, _max + (2 * _min)]
2553
+ # handle None seed
2554
+ if n_seed is None:
2555
+ from datetime import datetime
2556
+
2557
+ _seed = int(datetime.now().timestamp())
2558
+ n_seed = np.random.seed(_seed)
2559
+
2560
+ # --- set dataframes
2561
+ # scale of simulations
2562
+ n_scale = int(np.log10(n_sim)) + 1
2563
+
2564
+ # simulation labels
2565
+ lst_mcsim = ["MC{}".format(str(i).zfill(n_scale)) for i in range(n_sim + 1)]
2566
+ s_refsim = lst_mcsim[0]
2567
+
2568
+ # parameter labels
2569
+ _lst_params = list(self.linear_model["Parameter"].values)
2570
+ _labels = ["Mean", "Std"]
2571
+ lst_params = list()
2572
+ for i in range(len(_labels)):
2573
+ _label = _labels[i]
2574
+ for j in range(len(_lst_params)):
2575
+ _param = _lst_params[j]
2576
+ lst_params.append("{}_{}".format(_param, _label))
2577
+
2578
+ # models dataframe
2579
+ _cols = lst_params.copy()
2580
+ _cols.insert(0, "Model")
2581
+ _data = np.zeros(shape=(n_sim + 1, len(_cols)))
2582
+ df_models = pd.DataFrame(data=_data, columns=_cols)
2583
+ df_models["Model"] = lst_mcsim
2584
+
2585
+ # prediction dataframe
2586
+ _cols = ["{}_{}".format(self.yname, s) for s in lst_mcsim]
2587
+ _cols.insert(0, self.xname)
2588
+ _data = np.zeros(shape=(n_grid, len(_cols)))
2589
+ df_preds = pd.DataFrame(data=_data, columns=_cols)
2590
+ _min = np.min(lst_bounds)
2591
+ _max = np.max(lst_bounds)
2592
+ df_preds[self.xname] = np.linspace(_min, _max, n_grid)
2593
+
2594
+ # bands dataframe
2595
+ _cols = ["Min", "Max", "Mean", "p01", "p05", "p25", "p50", "p75", "p95", "p99"]
2596
+ _cols.insert(0, self.xname)
2597
+ _data = np.zeros(shape=(n_grid, len(_cols)))
2598
+ df_bands = pd.DataFrame(data=_data, columns=_cols)
2599
+ df_bands[self.xname] = df_preds[self.xname].values
2600
+
2601
+ # set reference simulation parameters
2602
+ _lst = ["Mean", "Std"]
2603
+ for s in _lst:
2604
+ for i in range(len(self.linear_model)):
2605
+ _s_par = self.linear_model["Parameter"].values[i]
2606
+ _s_key = "{}_{}".format(_s_par, s)
2607
+ _value = self.linear_model["Fit_{}".format(s)].values[i]
2608
+ df_models[_s_key].values[0] = _value
2609
+
2610
+ # set reference simulation prediction
2611
+ _x = df_preds[self.xname].values
2612
+ _params = self.linear_model["Fit_Mean"].values
2613
+ df_preds["y_{}".format(s_refsim)] = linear(_x, *_params)
2614
+
2615
+ # get standard deviation from fit sample
2616
+ n_std_e = self.linear_model_data["e"].std()
2617
+ vct_yfit = self.linear_model_data["{}_fit".format(self.yname)].values
2618
+
2619
+ # simulation loop:
2620
+ for n in range(1, n_sim + 1):
2621
+ _sim = lst_mcsim[n]
2622
+
2623
+ # get new error
2624
+ vct_e = np.random.normal(0.0, n_std_e, len(self.linear_model_data))
2625
+ # get new y obs
2626
+ vct_yobs = vct_yfit + vct_e
2627
+
2628
+ # fit new model
2629
+ if p0 is None:
2630
+ popt, pcov = curve_fit(
2631
+ f=linear, xdata=self.linear_model_data[self.xname], ydata=vct_yobs
2632
+ )
2633
+ else:
2634
+ popt, pcov = curve_fit(
2635
+ f=linear,
2636
+ xdata=self.linear_model_data[self.xname],
2637
+ ydata=vct_yobs,
2638
+ p0=p0,
2639
+ )
2640
+ pstd = np.sqrt(np.diag(pcov))
2641
+
2642
+ # store model values
2643
+ _dct = {"Mean": popt, "Std": pstd}
2644
+ for s in _dct:
2645
+ for i in range(len(self.linear_model)):
2646
+ _s_par = self.linear_model["Parameter"].values[i]
2647
+ _s_key = "{}_{}".format(_s_par, s)
2648
+ _value = _dct[s][i]
2649
+ df_models[_s_key].values[n] = _value
2650
+
2651
+ # store prediction values
2652
+ _x = df_preds[self.xname].values
2653
+ df_preds["y_{}".format(_sim)] = linear(_x, *popt)
2654
+
2655
+ # get bands
2656
+ for i in range(len(df_preds)):
2657
+ _values = df_preds.values[i, 1:]
2658
+ df_bands["Min"].values[i] = np.min(_values)
2659
+ df_bands["Max"].values[i] = np.max(_values)
2660
+ df_bands["Mean"].values[i] = np.mean(_values)
2661
+ df_bands["p01"].values[i] = np.quantile(_values, 0.01)
2662
+ df_bands["p05"].values[i] = np.quantile(_values, 0.05)
2663
+ df_bands["p25"].values[i] = np.quantile(_values, 0.25)
2664
+ df_bands["p50"].values[i] = np.quantile(_values, 0.5)
2665
+ df_bands["p75"].values[i] = np.quantile(_values, 0.75)
2666
+ df_bands["p95"].values[i] = np.quantile(_values, 0.95)
2667
+ df_bands["p99"].values[i] = np.quantile(_values, 0.99)
2668
+
2669
+ # return base_object
2670
+ return {"Models": df_models, "Predictions": df_preds, "Bands": df_bands}
2671
+
2672
+ def assess_error_normality(self, model_type="Linear", clevel=0.95):
2673
+ self.updata_model_data(model_type=model_type)
2674
+ vct_e = self.models[model_type]["Data"]["e_Mean"].values
2675
+ uni = Univar(data=vct_e, name=self.name)
2676
+ df = uni.get_normality(clevel=clevel)
2677
+ return df
2678
+
2679
+ @staticmethod
2680
+ def bias(pred, obs):
2681
+ """Compute the Bias between predicted and observated sample
2682
+
2683
+ :param pred: :class:`numpy.ndarray`
2684
+ vector of prediction vales
2685
+ :type pred: :class:`numpy.ndarray`
2686
+ :param obs: :class:`numpy.ndarray`
2687
+ vector of observed values
2688
+ :type obs: :class:`numpy.ndarray`
2689
+ :return: Bias value
2690
+ :rtype: float
2691
+ """
2692
+ return 100 * np.sum(pred - obs) / np.sum(obs)
2693
+
2694
+ @staticmethod
2695
+ def rmse(pred, obs):
2696
+ """Compute the RMSE metric between predicted and observated sample
2697
+
2698
+ :param pred: :class:`numpy.ndarray`
2699
+ vector of prediction vales
2700
+ :type pred: :class:`numpy.ndarray`
2701
+ :param obs: :class:`numpy.ndarray`
2702
+ vector of observed values
2703
+ :type obs: :class:`numpy.ndarray`
2704
+ :return: RMSE value
2705
+ :rtype: float
2706
+ """
2707
+ return np.sqrt(np.sum(np.power(pred - obs, 2)) / len(pred))
2708
+
2709
+ @staticmethod
2710
+ def mae(pred, obs):
2711
+ """Compute the Mean Absolute Error (MAE) between predicted and observated sample
2712
+
2713
+ :param pred: :class:`numpy.ndarray`
2714
+ vector of prediction vales
2715
+ :type pred: :class:`numpy.ndarray`
2716
+ :param obs: :class:`numpy.ndarray`
2717
+ vector of observed values
2718
+ :type obs: :class:`numpy.ndarray`
2719
+ :return: MAE value
2720
+ :rtype: float
2721
+ """
2722
+ return np.sum(np.abs(pred - obs)) / len(pred)
2723
+
2724
+ @staticmethod
2725
+ def rsq(pred, obs):
2726
+ """Compute the R-square between predicted and observated sample
2727
+
2728
+ :param pred: :class:`numpy.ndarray`
2729
+ vector of prediction vales
2730
+ :type pred: :class:`numpy.ndarray`
2731
+ :param obs: :class:`numpy.ndarray`
2732
+ vector of observed values
2733
+ :type obs: :class:`numpy.ndarray`
2734
+ :return: R-square value
2735
+ :rtype: float
2736
+ """
2737
+ return 1 - (
2738
+ np.sum(np.power(pred - obs, 2)) / np.sum(np.power(obs - np.mean(obs), 2))
2739
+ )
2740
+
2741
+
2742
+ # todo [upgrade] make it a child of some MbaE class
2743
+ class Bayes:
2744
+ # todo [docstring] improve major docstring
2745
+ """
2746
+ The Bayes Theorem Analyst Object
2747
+ """
2748
+
2749
+ def __init__(self, df_hypotheses, name="myBayes", nomenclature=None, gridsize=100):
2750
+ """Deploy the Bayes Analyst
2751
+ :param df_hypotheses: dataframe listing all model hypotheses. Must contain a field Name (of parameter), Min and Max.
2752
+ :type df_hypotheses: :class:pandas.DataFrame
2753
+ :param name: name of analyst
2754
+ :type name: str
2755
+ :param nomenclature: dictionary for rename nomenclature
2756
+ :type nomenclature: dict
2757
+ :param gridsize: grid resolution in histrograms (bins)
2758
+ :type gridsize: int
2759
+ """
2760
+ self.hypotheses = df_hypotheses
2761
+ # list of hypotheses by step
2762
+ self.bands = list()
2763
+ self.bands.append(self.hypotheses.copy())
2764
+ # other parameters
2765
+ self.name = name
2766
+ self.gridsize = gridsize
2767
+
2768
+ # zero
2769
+ self.zero = 1 / gridsize
2770
+
2771
+ # set labels
2772
+ self.shyp = "H"
2773
+ self.sevid = "E"
2774
+ self.sprior = "P(H)"
2775
+ self.slike = "P(E | H)"
2776
+ self.spost = "P(H | E)"
2777
+ self.saux = self.sprior + self.slike
2778
+ if nomenclature is not None:
2779
+ self._reset_nomenclature(dct_names=nomenclature)
2780
+
2781
+ # set omega
2782
+ self.steps = list()
2783
+ # create step 0
2784
+ self._insert_new_step()
2785
+
2786
+ def __str__(self):
2787
+ s1 = self.hypotheses.to_string()
2788
+ return s1
2789
+
2790
+ def _reset_nomenclature(self, dct_names):
2791
+ """Reset nomenclature
2792
+ :param dct_names: dictionary to rename nomenclatures
2793
+ :type dct_names: dict
2794
+ :return: None
2795
+ :rtype: None
2796
+ """
2797
+ for k in dct_names:
2798
+ self.sevid = self.sevid.replace(k, dct_names[k])
2799
+ self.shyp = self.shyp.replace(k, dct_names[k])
2800
+ self.sprior = self.sprior.replace(k, dct_names[k])
2801
+ self.slike = self.slike.replace(k, dct_names[k])
2802
+ self.spost = self.spost.replace(k, dct_names[k])
2803
+ self.saux = self.sprior + self.slike
2804
+
2805
+ def _insert_new_step(self):
2806
+ """convenience void function for inserting new step objects
2807
+ :return: None
2808
+ :rtype:
2809
+ """
2810
+ self.steps.append(dict())
2811
+ self.steps[len(self.steps) - 1]["Omega"] = dict()
2812
+ self.steps[len(self.steps) - 1]["Evidence"] = dict()
2813
+ self.steps[len(self.steps) - 1]["Bands"] = dict()
2814
+ for h in self.hypotheses["Name"].values:
2815
+ n_min = self.hypotheses.loc[self.hypotheses["Name"] == h, "Min"].values[0]
2816
+ n_max = self.hypotheses.loc[self.hypotheses["Name"] == h, "Max"].values[0]
2817
+ self.steps[len(self.steps) - 1]["Omega"][h] = pd.DataFrame(
2818
+ {
2819
+ self.shyp: np.linspace(n_min, n_max, self.gridsize),
2820
+ self.sprior: np.ones(self.gridsize) / self.gridsize,
2821
+ self.slike: np.zeros(self.gridsize),
2822
+ self.saux: np.zeros(self.gridsize),
2823
+ self.spost: np.zeros(self.gridsize),
2824
+ }
2825
+ )
2826
+ return None
2827
+
2828
+ def _accumulate(self, n_step):
2829
+ """convenience void function for accumulate probability
2830
+ :param n_step: step number to accumulate
2831
+ :type n_step: int
2832
+ :return: None
2833
+ :rtype: none
2834
+ """
2835
+ lst_labels1 = [self.sprior, self.slike, self.spost]
2836
+ for h in self.hypotheses["Name"].values:
2837
+ for i in range(len(lst_labels1)):
2838
+ s_field0 = lst_labels1[i]
2839
+ s_field1 = "{}_acc".format(lst_labels1[i])
2840
+ self.steps[n_step]["Omega"][h][s_field1] = 0.0
2841
+ for j in range(len(self.steps[n_step]["Omega"][h])):
2842
+ if j == 0:
2843
+ self.steps[n_step]["Omega"][h][s_field1].values[j] = self.steps[
2844
+ n_step
2845
+ ]["Omega"][h][s_field0].values[j]
2846
+ else:
2847
+ self.steps[n_step]["Omega"][h][s_field1].values[j] = (
2848
+ self.steps[n_step]["Omega"][h][s_field1].values[j - 1]
2849
+ + self.steps[n_step]["Omega"][h][s_field0].values[j]
2850
+ )
2851
+ return None
2852
+
2853
+ def conditionalize(self, dct_evidence, s_varfield="E", s_weightfield="W"):
2854
+ """Conditionalize procedure of the Bayes Theorem
2855
+ :param dct_evidence: base_object of evidence dataframes
2856
+ :type dct_evidence: dict
2857
+ :param s_varfield: name of variable field in evidence dataframes
2858
+ :type s_varfield: str
2859
+ :param s_weightfield: name of weights field in evidence dataframes
2860
+ :type s_weightfield: str
2861
+ :return: None
2862
+ :rtype: none
2863
+ """
2864
+ # instantiate new step
2865
+ n_step = len(self.steps)
2866
+ self._insert_new_step()
2867
+
2868
+ # set evidence dict for step
2869
+ self.steps[n_step]["Evidence"] = dct_evidence
2870
+
2871
+ # main loop
2872
+ for h in self.hypotheses["Name"].values:
2873
+ # get prior from last step
2874
+ if n_step > 1:
2875
+ self.steps[n_step]["Omega"][h][self.sprior] = self.steps[n_step - 1][
2876
+ "Omega"
2877
+ ][h][self.spost].values
2878
+ # get likelihood
2879
+ hist, edges = np.histogram(
2880
+ a=dct_evidence[h][s_varfield],
2881
+ bins=self.steps[n_step]["Omega"][h][self.shyp],
2882
+ weights=dct_evidence[h][s_weightfield],
2883
+ )
2884
+ hist = hist / np.sum(hist)
2885
+ hist = list(hist)
2886
+ hist.append(0)
2887
+ self.steps[n_step]["Omega"][h][self.slike] = hist
2888
+ # insert baseline and normalize
2889
+ v_like = self.steps[n_step]["Omega"][h][self.slike].values
2890
+ v_like = v_like + self.zero
2891
+ self.steps[n_step]["Omega"][h][self.slike] = v_like / np.sum(v_like)
2892
+
2893
+ # get posterior with Bayes Theorem
2894
+ v_prior = self.steps[n_step]["Omega"][h][self.sprior].values
2895
+ v_like = self.steps[n_step]["Omega"][h][self.slike].values
2896
+ v_posterior = (v_prior * v_like) / np.sum(v_prior * v_like)
2897
+ self.steps[n_step]["Omega"][h][self.spost] = v_posterior
2898
+
2899
+ # get accumulated values
2900
+ self._accumulate(n_step)
2901
+
2902
+ self.steps[n_step]["Bands"][h] = dict()
2903
+
2904
+ # get percentiles of accumulated
2905
+ lst_aux = [self.sprior, self.slike, self.spost]
2906
+ for s in lst_aux:
2907
+ self.steps[n_step]["Bands"][h][s] = dict()
2908
+ # 90% band
2909
+ s_query = "`{}` > 0.05 and `{}` <= 0.95".format(s + "_acc", s + "_acc")
2910
+ df_aux = self.steps[n_step]["Omega"][h].query(s_query)
2911
+ self.steps[n_step]["Bands"][h][s]["p05"] = df_aux[self.shyp].min()
2912
+ self.steps[n_step]["Bands"][h][s]["p95"] = df_aux[self.shyp].max()
2913
+ # 50% band
2914
+ s_query = "`{}` > 0.25 and `{}` <= 0.75".format(s + "_acc", s + "_acc")
2915
+ df_aux = self.steps[n_step]["Omega"][h].query(s_query)
2916
+ self.steps[n_step]["Bands"][h][s]["p25"] = df_aux[self.shyp].min()
2917
+ self.steps[n_step]["Bands"][h][s]["p75"] = df_aux[self.shyp].max()
2918
+
2919
+ # return
2920
+ return None
2921
+
2922
+ def plot_step(
2923
+ self,
2924
+ n_step,
2925
+ folder="C:/sample",
2926
+ filename="bayes",
2927
+ specs=None,
2928
+ dpi=300,
2929
+ show=False,
2930
+ ):
2931
+ """Void function for plot pannel of conditionalization step
2932
+ :param n_step: step number
2933
+ :type n_step: int
2934
+ :param folder: export folder
2935
+ :type folder: str
2936
+ :param filename: file name
2937
+ :type filename: str
2938
+ :param specs: plot specs base_object
2939
+ :type specs: dict
2940
+ :param dpi: plot resolution
2941
+ :type dpi: int
2942
+ :param show: control to show plot instead of saving
2943
+ :type show: bool
2944
+ :return: None
2945
+ :rtype: None
2946
+ """
2947
+ # get specs
2948
+ default_specs = {
2949
+ "color": "tab:grey",
2950
+ "title": "Conditionalization",
2951
+ "width": 10,
2952
+ "height": 5,
2953
+ "xlabel": "value",
2954
+ "range": (0, 1),
2955
+ "range_x": (0, 1),
2956
+ "subtitle": None,
2957
+ }
2958
+ # handle inputs specs
2959
+ if specs is None:
2960
+ pass
2961
+ else: # override default
2962
+ for k in specs:
2963
+ default_specs[k] = specs[k]
2964
+ specs = default_specs
2965
+
2966
+ # hunt some parameters
2967
+ lst_pmax = list()
2968
+ for h in self.hypotheses["Name"].values: # get objects
2969
+ _df = self.steps[n_step]["Omega"][h]
2970
+ # get max p
2971
+ lst_pmax.append(
2972
+ np.max(
2973
+ [
2974
+ _df[self.sprior].max(),
2975
+ _df[self.slike].max(),
2976
+ _df[self.spost].max(),
2977
+ ]
2978
+ )
2979
+ )
2980
+ n_pmax = 1.5 * np.max(lst_pmax)
2981
+
2982
+ # main loop
2983
+ for h in self.hypotheses["Name"].values:
2984
+ specs["subtitle"] = "${}$".format(h)
2985
+ # get objects
2986
+ _df = self.steps[n_step]["Omega"][h]
2987
+ _bands = self.steps[n_step]["Bands"][h]
2988
+
2989
+ # get width
2990
+ _wid = 0.7 * (_df[self.shyp].values[1] - _df[self.shyp].values[0])
2991
+
2992
+ # get min max
2993
+ n_min = self.hypotheses.loc[self.hypotheses["Name"] == h, "Min"].values[0]
2994
+ n_max = self.hypotheses.loc[self.hypotheses["Name"] == h, "Max"].values[0]
2995
+ # start plot
2996
+
2997
+ fig = plt.figure(figsize=(specs["width"], specs["height"])) # Width, Height
2998
+
2999
+ if specs["subtitle"] is None:
3000
+ plt.suptitle(specs["title"])
3001
+ else:
3002
+ plt.suptitle("{} | {}".format(specs["title"], specs["subtitle"]))
3003
+ # grid
3004
+ gs = mpl.gridspec.GridSpec(
3005
+ 2, 3, wspace=0.4, hspace=0.5, left=0.1, bottom=0.1, top=0.85, right=0.95
3006
+ ) # nrows, ncols
3007
+
3008
+ # grid loop
3009
+ lst_aux = [self.sprior, self.slike, self.spost]
3010
+ for i in range(len(lst_aux)):
3011
+ # Histogram
3012
+ ax = fig.add_subplot(gs[0, i])
3013
+ plt.title(lst_aux[i])
3014
+ plt.bar(
3015
+ _df[self.shyp],
3016
+ _df[lst_aux[i]],
3017
+ width=_wid,
3018
+ color=specs["color"],
3019
+ align="edge",
3020
+ )
3021
+ plt.ylim(0, n_pmax)
3022
+ plt.xlim(n_min, n_max)
3023
+ plt.ylabel("$p({})$".format(h))
3024
+ plt.xlabel("${}$".format(h))
3025
+ # plt.gca().set_aspect(1 / (1.5))
3026
+
3027
+ if i == 1:
3028
+ # Accumulated
3029
+ ax = fig.add_subplot(gs[1, i])
3030
+ # plt.title(lst_aux[i])
3031
+ plt.scatter(
3032
+ self.steps[n_step]["Evidence"][h]["E"],
3033
+ self.steps[n_step]["Evidence"][h]["W"],
3034
+ marker=".",
3035
+ c="k",
3036
+ alpha=0.5,
3037
+ linewidths=0.0,
3038
+ edgecolors=None,
3039
+ )
3040
+ plt.xlim(n_min, n_max)
3041
+ plt.ylabel("$W({})$".format(h))
3042
+ plt.xlabel("${}$".format(h))
3043
+ # plt.gca().set_aspect(1 / (1.5))
3044
+ else:
3045
+ # Accumulated
3046
+ ax = fig.add_subplot(gs[1, i])
3047
+ # plt.title(lst_aux[i])
3048
+ plt.plot(
3049
+ _df[self.shyp],
3050
+ _df["{}_acc".format(lst_aux[i])],
3051
+ color="tab:blue",
3052
+ zorder=3,
3053
+ )
3054
+ plt.fill_betweenx(
3055
+ [-1, 2],
3056
+ x1=self.steps[n_step]["Bands"][h][lst_aux[i]]["p25"],
3057
+ x2=self.steps[n_step]["Bands"][h][lst_aux[i]]["p75"],
3058
+ color="tab:grey",
3059
+ alpha=0.5,
3060
+ zorder=2,
3061
+ )
3062
+ plt.fill_betweenx(
3063
+ [-1, 2],
3064
+ x1=self.steps[n_step]["Bands"][h][lst_aux[i]]["p05"],
3065
+ x2=self.steps[n_step]["Bands"][h][lst_aux[i]]["p95"],
3066
+ color="tab:grey",
3067
+ alpha=0.3,
3068
+ zorder=1,
3069
+ )
3070
+ plt.ylim(-0.05, 1.05)
3071
+ plt.xlim(n_min, n_max)
3072
+ plt.ylabel("$P({})$".format(h))
3073
+ plt.xlabel("${}$".format(h))
3074
+ # plt.gca().set_aspect(1 / (1.5))
3075
+
3076
+ # show or save
3077
+ if show:
3078
+ plt.show()
3079
+ else:
3080
+ plt.savefig(
3081
+ "{}/{}_{}_step{}.png".format(folder, filename, h, n_step), dpi=dpi
3082
+ )
3083
+
3084
+ # return
3085
+ return None
3086
+
3087
+
3088
+ # ----- DEPRECATED CLASSES -----
3089
+
3090
+
3091
+ if __name__ == "__main__":
3092
+ print("Hello world!")