pastastore 1.10.2__py3-none-any.whl → 1.12.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- docs/conf.py +10 -97
- pastastore/__init__.py +5 -1
- pastastore/base.py +875 -272
- pastastore/connectors.py +359 -816
- pastastore/datasets.py +23 -33
- pastastore/extensions/__init__.py +7 -3
- pastastore/extensions/hpd.py +39 -17
- pastastore/plotting.py +71 -38
- pastastore/store.py +205 -186
- pastastore/styling.py +4 -2
- pastastore/typing.py +12 -0
- pastastore/util.py +322 -88
- pastastore/validator.py +524 -0
- pastastore/version.py +2 -3
- pastastore/yaml_interface.py +37 -39
- {pastastore-1.10.2.dist-info → pastastore-1.12.0.dist-info}/METADATA +17 -11
- pastastore-1.12.0.dist-info/RECORD +31 -0
- {pastastore-1.10.2.dist-info → pastastore-1.12.0.dist-info}/WHEEL +1 -1
- tests/conftest.py +156 -59
- tests/test_001_import.py +2 -1
- tests/test_002_connectors.py +40 -3
- tests/test_003_pastastore.py +60 -29
- tests/test_005_maps_plots.py +12 -0
- tests/test_006_benchmark.py +1 -1
- tests/test_007_hpdextension.py +46 -8
- tests/test_009_parallel.py +393 -0
- pastastore-1.10.2.dist-info/RECORD +0 -28
- {pastastore-1.10.2.dist-info → pastastore-1.12.0.dist-info}/licenses/LICENSE +0 -0
- {pastastore-1.10.2.dist-info → pastastore-1.12.0.dist-info}/top_level.txt +0 -0
pastastore/store.py
CHANGED
|
@@ -6,23 +6,23 @@ import os
|
|
|
6
6
|
import warnings
|
|
7
7
|
from functools import partial
|
|
8
8
|
from pathlib import Path
|
|
9
|
-
from typing import Dict, Iterable, List,
|
|
9
|
+
from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union
|
|
10
10
|
|
|
11
11
|
import numpy as np
|
|
12
12
|
import pandas as pd
|
|
13
13
|
import pastas as ps
|
|
14
|
-
from packaging.version import parse as parse_version
|
|
15
14
|
from pastas.io.pas import pastas_hook
|
|
16
15
|
from tqdm.auto import tqdm
|
|
17
16
|
|
|
18
17
|
from pastastore.base import BaseConnector
|
|
19
18
|
from pastastore.connectors import ArcticDBConnector, DictConnector, PasConnector
|
|
20
19
|
from pastastore.plotting import Maps, Plots
|
|
21
|
-
from pastastore.
|
|
22
|
-
from pastastore.
|
|
20
|
+
from pastastore.styling import boolean_styler
|
|
21
|
+
from pastastore.typing import FrameOrSeriesUnion, PastasLibs, TimeSeriesLibs
|
|
22
|
+
from pastastore.util import ZipUtils, _custom_warning
|
|
23
|
+
from pastastore.version import PASTAS_GEQ_150
|
|
23
24
|
from pastastore.yaml_interface import PastastoreYAML
|
|
24
25
|
|
|
25
|
-
FrameorSeriesUnion = Union[pd.DataFrame, pd.Series]
|
|
26
26
|
warnings.showwarning = _custom_warning
|
|
27
27
|
|
|
28
28
|
logger = logging.getLogger(__name__)
|
|
@@ -72,13 +72,15 @@ class PastaStore:
|
|
|
72
72
|
if connector is None:
|
|
73
73
|
connector = DictConnector("pastas_db")
|
|
74
74
|
self.conn = connector
|
|
75
|
+
self.validator = self.conn.validator
|
|
75
76
|
self.name = name if name is not None else self.conn.name
|
|
76
77
|
self._register_connector_methods()
|
|
77
78
|
|
|
78
|
-
# register map, plot and
|
|
79
|
+
# register map, plot, yaml and zip classes
|
|
79
80
|
self.maps = Maps(self)
|
|
80
81
|
self.plots = Plots(self)
|
|
81
82
|
self.yaml = PastastoreYAML(self)
|
|
83
|
+
self.zip = ZipUtils(self)
|
|
82
84
|
|
|
83
85
|
@classmethod
|
|
84
86
|
def from_pastastore_config_file(cls, fname, update_path: bool = True):
|
|
@@ -100,7 +102,7 @@ class PastaStore:
|
|
|
100
102
|
PastaStore
|
|
101
103
|
PastaStore
|
|
102
104
|
"""
|
|
103
|
-
with open(fname, "r") as f:
|
|
105
|
+
with open(fname, "r", encoding="utf-8") as f:
|
|
104
106
|
cfg = json.load(f)
|
|
105
107
|
conn_type = cfg.pop("connector_type")
|
|
106
108
|
if conn_type == "pas":
|
|
@@ -113,7 +115,7 @@ class PastaStore:
|
|
|
113
115
|
if update_path:
|
|
114
116
|
prefix, _ = cfg["uri"].split("://")
|
|
115
117
|
if prefix.lower() == "lmbd":
|
|
116
|
-
cfg["uri"] = prefix + "://" + str(Path(fname).parent)
|
|
118
|
+
cfg["uri"] = prefix + "://" + str(Path(fname).parent.parent)
|
|
117
119
|
conn = ArcticDBConnector(**cfg)
|
|
118
120
|
else:
|
|
119
121
|
raise ValueError(
|
|
@@ -238,7 +240,8 @@ class PastaStore:
|
|
|
238
240
|
|
|
239
241
|
@property
|
|
240
242
|
def _modelnames_cache(self):
|
|
241
|
-
|
|
243
|
+
# Backward-compat shim; prefer public property model_names
|
|
244
|
+
return self.conn.model_names
|
|
242
245
|
|
|
243
246
|
@property
|
|
244
247
|
def n_oseries(self):
|
|
@@ -295,13 +298,35 @@ class PastaStore:
|
|
|
295
298
|
"""
|
|
296
299
|
return self.conn.oseries_with_models
|
|
297
300
|
|
|
301
|
+
@property
|
|
302
|
+
def stresses_models(self):
|
|
303
|
+
"""Return dictionary of models per stress.
|
|
304
|
+
|
|
305
|
+
Returns
|
|
306
|
+
-------
|
|
307
|
+
dict
|
|
308
|
+
dictionary containing list of models (values) for each stress (keys).
|
|
309
|
+
"""
|
|
310
|
+
return self.conn.stresses_models
|
|
311
|
+
|
|
312
|
+
@property
|
|
313
|
+
def stresses_with_models(self):
|
|
314
|
+
"""Return list of stresses for which models are contained in the database.
|
|
315
|
+
|
|
316
|
+
Returns
|
|
317
|
+
-------
|
|
318
|
+
list
|
|
319
|
+
list of stress names for which models are contained in the database.
|
|
320
|
+
"""
|
|
321
|
+
return self.conn.stresses_with_models
|
|
322
|
+
|
|
298
323
|
def __repr__(self):
|
|
299
324
|
"""Representation string of the object."""
|
|
300
325
|
return f"<PastaStore> {self.name}: \n - " + self.conn.__str__()
|
|
301
326
|
|
|
302
327
|
def get_oseries_distances(
|
|
303
328
|
self, names: Optional[Union[list, str]] = None
|
|
304
|
-
) ->
|
|
329
|
+
) -> FrameOrSeriesUnion:
|
|
305
330
|
"""Get the distances in meters between the oseries.
|
|
306
331
|
|
|
307
332
|
Parameters
|
|
@@ -317,7 +342,7 @@ class PastaStore:
|
|
|
317
342
|
oseries_df = self.conn.oseries
|
|
318
343
|
other_df = self.conn.oseries
|
|
319
344
|
|
|
320
|
-
names = self.conn.
|
|
345
|
+
names = self.conn.parse_names(names)
|
|
321
346
|
|
|
322
347
|
xo = pd.to_numeric(oseries_df.loc[names, "x"])
|
|
323
348
|
xt = pd.to_numeric(other_df.loc[:, "x"])
|
|
@@ -340,7 +365,7 @@ class PastaStore:
|
|
|
340
365
|
names: Optional[Union[list, str]] = None,
|
|
341
366
|
n: int = 1,
|
|
342
367
|
maxdist: Optional[float] = None,
|
|
343
|
-
) ->
|
|
368
|
+
) -> FrameOrSeriesUnion:
|
|
344
369
|
"""Get the nearest (n) oseries.
|
|
345
370
|
|
|
346
371
|
Parameters
|
|
@@ -378,7 +403,7 @@ class PastaStore:
|
|
|
378
403
|
oseries: Optional[Union[list, str]] = None,
|
|
379
404
|
stresses: Optional[Union[list, str]] = None,
|
|
380
405
|
kind: Optional[Union[str, List[str]]] = None,
|
|
381
|
-
) ->
|
|
406
|
+
) -> FrameOrSeriesUnion:
|
|
382
407
|
"""Get the distances in meters between the oseries and stresses.
|
|
383
408
|
|
|
384
409
|
Parameters
|
|
@@ -400,7 +425,7 @@ class PastaStore:
|
|
|
400
425
|
oseries_df = self.conn.oseries
|
|
401
426
|
stresses_df = self.conn.stresses
|
|
402
427
|
|
|
403
|
-
oseries = self.conn.
|
|
428
|
+
oseries = self.conn.parse_names(oseries)
|
|
404
429
|
|
|
405
430
|
if stresses is None and kind is None:
|
|
406
431
|
stresses = stresses_df.index
|
|
@@ -437,7 +462,7 @@ class PastaStore:
|
|
|
437
462
|
kind: Optional[Union[list, str]] = None,
|
|
438
463
|
n: int = 1,
|
|
439
464
|
maxdist: Optional[float] = None,
|
|
440
|
-
) ->
|
|
465
|
+
) -> FrameOrSeriesUnion:
|
|
441
466
|
"""Get the nearest (n) stresses of a specific kind.
|
|
442
467
|
|
|
443
468
|
Parameters
|
|
@@ -474,24 +499,24 @@ class PastaStore:
|
|
|
474
499
|
|
|
475
500
|
def get_signatures(
|
|
476
501
|
self,
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
libname="oseries",
|
|
480
|
-
progressbar=False,
|
|
481
|
-
ignore_errors=False,
|
|
482
|
-
):
|
|
502
|
+
names: list[str] | str | None = None,
|
|
503
|
+
signatures: list[str] | None = None,
|
|
504
|
+
libname: TimeSeriesLibs = "oseries",
|
|
505
|
+
progressbar: bool = False,
|
|
506
|
+
ignore_errors: bool = False,
|
|
507
|
+
) -> pd.DataFrame | pd.Series:
|
|
483
508
|
"""Get groundwater signatures.
|
|
484
509
|
|
|
485
510
|
NaN-values are returned when the signature cannot be computed.
|
|
486
511
|
|
|
487
512
|
Parameters
|
|
488
513
|
----------
|
|
489
|
-
signatures : list of str, optional
|
|
490
|
-
list of groundwater signatures to compute, if None all groundwater
|
|
491
|
-
signatures in ps.stats.signatures.__all__ are used, by default None
|
|
492
514
|
names : str, list of str, or None, optional
|
|
493
515
|
names of the time series, by default None which
|
|
494
516
|
uses all the time series in the library
|
|
517
|
+
signatures : list of str, optional
|
|
518
|
+
list of groundwater signatures to compute, if None all groundwater
|
|
519
|
+
signatures in ps.stats.signatures.__all__ are used, by default None
|
|
495
520
|
libname : str
|
|
496
521
|
name of the library containing the time series
|
|
497
522
|
('oseries' or 'stresses'), by default "oseries"
|
|
@@ -503,34 +528,41 @@ class PastaStore:
|
|
|
503
528
|
|
|
504
529
|
Returns
|
|
505
530
|
-------
|
|
506
|
-
signatures_df : pandas.DataFrame
|
|
507
|
-
|
|
508
|
-
"""
|
|
509
|
-
names = self.conn._parse_names(names, libname=libname)
|
|
531
|
+
signatures_df : pandas.DataFrame or pandas.Series
|
|
532
|
+
Containing the time series (columns) and the signatures (index).
|
|
510
533
|
|
|
511
|
-
|
|
512
|
-
|
|
534
|
+
Note
|
|
535
|
+
----
|
|
536
|
+
Names is set as the first argument to allow parallelization.
|
|
537
|
+
"""
|
|
538
|
+
names = self.conn.parse_names(names, libname=libname)
|
|
513
539
|
|
|
514
|
-
|
|
515
|
-
|
|
540
|
+
signatures = (
|
|
541
|
+
ps.stats.signatures.__all__.copy() if signatures is None else signatures
|
|
542
|
+
)
|
|
516
543
|
|
|
544
|
+
sigs = []
|
|
517
545
|
# loop through oseries names
|
|
518
|
-
|
|
519
|
-
|
|
546
|
+
for name in (
|
|
547
|
+
tqdm(names, desc="Get groundwater signatures") if progressbar else names
|
|
548
|
+
):
|
|
549
|
+
sig = pd.Series(np.nan, index=signatures, name=name)
|
|
520
550
|
try:
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
551
|
+
s = (
|
|
552
|
+
self.conn.get_oseries(name)
|
|
553
|
+
if libname == "oseries"
|
|
554
|
+
else self.conn.get_stresses(name)
|
|
555
|
+
)
|
|
525
556
|
except Exception as e:
|
|
526
557
|
if ignore_errors:
|
|
527
|
-
|
|
528
|
-
continue
|
|
558
|
+
return sig
|
|
529
559
|
else:
|
|
530
560
|
raise e
|
|
531
561
|
|
|
532
562
|
try:
|
|
533
|
-
i_signatures = ps.stats.signatures.summary(
|
|
563
|
+
i_signatures: pd.Series = ps.stats.signatures.summary(
|
|
564
|
+
s.squeeze(), signatures
|
|
565
|
+
).squeeze("columns")
|
|
534
566
|
except Exception as e:
|
|
535
567
|
if ignore_errors:
|
|
536
568
|
i_signatures = []
|
|
@@ -544,23 +576,27 @@ class PastaStore:
|
|
|
544
576
|
i_signatures.append(sign_val)
|
|
545
577
|
else:
|
|
546
578
|
raise e
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
579
|
+
sig.loc[signatures] = i_signatures.values
|
|
580
|
+
sigs.append(sig)
|
|
581
|
+
if len(sigs) == 1:
|
|
582
|
+
return sigs[0]
|
|
583
|
+
else:
|
|
584
|
+
return pd.concat(sigs, axis=1)
|
|
550
585
|
|
|
551
586
|
def get_tmin_tmax(
|
|
552
587
|
self,
|
|
553
|
-
libname:
|
|
588
|
+
libname: Optional[PastasLibs] = None,
|
|
554
589
|
names: Union[str, List[str], None] = None,
|
|
555
590
|
progressbar: bool = False,
|
|
556
|
-
):
|
|
557
|
-
"""Get tmin and tmax for time series.
|
|
591
|
+
) -> pd.DataFrame:
|
|
592
|
+
"""Get tmin and tmax for time series and/or models.
|
|
558
593
|
|
|
559
594
|
Parameters
|
|
560
595
|
----------
|
|
561
|
-
libname : str
|
|
562
|
-
name of the library containing the time series
|
|
563
|
-
|
|
596
|
+
libname : str, optional
|
|
597
|
+
name of the library containing the time series ('oseries', 'stresses',
|
|
598
|
+
'models', or None), by default None which returns tmin/tmax for all
|
|
599
|
+
libraries
|
|
564
600
|
names : str, list of str, or None, optional
|
|
565
601
|
names of the time series, by default None which
|
|
566
602
|
uses all the time series in the library
|
|
@@ -570,31 +606,43 @@ class PastaStore:
|
|
|
570
606
|
Returns
|
|
571
607
|
-------
|
|
572
608
|
tmintmax : pd.dataframe
|
|
573
|
-
Dataframe containing tmin and tmax per time series
|
|
609
|
+
Dataframe containing tmin and tmax per time series and/or model
|
|
574
610
|
"""
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
611
|
+
results = {}
|
|
612
|
+
if libname is None:
|
|
613
|
+
libs = ["oseries", "stresses", "models"]
|
|
614
|
+
else:
|
|
615
|
+
libs = [libname]
|
|
616
|
+
|
|
617
|
+
for lib in libs:
|
|
618
|
+
_names = self.conn.parse_names(names, libname=lib)
|
|
619
|
+
tmintmax = pd.DataFrame(
|
|
620
|
+
index=names, columns=["tmin", "tmax"], dtype="datetime64[ns]"
|
|
621
|
+
)
|
|
622
|
+
desc = f"Get tmin/tmax {lib}"
|
|
623
|
+
for n in tqdm(_names, desc=desc) if progressbar else _names:
|
|
624
|
+
if lib == "models":
|
|
625
|
+
mld = self.conn.get_models(
|
|
626
|
+
n,
|
|
627
|
+
return_dict=True,
|
|
628
|
+
)
|
|
629
|
+
tmintmax.loc[n, "tmin"] = mld["settings"]["tmin"]
|
|
630
|
+
tmintmax.loc[n, "tmax"] = mld["settings"]["tmax"]
|
|
631
|
+
else:
|
|
632
|
+
s = (
|
|
633
|
+
self.conn.get_oseries(n)
|
|
634
|
+
if lib == "oseries"
|
|
635
|
+
else self.conn.get_stresses(n)
|
|
636
|
+
)
|
|
637
|
+
tmintmax.loc[n, "tmin"] = s.first_valid_index()
|
|
638
|
+
tmintmax.loc[n, "tmax"] = s.last_valid_index()
|
|
639
|
+
|
|
640
|
+
results[lib] = tmintmax
|
|
596
641
|
|
|
597
|
-
|
|
642
|
+
if len(results) == 1:
|
|
643
|
+
return results[lib]
|
|
644
|
+
else:
|
|
645
|
+
return pd.concat(results, axis=0)
|
|
598
646
|
|
|
599
647
|
def get_extent(self, libname, names=None, buffer=0.0):
|
|
600
648
|
"""Get extent [xmin, xmax, ymin, ymax] from library.
|
|
@@ -614,7 +662,7 @@ class PastaStore:
|
|
|
614
662
|
extent : list
|
|
615
663
|
extent [xmin, xmax, ymin, ymax]
|
|
616
664
|
"""
|
|
617
|
-
names = self.conn.
|
|
665
|
+
names = self.conn.parse_names(names, libname=libname)
|
|
618
666
|
if libname in ["oseries", "stresses"]:
|
|
619
667
|
df = getattr(self, libname)
|
|
620
668
|
elif libname == "models":
|
|
@@ -637,7 +685,7 @@ class PastaStore:
|
|
|
637
685
|
param_value: Optional[str] = "optimal",
|
|
638
686
|
progressbar: Optional[bool] = False,
|
|
639
687
|
ignore_errors: Optional[bool] = True,
|
|
640
|
-
) ->
|
|
688
|
+
) -> FrameOrSeriesUnion:
|
|
641
689
|
"""Get model parameters.
|
|
642
690
|
|
|
643
691
|
NaN-values are returned when the parameters are not present in the model or the
|
|
@@ -665,7 +713,7 @@ class PastaStore:
|
|
|
665
713
|
p : pandas.DataFrame
|
|
666
714
|
DataFrame containing the parameters (columns) per model (rows)
|
|
667
715
|
"""
|
|
668
|
-
modelnames = self.conn.
|
|
716
|
+
modelnames = self.conn.parse_names(modelnames, libname="models")
|
|
669
717
|
|
|
670
718
|
# create dataframe for results
|
|
671
719
|
p = pd.DataFrame(index=modelnames, columns=parameters)
|
|
@@ -692,8 +740,7 @@ class PastaStore:
|
|
|
692
740
|
else:
|
|
693
741
|
p.loc[mlname, c] = np.nan
|
|
694
742
|
|
|
695
|
-
|
|
696
|
-
return p.astype(float)
|
|
743
|
+
return p.astype(float).squeeze()
|
|
697
744
|
|
|
698
745
|
def get_statistics(
|
|
699
746
|
self,
|
|
@@ -704,7 +751,7 @@ class PastaStore:
|
|
|
704
751
|
ignore_errors: Optional[bool] = False,
|
|
705
752
|
fancy_output: bool = True,
|
|
706
753
|
**kwargs,
|
|
707
|
-
) ->
|
|
754
|
+
) -> FrameOrSeriesUnion:
|
|
708
755
|
"""Get model statistics.
|
|
709
756
|
|
|
710
757
|
Parameters
|
|
@@ -733,7 +780,7 @@ class PastaStore:
|
|
|
733
780
|
-------
|
|
734
781
|
s : pandas.DataFrame
|
|
735
782
|
"""
|
|
736
|
-
modelnames = self.conn.
|
|
783
|
+
modelnames = self.conn.parse_names(modelnames, libname="models")
|
|
737
784
|
|
|
738
785
|
# if statistics is str
|
|
739
786
|
if isinstance(statistics, str):
|
|
@@ -760,7 +807,7 @@ class PastaStore:
|
|
|
760
807
|
desc = "Get model statistics"
|
|
761
808
|
for mlname in tqdm(modelnames, desc=desc) if progressbar else modelnames:
|
|
762
809
|
try:
|
|
763
|
-
ml = self.get_models(mlname, progressbar=False)
|
|
810
|
+
ml = self.conn.get_models(mlname, progressbar=False)
|
|
764
811
|
except Exception as e:
|
|
765
812
|
if ignore_errors:
|
|
766
813
|
continue
|
|
@@ -770,8 +817,7 @@ class PastaStore:
|
|
|
770
817
|
value = getattr(ml.stats, stat)(**kwargs)
|
|
771
818
|
s.loc[mlname, stat] = value
|
|
772
819
|
|
|
773
|
-
|
|
774
|
-
return s.astype(float)
|
|
820
|
+
return s.astype(float).squeeze()
|
|
775
821
|
|
|
776
822
|
def create_model(
|
|
777
823
|
self,
|
|
@@ -893,11 +939,14 @@ class PastaStore:
|
|
|
893
939
|
if solve:
|
|
894
940
|
iml.solve(**kwargs)
|
|
895
941
|
if store_models:
|
|
896
|
-
self.
|
|
942
|
+
self.add_model(iml, overwrite=True)
|
|
897
943
|
else:
|
|
898
944
|
models[o] = iml
|
|
899
945
|
if len(errors) > 0:
|
|
900
|
-
|
|
946
|
+
logger.warning(
|
|
947
|
+
"Warning! Some models were not created. See errors dictionary "
|
|
948
|
+
"for more information."
|
|
949
|
+
)
|
|
901
950
|
if store_models:
|
|
902
951
|
return errors
|
|
903
952
|
else:
|
|
@@ -1208,9 +1257,7 @@ class PastaStore:
|
|
|
1208
1257
|
# override rfunc and set to HantushWellModel
|
|
1209
1258
|
rfunc = ps.HantushWellModel
|
|
1210
1259
|
|
|
1211
|
-
|
|
1212
|
-
if not PASTAS_LEQ_022 and (stressmodel._name != "WellModel"):
|
|
1213
|
-
kwargs["metadata"] = metadata
|
|
1260
|
+
kwargs["metadata"] = metadata
|
|
1214
1261
|
|
|
1215
1262
|
return stressmodel(
|
|
1216
1263
|
**stresses,
|
|
@@ -1282,12 +1329,13 @@ class PastaStore:
|
|
|
1282
1329
|
**kwargs,
|
|
1283
1330
|
)
|
|
1284
1331
|
if isinstance(ml, str):
|
|
1285
|
-
ml = self.get_model(ml)
|
|
1332
|
+
ml = self.conn.get_model(ml)
|
|
1286
1333
|
ml.add_stressmodel(sm)
|
|
1287
1334
|
self.conn.add_model(ml, overwrite=True)
|
|
1288
1335
|
logger.info(
|
|
1289
|
-
|
|
1290
|
-
|
|
1336
|
+
"Stressmodel '%s' added to model '%s' and stored in database.",
|
|
1337
|
+
sm.name,
|
|
1338
|
+
ml.name,
|
|
1291
1339
|
)
|
|
1292
1340
|
else:
|
|
1293
1341
|
ml.add_stressmodel(sm)
|
|
@@ -1344,7 +1392,7 @@ class PastaStore:
|
|
|
1344
1392
|
modelnames = kwargs.pop("mls")
|
|
1345
1393
|
logger.warning("Argument `mls` is deprecated, use `modelnames` instead.")
|
|
1346
1394
|
|
|
1347
|
-
modelnames = self.conn.
|
|
1395
|
+
modelnames = self.conn.parse_names(modelnames, libname="models")
|
|
1348
1396
|
|
|
1349
1397
|
# prepare parallel
|
|
1350
1398
|
if parallel and self.conn.conn_type == "dict":
|
|
@@ -1398,7 +1446,7 @@ class PastaStore:
|
|
|
1398
1446
|
):
|
|
1399
1447
|
solve_model(ml_name=ml_name)
|
|
1400
1448
|
|
|
1401
|
-
def check_models(self, checklist=None, modelnames=None):
|
|
1449
|
+
def check_models(self, checklist=None, modelnames=None, style_output: bool = False):
|
|
1402
1450
|
"""Check models against checklist.
|
|
1403
1451
|
|
|
1404
1452
|
Parameters
|
|
@@ -1413,6 +1461,9 @@ class PastaStore:
|
|
|
1413
1461
|
- model parameters are not on bounds
|
|
1414
1462
|
modelnames : list of str, optional
|
|
1415
1463
|
list of modelnames to perform checks on, by default None
|
|
1464
|
+
style_output : bool, optional
|
|
1465
|
+
if True, return styled dataframe with pass/fail colors,
|
|
1466
|
+
by default False
|
|
1416
1467
|
|
|
1417
1468
|
Returns
|
|
1418
1469
|
-------
|
|
@@ -1422,7 +1473,7 @@ class PastaStore:
|
|
|
1422
1473
|
if checklist is None:
|
|
1423
1474
|
checklist = ps.check.checks_brakenhoff_2022
|
|
1424
1475
|
|
|
1425
|
-
names = self.conn.
|
|
1476
|
+
names = self.conn.parse_names(modelnames, libname="models")
|
|
1426
1477
|
|
|
1427
1478
|
check_dfs = []
|
|
1428
1479
|
for n in names:
|
|
@@ -1431,7 +1482,10 @@ class PastaStore:
|
|
|
1431
1482
|
check_dfs.append(cdf)
|
|
1432
1483
|
chkdf = pd.concat(check_dfs, axis=1)
|
|
1433
1484
|
chkdf.columns.name = "models"
|
|
1434
|
-
|
|
1485
|
+
if style_output:
|
|
1486
|
+
return chkdf.style.map(boolean_styler)
|
|
1487
|
+
else:
|
|
1488
|
+
return chkdf
|
|
1435
1489
|
|
|
1436
1490
|
def to_zip(self, fname: str, overwrite=False, progressbar: bool = True):
|
|
1437
1491
|
"""Write data to zipfile.
|
|
@@ -1447,25 +1501,26 @@ class PastaStore:
|
|
|
1447
1501
|
"""
|
|
1448
1502
|
from zipfile import ZIP_DEFLATED, ZipFile
|
|
1449
1503
|
|
|
1450
|
-
|
|
1504
|
+
fname = Path(fname)
|
|
1505
|
+
if fname.exists() and not overwrite:
|
|
1451
1506
|
raise FileExistsError(
|
|
1452
1507
|
"File already exists! Use 'overwrite=True' to force writing file."
|
|
1453
1508
|
)
|
|
1454
|
-
elif
|
|
1455
|
-
warnings.warn(f"Overwriting file '{
|
|
1509
|
+
elif fname.exists():
|
|
1510
|
+
warnings.warn(f"Overwriting file '{fname.name}'", stacklevel=1)
|
|
1456
1511
|
|
|
1457
1512
|
with ZipFile(fname, "w", compression=ZIP_DEFLATED) as archive:
|
|
1458
1513
|
# oseries
|
|
1459
|
-
self.
|
|
1514
|
+
self.zip.series_to_archive(archive, "oseries", progressbar=progressbar)
|
|
1460
1515
|
# stresses
|
|
1461
|
-
self.
|
|
1516
|
+
self.zip.series_to_archive(archive, "stresses", progressbar=progressbar)
|
|
1462
1517
|
# models
|
|
1463
|
-
self.
|
|
1518
|
+
self.zip.models_to_archive(archive, progressbar=progressbar)
|
|
1464
1519
|
|
|
1465
1520
|
def export_model_series_to_csv(
|
|
1466
1521
|
self,
|
|
1467
1522
|
names: Optional[Union[list, str]] = None,
|
|
1468
|
-
exportdir: str = ".",
|
|
1523
|
+
exportdir: Path | str = ".",
|
|
1469
1524
|
exportmeta: bool = True,
|
|
1470
1525
|
): # pragma: no cover
|
|
1471
1526
|
"""Export model time series to csv files.
|
|
@@ -1480,40 +1535,39 @@ class PastaStore:
|
|
|
1480
1535
|
exportmeta : bool, optional
|
|
1481
1536
|
export metadata for all time series as csv file, default is True
|
|
1482
1537
|
"""
|
|
1483
|
-
names = self.conn.
|
|
1538
|
+
names = self.conn.parse_names(names, libname="models")
|
|
1539
|
+
exportdir = Path(exportdir)
|
|
1484
1540
|
for name in names:
|
|
1485
1541
|
mldict = self.get_models(name, return_dict=True)
|
|
1486
1542
|
|
|
1487
1543
|
oname = mldict["oseries"]["name"]
|
|
1488
1544
|
o = self.get_oseries(oname)
|
|
1489
|
-
o.to_csv(
|
|
1545
|
+
o.to_csv(exportdir / f"{oname}.csv")
|
|
1490
1546
|
|
|
1491
1547
|
if exportmeta:
|
|
1492
|
-
metalist = [self.get_metadata("oseries", oname)]
|
|
1548
|
+
metalist = [self.conn.get_metadata("oseries", oname)]
|
|
1493
1549
|
|
|
1494
1550
|
for sm in mldict["stressmodels"]:
|
|
1495
1551
|
if mldict["stressmodels"][sm]["stressmodel"] == "RechargeModel":
|
|
1496
1552
|
for istress in ["prec", "evap"]:
|
|
1497
1553
|
istress = mldict["stressmodels"][sm][istress]
|
|
1498
1554
|
stress_name = istress["name"]
|
|
1499
|
-
ts = self.get_stresses(stress_name)
|
|
1500
|
-
ts.to_csv(
|
|
1555
|
+
ts = self.conn.get_stresses(stress_name)
|
|
1556
|
+
ts.to_csv(exportdir / f"{stress_name}.csv")
|
|
1501
1557
|
if exportmeta:
|
|
1502
1558
|
tsmeta = self.get_metadata("stresses", stress_name)
|
|
1503
1559
|
metalist.append(tsmeta)
|
|
1504
1560
|
else:
|
|
1505
1561
|
for istress in mldict["stressmodels"][sm]["stress"]:
|
|
1506
1562
|
stress_name = istress["name"]
|
|
1507
|
-
ts = self.get_stresses(stress_name)
|
|
1508
|
-
ts.to_csv(
|
|
1563
|
+
ts = self.conn.get_stresses(stress_name)
|
|
1564
|
+
ts.to_csv(exportdir / f"{stress_name}.csv")
|
|
1509
1565
|
if exportmeta:
|
|
1510
|
-
tsmeta = self.get_metadata("stresses", stress_name)
|
|
1566
|
+
tsmeta = self.conn.get_metadata("stresses", stress_name)
|
|
1511
1567
|
metalist.append(tsmeta)
|
|
1512
1568
|
|
|
1513
1569
|
if exportmeta:
|
|
1514
|
-
pd.concat(metalist, axis=0).to_csv(
|
|
1515
|
-
os.path.join(exportdir, f"metadata_{name}.csv")
|
|
1516
|
-
)
|
|
1570
|
+
pd.concat(metalist, axis=0).to_csv(exportdir / f"metadata_{name}.csv")
|
|
1517
1571
|
|
|
1518
1572
|
@classmethod
|
|
1519
1573
|
def from_zip(
|
|
@@ -1598,8 +1652,8 @@ class PastaStore:
|
|
|
1598
1652
|
|
|
1599
1653
|
def search(
|
|
1600
1654
|
self,
|
|
1601
|
-
libname: str,
|
|
1602
1655
|
s: Optional[Union[list, str]] = None,
|
|
1656
|
+
libname: Optional[PastasLibs] = None,
|
|
1603
1657
|
case_sensitive: bool = True,
|
|
1604
1658
|
sort=True,
|
|
1605
1659
|
):
|
|
@@ -1621,6 +1675,16 @@ class PastaStore:
|
|
|
1621
1675
|
matches : list
|
|
1622
1676
|
list of names that match search result
|
|
1623
1677
|
"""
|
|
1678
|
+
if isinstance(s, str) and s in [None, "models", "stresses", "oseries"]:
|
|
1679
|
+
warnings.warn(
|
|
1680
|
+
"The order of arguments 's' and 'libname' has been flipped since v1.11."
|
|
1681
|
+
" Please update your code to use pstore.search(s=..., libname=...).",
|
|
1682
|
+
DeprecationWarning,
|
|
1683
|
+
stacklevel=2,
|
|
1684
|
+
)
|
|
1685
|
+
temp = str(s) # create copy
|
|
1686
|
+
s = str(libname) # create copy
|
|
1687
|
+
libname = temp
|
|
1624
1688
|
if libname == "models":
|
|
1625
1689
|
lib_names = {"models": self.model_names}
|
|
1626
1690
|
elif libname == "stresses":
|
|
@@ -1661,76 +1725,13 @@ class PastaStore:
|
|
|
1661
1725
|
result[lib] = matches
|
|
1662
1726
|
|
|
1663
1727
|
if len(result) == 1:
|
|
1664
|
-
return result[
|
|
1728
|
+
return result[libname]
|
|
1665
1729
|
else:
|
|
1666
1730
|
return result
|
|
1667
1731
|
|
|
1668
|
-
def get_model_timeseries_names(
|
|
1669
|
-
self,
|
|
1670
|
-
modelnames: Optional[Union[list, str]] = None,
|
|
1671
|
-
dropna: bool = True,
|
|
1672
|
-
progressbar: bool = True,
|
|
1673
|
-
) -> FrameorSeriesUnion:
|
|
1674
|
-
"""Get time series names contained in model.
|
|
1675
|
-
|
|
1676
|
-
Parameters
|
|
1677
|
-
----------
|
|
1678
|
-
modelnames : Optional[Union[list, str]], optional
|
|
1679
|
-
list or name of models to get time series names for,
|
|
1680
|
-
by default None which will use all modelnames
|
|
1681
|
-
dropna : bool, optional
|
|
1682
|
-
drop stresses from table if stress is not included in any
|
|
1683
|
-
model, by default True
|
|
1684
|
-
progressbar : bool, optional
|
|
1685
|
-
show progressbar, by default True
|
|
1686
|
-
|
|
1687
|
-
Returns
|
|
1688
|
-
-------
|
|
1689
|
-
structure : pandas.DataFrame
|
|
1690
|
-
returns DataFrame with oseries name per model, and a flag
|
|
1691
|
-
indicating whether a stress is contained within a time series
|
|
1692
|
-
model.
|
|
1693
|
-
"""
|
|
1694
|
-
model_names = self.conn._parse_names(modelnames, libname="models")
|
|
1695
|
-
structure = pd.DataFrame(
|
|
1696
|
-
index=model_names, columns=["oseries"] + self.stresses_names
|
|
1697
|
-
)
|
|
1698
|
-
|
|
1699
|
-
for mlnam in (
|
|
1700
|
-
tqdm(model_names, desc="Get model time series names")
|
|
1701
|
-
if progressbar
|
|
1702
|
-
else model_names
|
|
1703
|
-
):
|
|
1704
|
-
iml = self.get_models(mlnam, return_dict=True)
|
|
1705
|
-
|
|
1706
|
-
PASFILE_LEQ_022 = parse_version(
|
|
1707
|
-
iml["file_info"]["pastas_version"]
|
|
1708
|
-
) <= parse_version("0.22.0")
|
|
1709
|
-
|
|
1710
|
-
# oseries
|
|
1711
|
-
structure.loc[mlnam, "oseries"] = iml["oseries"]["name"]
|
|
1712
|
-
|
|
1713
|
-
for sm in iml["stressmodels"].values():
|
|
1714
|
-
class_key = "stressmodel" if PASFILE_LEQ_022 else "class"
|
|
1715
|
-
if sm[class_key] == "RechargeModel":
|
|
1716
|
-
pnam = sm["prec"]["name"]
|
|
1717
|
-
enam = sm["evap"]["name"]
|
|
1718
|
-
structure.loc[mlnam, pnam] = 1
|
|
1719
|
-
structure.loc[mlnam, enam] = 1
|
|
1720
|
-
elif "stress" in sm:
|
|
1721
|
-
smstress = sm["stress"]
|
|
1722
|
-
if isinstance(smstress, dict):
|
|
1723
|
-
smstress = [smstress]
|
|
1724
|
-
for s in smstress:
|
|
1725
|
-
structure.loc[mlnam, s["name"]] = 1
|
|
1726
|
-
if dropna:
|
|
1727
|
-
return structure.dropna(how="all", axis=1)
|
|
1728
|
-
else:
|
|
1729
|
-
return structure
|
|
1730
|
-
|
|
1731
1732
|
def apply(
|
|
1732
1733
|
self,
|
|
1733
|
-
libname:
|
|
1734
|
+
libname: PastasLibs,
|
|
1734
1735
|
func: callable,
|
|
1735
1736
|
names: Optional[Union[str, List[str]]] = None,
|
|
1736
1737
|
kwargs: Optional[dict] = None,
|
|
@@ -1738,6 +1739,8 @@ class PastaStore:
|
|
|
1738
1739
|
parallel: bool = False,
|
|
1739
1740
|
max_workers: Optional[int] = None,
|
|
1740
1741
|
fancy_output: bool = True,
|
|
1742
|
+
initializer: Callable = None,
|
|
1743
|
+
initargs: Optional[tuple] = None,
|
|
1741
1744
|
) -> Union[dict, pd.Series, pd.DataFrame]:
|
|
1742
1745
|
"""Apply function to items in library.
|
|
1743
1746
|
|
|
@@ -1766,6 +1769,10 @@ class PastaStore:
|
|
|
1766
1769
|
fancy_output : bool, optional
|
|
1767
1770
|
if True, try returning result as pandas Series or DataFrame, by default
|
|
1768
1771
|
False
|
|
1772
|
+
initializer : Callable, optional
|
|
1773
|
+
function to initialize each worker process, only used if parallel is True
|
|
1774
|
+
initargs : tuple, optional
|
|
1775
|
+
arguments to pass to initializer, only used if parallel is True
|
|
1769
1776
|
|
|
1770
1777
|
Returns
|
|
1771
1778
|
-------
|
|
@@ -1786,7 +1793,7 @@ class PastaStore:
|
|
|
1786
1793
|
freeze_support()
|
|
1787
1794
|
pstore.apply("models", some_func, parallel=True)
|
|
1788
1795
|
"""
|
|
1789
|
-
names = self.conn.
|
|
1796
|
+
names = self.conn.parse_names(names, libname)
|
|
1790
1797
|
if kwargs is None:
|
|
1791
1798
|
kwargs = {}
|
|
1792
1799
|
if libname not in ("oseries", "stresses", "models"):
|
|
@@ -1801,16 +1808,22 @@ class PastaStore:
|
|
|
1801
1808
|
progressbar=progressbar,
|
|
1802
1809
|
max_workers=max_workers,
|
|
1803
1810
|
chunksize=None,
|
|
1804
|
-
desc=f"
|
|
1811
|
+
desc=f"Computing {getattr(func, '__name__', '')} (parallel)",
|
|
1812
|
+
initializer=initializer,
|
|
1813
|
+
initargs=initargs,
|
|
1805
1814
|
)
|
|
1806
1815
|
else:
|
|
1807
1816
|
result = []
|
|
1808
1817
|
for n in tqdm(
|
|
1809
|
-
names,
|
|
1818
|
+
names,
|
|
1819
|
+
desc=f"Computing {getattr(func, '__name__', '')}",
|
|
1820
|
+
disable=not progressbar,
|
|
1810
1821
|
):
|
|
1811
1822
|
result.append(func(n, **kwargs))
|
|
1812
1823
|
if fancy_output:
|
|
1813
|
-
return PastaStore._fancy_output(
|
|
1824
|
+
return PastaStore._fancy_output(
|
|
1825
|
+
result, names, getattr(func, "__name__", "")
|
|
1826
|
+
)
|
|
1814
1827
|
else:
|
|
1815
1828
|
return result
|
|
1816
1829
|
|
|
@@ -1841,7 +1854,13 @@ class PastaStore:
|
|
|
1841
1854
|
if isinstance(result[0], (float, int, np.integer)):
|
|
1842
1855
|
return pd.Series(result, index=names)
|
|
1843
1856
|
elif isinstance(result[0], (pd.Series, pd.DataFrame)):
|
|
1844
|
-
|
|
1857
|
+
if isinstance(result[0], pd.Series):
|
|
1858
|
+
if names[0] == result[0].name:
|
|
1859
|
+
df = pd.concat(result, axis=1)
|
|
1860
|
+
else:
|
|
1861
|
+
df = pd.concat(dict(zip(names, result, strict=True)), axis=1)
|
|
1862
|
+
else:
|
|
1863
|
+
df = pd.concat(dict(zip(names, result, strict=True)), axis=1)
|
|
1845
1864
|
if label is not None:
|
|
1846
1865
|
df.columns.name = label
|
|
1847
1866
|
return df
|
|
@@ -1854,7 +1873,7 @@ class PastaStore:
|
|
|
1854
1873
|
self,
|
|
1855
1874
|
extent: list,
|
|
1856
1875
|
names: Optional[list[str]] = None,
|
|
1857
|
-
libname:
|
|
1876
|
+
libname: PastasLibs = "oseries",
|
|
1858
1877
|
):
|
|
1859
1878
|
"""Get names of items within extent.
|
|
1860
1879
|
|
|
@@ -1874,7 +1893,7 @@ class PastaStore:
|
|
|
1874
1893
|
list of items within extent
|
|
1875
1894
|
"""
|
|
1876
1895
|
xmin, xmax, ymin, ymax = extent
|
|
1877
|
-
names = self.conn.
|
|
1896
|
+
names = self.conn.parse_names(names, libname)
|
|
1878
1897
|
if libname == "oseries":
|
|
1879
1898
|
df = self.oseries.loc[names]
|
|
1880
1899
|
elif libname == "stresses":
|