pastastore 1.11.0__py3-none-any.whl → 1.12.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- docs/conf.py +10 -97
- pastastore/base.py +171 -37
- pastastore/connectors.py +151 -25
- pastastore/datasets.py +0 -4
- pastastore/store.py +17 -5
- pastastore/styling.py +4 -2
- pastastore/util.py +3 -0
- pastastore/validator.py +53 -3
- pastastore/version.py +2 -2
- {pastastore-1.11.0.dist-info → pastastore-1.12.1.dist-info}/METADATA +4 -4
- pastastore-1.12.1.dist-info/RECORD +31 -0
- {pastastore-1.11.0.dist-info → pastastore-1.12.1.dist-info}/WHEEL +1 -1
- tests/conftest.py +153 -51
- tests/test_003_pastastore.py +3 -1
- tests/test_009_parallel.py +393 -0
- pastastore-1.11.0.dist-info/RECORD +0 -30
- {pastastore-1.11.0.dist-info → pastastore-1.12.1.dist-info}/licenses/LICENSE +0 -0
- {pastastore-1.11.0.dist-info → pastastore-1.12.1.dist-info}/top_level.txt +0 -0
pastastore/connectors.py
CHANGED
|
@@ -7,6 +7,7 @@ import warnings
|
|
|
7
7
|
from concurrent.futures import ProcessPoolExecutor
|
|
8
8
|
from copy import deepcopy
|
|
9
9
|
from functools import partial
|
|
10
|
+
from multiprocessing import Manager
|
|
10
11
|
from pathlib import Path
|
|
11
12
|
|
|
12
13
|
# import weakref
|
|
@@ -142,7 +143,9 @@ class ArcticDBConnector(BaseConnector, ParallelUtil):
|
|
|
142
143
|
|
|
143
144
|
_conn_type = "arcticdb"
|
|
144
145
|
|
|
145
|
-
def __init__(
|
|
146
|
+
def __init__(
|
|
147
|
+
self, name: str, uri: str, verbose: bool = True, worker_process: bool = False
|
|
148
|
+
):
|
|
146
149
|
"""Create an ArcticDBConnector object using ArcticDB to store data.
|
|
147
150
|
|
|
148
151
|
Parameters
|
|
@@ -153,6 +156,9 @@ class ArcticDBConnector(BaseConnector, ParallelUtil):
|
|
|
153
156
|
URI connection string (e.g. 'lmdb://<your path here>')
|
|
154
157
|
verbose : bool, optional
|
|
155
158
|
whether to log messages when database is initialized, by default True
|
|
159
|
+
worker_process : bool, optional
|
|
160
|
+
whether the connector is created in a worker process for parallel
|
|
161
|
+
processing, by default False
|
|
156
162
|
"""
|
|
157
163
|
try:
|
|
158
164
|
import arcticdb
|
|
@@ -160,7 +166,12 @@ class ArcticDBConnector(BaseConnector, ParallelUtil):
|
|
|
160
166
|
except ModuleNotFoundError as e:
|
|
161
167
|
logger.error("Please install arcticdb with `pip install arcticdb`!")
|
|
162
168
|
raise e
|
|
163
|
-
|
|
169
|
+
|
|
170
|
+
# avoid warn on all metadata writes
|
|
171
|
+
from arcticdb_ext import set_config_string
|
|
172
|
+
|
|
173
|
+
set_config_string("PickledMetadata.LogLevel", "DEBUG")
|
|
174
|
+
|
|
164
175
|
self.uri = uri
|
|
165
176
|
self.name = name
|
|
166
177
|
|
|
@@ -172,12 +183,31 @@ class ArcticDBConnector(BaseConnector, ParallelUtil):
|
|
|
172
183
|
self.arc = arcticdb.Arctic(uri)
|
|
173
184
|
self._initialize(verbose=verbose)
|
|
174
185
|
self.models = ModelAccessor(self)
|
|
175
|
-
|
|
176
|
-
#
|
|
177
|
-
|
|
178
|
-
#
|
|
179
|
-
|
|
180
|
-
|
|
186
|
+
|
|
187
|
+
# set shared memory manager flags for parallel operations
|
|
188
|
+
# NOTE: there is no stored reference to manager object, meaning
|
|
189
|
+
# that it cannot be properly shutdown. We let the Python garbage collector
|
|
190
|
+
# do this, but the downside is there is a risk some background
|
|
191
|
+
# processes potentially continue to run.
|
|
192
|
+
mgr = Manager()
|
|
193
|
+
self._oseries_links_need_update = mgr.Value(
|
|
194
|
+
"_oseries_links_need_update",
|
|
195
|
+
False,
|
|
196
|
+
)
|
|
197
|
+
self._stresses_links_need_update = mgr.Value(
|
|
198
|
+
"_stresses_links_need_update",
|
|
199
|
+
False,
|
|
200
|
+
)
|
|
201
|
+
if not worker_process:
|
|
202
|
+
# for older versions of PastaStore, if oseries_models library is empty
|
|
203
|
+
# populate oseries - models database
|
|
204
|
+
if (self.n_models > 0) and (
|
|
205
|
+
len(self.oseries_models) == 0 or len(self.stresses_models) == 0
|
|
206
|
+
):
|
|
207
|
+
self._update_time_series_model_links(recompute=False, progressbar=True)
|
|
208
|
+
# write pstore file to store database info that can be used to load pstore
|
|
209
|
+
if "lmdb" in self.uri:
|
|
210
|
+
self.write_pstore_config_file()
|
|
181
211
|
|
|
182
212
|
def _initialize(self, verbose: bool = True) -> None:
|
|
183
213
|
"""Initialize the libraries (internal method)."""
|
|
@@ -273,8 +303,12 @@ class ArcticDBConnector(BaseConnector, ParallelUtil):
|
|
|
273
303
|
# only normalizable datatypes can be written with write, else use write_pickle
|
|
274
304
|
# normalizable: Series, DataFrames, Numpy Arrays
|
|
275
305
|
if isinstance(item, (dict, list)):
|
|
306
|
+
logger.debug(
|
|
307
|
+
"Writing pickled item '%s' to ArcticDB library '%s'.", name, libname
|
|
308
|
+
)
|
|
276
309
|
lib.write_pickle(name, item, metadata=metadata)
|
|
277
310
|
else:
|
|
311
|
+
logger.debug("Writing item '%s' to ArcticDB library '%s'.", name, libname)
|
|
278
312
|
lib.write(name, item, metadata=metadata)
|
|
279
313
|
|
|
280
314
|
def _get_item(self, libname: AllLibs, name: str) -> Union[FrameOrSeriesUnion, Dict]:
|
|
@@ -339,6 +373,8 @@ class ArcticDBConnector(BaseConnector, ParallelUtil):
|
|
|
339
373
|
max_workers: Optional[int] = None,
|
|
340
374
|
chunksize: Optional[int] = None,
|
|
341
375
|
desc: str = "",
|
|
376
|
+
initializer: Callable = None,
|
|
377
|
+
initargs: Optional[tuple] = None,
|
|
342
378
|
):
|
|
343
379
|
"""Parallel processing of function.
|
|
344
380
|
|
|
@@ -374,16 +410,24 @@ class ArcticDBConnector(BaseConnector, ParallelUtil):
|
|
|
374
410
|
chunksize for parallel processing, by default None
|
|
375
411
|
desc : str, optional
|
|
376
412
|
description for progressbar, by default ""
|
|
413
|
+
initializer : Callable, optional
|
|
414
|
+
function to initialize each worker process, by default None
|
|
415
|
+
initargs : tuple, optional
|
|
416
|
+
arguments to pass to initializer function, by default None
|
|
377
417
|
"""
|
|
378
418
|
max_workers, chunksize = self._get_max_workers_and_chunksize(
|
|
379
419
|
max_workers, len(names), chunksize
|
|
380
420
|
)
|
|
421
|
+
if initializer is None:
|
|
381
422
|
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
423
|
+
def initializer(*args):
|
|
424
|
+
# assign to module-level variable without using 'global' statement
|
|
425
|
+
globals()["conn"] = ArcticDBConnector(*args, worker_process=True)
|
|
385
426
|
|
|
386
|
-
|
|
427
|
+
initargs = (self.name, self.uri, False)
|
|
428
|
+
|
|
429
|
+
if initargs is None:
|
|
430
|
+
initargs = ()
|
|
387
431
|
|
|
388
432
|
if kwargs is None:
|
|
389
433
|
kwargs = {}
|
|
@@ -406,6 +450,10 @@ class ArcticDBConnector(BaseConnector, ParallelUtil):
|
|
|
406
450
|
result = executor.map(
|
|
407
451
|
partial(func, **kwargs), names, chunksize=chunksize
|
|
408
452
|
)
|
|
453
|
+
|
|
454
|
+
# update links if models were stored
|
|
455
|
+
self._trigger_links_update_if_needed(modelnames=names)
|
|
456
|
+
|
|
409
457
|
return result
|
|
410
458
|
|
|
411
459
|
def _list_symbols(self, libname: AllLibs) -> List[str]:
|
|
@@ -423,6 +471,11 @@ class ArcticDBConnector(BaseConnector, ParallelUtil):
|
|
|
423
471
|
"""
|
|
424
472
|
return self._get_library(libname).list_symbols()
|
|
425
473
|
|
|
474
|
+
def _item_exists(self, libname: str, name: str) -> bool:
|
|
475
|
+
"""Check if item exists without scanning directory."""
|
|
476
|
+
lib = self._get_library(libname)
|
|
477
|
+
return lib.has_symbol(name)
|
|
478
|
+
|
|
426
479
|
|
|
427
480
|
class DictConnector(BaseConnector, ParallelUtil):
|
|
428
481
|
"""DictConnector object that stores timeseries and models in dictionaries."""
|
|
@@ -447,7 +500,14 @@ class DictConnector(BaseConnector, ParallelUtil):
|
|
|
447
500
|
self.models = ModelAccessor(self)
|
|
448
501
|
# for older versions of PastaStore, if oseries_models library is empty
|
|
449
502
|
# populate oseries - models database
|
|
450
|
-
self.
|
|
503
|
+
if (self.n_models > 0) and (
|
|
504
|
+
len(self.oseries_models) == 0 or len(self.stresses_models) == 0
|
|
505
|
+
):
|
|
506
|
+
self._update_time_series_model_links(recompute=False, progressbar=True)
|
|
507
|
+
|
|
508
|
+
# delayed update flags
|
|
509
|
+
self._oseries_links_need_update = False
|
|
510
|
+
self._stresses_links_need_update = False
|
|
451
511
|
|
|
452
512
|
def _get_library(self, libname: AllLibs):
|
|
453
513
|
"""Get reference to dictionary holding data.
|
|
@@ -586,6 +646,11 @@ class DictConnector(BaseConnector, ParallelUtil):
|
|
|
586
646
|
lib = self._get_library(libname)
|
|
587
647
|
return list(lib.keys())
|
|
588
648
|
|
|
649
|
+
def _item_exists(self, libname: str, name: str) -> bool:
|
|
650
|
+
"""Check if item exists without scanning directory."""
|
|
651
|
+
lib = self._get_library(libname)
|
|
652
|
+
return name in lib
|
|
653
|
+
|
|
589
654
|
|
|
590
655
|
class PasConnector(BaseConnector, ParallelUtil):
|
|
591
656
|
"""PasConnector object that stores time series and models as JSON files on disk."""
|
|
@@ -607,7 +672,8 @@ class PasConnector(BaseConnector, ParallelUtil):
|
|
|
607
672
|
verbose : bool, optional
|
|
608
673
|
whether to print message when database is initialized, by default True
|
|
609
674
|
"""
|
|
610
|
-
#
|
|
675
|
+
# set shared memory flags for parallel processing
|
|
676
|
+
super().__init__()
|
|
611
677
|
self.name = name
|
|
612
678
|
self.parentdir = Path(path)
|
|
613
679
|
self.path = (self.parentdir / self.name).absolute()
|
|
@@ -615,9 +681,28 @@ class PasConnector(BaseConnector, ParallelUtil):
|
|
|
615
681
|
self._validator = Validator(self)
|
|
616
682
|
self._initialize(verbose=verbose)
|
|
617
683
|
self.models = ModelAccessor(self)
|
|
684
|
+
|
|
685
|
+
# set shared memory manager flags for parallel operations
|
|
686
|
+
# NOTE: there is no stored reference to manager object, meaning
|
|
687
|
+
# that it cannot be properly shutdown. We let the Python garbage collector
|
|
688
|
+
# do this, but the downside is there is a risk some background
|
|
689
|
+
# processes potentially continue to run.
|
|
690
|
+
mgr = Manager()
|
|
691
|
+
self._oseries_links_need_update = mgr.Value(
|
|
692
|
+
"_oseries_links_need_update",
|
|
693
|
+
False,
|
|
694
|
+
)
|
|
695
|
+
self._stresses_links_need_update = mgr.Value(
|
|
696
|
+
"_stresses_links_need_update",
|
|
697
|
+
False,
|
|
698
|
+
)
|
|
699
|
+
|
|
618
700
|
# for older versions of PastaStore, if oseries_models library is empty
|
|
619
701
|
# populate oseries_models library
|
|
620
|
-
self.
|
|
702
|
+
if (self.n_models > 0) and (
|
|
703
|
+
len(self.oseries_models) == 0 or len(self.stresses_models) == 0
|
|
704
|
+
):
|
|
705
|
+
self._update_time_series_model_links(recompute=False, progressbar=True)
|
|
621
706
|
# write pstore file to store database info that can be used to load pstore
|
|
622
707
|
self._write_pstore_config_file()
|
|
623
708
|
|
|
@@ -697,7 +782,12 @@ class PasConnector(BaseConnector, ParallelUtil):
|
|
|
697
782
|
if isinstance(item, pd.Series):
|
|
698
783
|
item = item.to_frame()
|
|
699
784
|
if isinstance(item, pd.DataFrame):
|
|
700
|
-
|
|
785
|
+
if type(item) is pd.DataFrame:
|
|
786
|
+
sjson = item.to_json(orient="columns")
|
|
787
|
+
else:
|
|
788
|
+
# workaround for subclasses of DataFrame that override to_json,
|
|
789
|
+
# looking at you hydropandas...
|
|
790
|
+
sjson = pd.DataFrame(item).to_json(orient="columns")
|
|
701
791
|
if name.endswith("_meta"):
|
|
702
792
|
raise ValueError(
|
|
703
793
|
"Time series name cannot end with '_meta'. "
|
|
@@ -705,23 +795,29 @@ class PasConnector(BaseConnector, ParallelUtil):
|
|
|
705
795
|
)
|
|
706
796
|
fname = lib / f"{name}.pas"
|
|
707
797
|
with fname.open("w", encoding="utf-8") as f:
|
|
798
|
+
logger.debug("Writing time series '%s' to disk at '%s'.", name, fname)
|
|
708
799
|
f.write(sjson)
|
|
709
800
|
if metadata is not None:
|
|
710
801
|
mjson = json.dumps(metadata, cls=PastasEncoder, indent=4)
|
|
711
802
|
fname_meta = lib / f"{name}_meta.pas"
|
|
712
803
|
with fname_meta.open("w", encoding="utf-8") as m:
|
|
804
|
+
logger.debug(
|
|
805
|
+
"Writing metadata '%s' to disk at '%s'.", name, fname_meta
|
|
806
|
+
)
|
|
713
807
|
m.write(mjson)
|
|
714
808
|
# pastas model dict
|
|
715
809
|
elif isinstance(item, dict):
|
|
716
810
|
jsondict = json.dumps(item, cls=PastasEncoder, indent=4)
|
|
717
811
|
fmodel = lib / f"{name}.pas"
|
|
718
812
|
with fmodel.open("w", encoding="utf-8") as fm:
|
|
813
|
+
logger.debug("Writing model '%s' to disk at '%s'.", name, fmodel)
|
|
719
814
|
fm.write(jsondict)
|
|
720
815
|
# oseries_models or stresses_models list
|
|
721
816
|
elif isinstance(item, list):
|
|
722
817
|
jsondict = json.dumps(item)
|
|
723
818
|
fname = lib / f"{name}.pas"
|
|
724
819
|
with fname.open("w", encoding="utf-8") as fm:
|
|
820
|
+
logger.debug("Writing link list '%s' to disk at '%s'.", name, fname)
|
|
725
821
|
fm.write(jsondict)
|
|
726
822
|
|
|
727
823
|
def _get_item(self, libname: AllLibs, name: str) -> Union[FrameOrSeriesUnion, Dict]:
|
|
@@ -814,6 +910,8 @@ class PasConnector(BaseConnector, ParallelUtil):
|
|
|
814
910
|
max_workers: Optional[int] = None,
|
|
815
911
|
chunksize: Optional[int] = None,
|
|
816
912
|
desc: str = "",
|
|
913
|
+
initializer: Callable = None,
|
|
914
|
+
initargs: Optional[tuple] = None,
|
|
817
915
|
):
|
|
818
916
|
"""Parallel processing of function.
|
|
819
917
|
|
|
@@ -833,6 +931,10 @@ class PasConnector(BaseConnector, ParallelUtil):
|
|
|
833
931
|
chunksize for parallel processing, by default None
|
|
834
932
|
desc : str, optional
|
|
835
933
|
description for progressbar, by default ""
|
|
934
|
+
initializer : Callable, optional
|
|
935
|
+
function to initialize each worker process, by default None
|
|
936
|
+
initargs : tuple, optional
|
|
937
|
+
arguments to pass to initializer function, by default None
|
|
836
938
|
"""
|
|
837
939
|
max_workers, chunksize = self._get_max_workers_and_chunksize(
|
|
838
940
|
max_workers, len(names), chunksize
|
|
@@ -842,20 +944,38 @@ class PasConnector(BaseConnector, ParallelUtil):
|
|
|
842
944
|
kwargs = {}
|
|
843
945
|
|
|
844
946
|
if progressbar:
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
names,
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
947
|
+
if initializer is not None:
|
|
948
|
+
result = []
|
|
949
|
+
with tqdm(total=len(names), desc=desc) as pbar:
|
|
950
|
+
with ProcessPoolExecutor(
|
|
951
|
+
max_workers=max_workers,
|
|
952
|
+
initializer=initializer,
|
|
953
|
+
initargs=initargs,
|
|
954
|
+
) as executor:
|
|
955
|
+
for item in executor.map(
|
|
956
|
+
partial(func, **kwargs), names, chunksize=chunksize
|
|
957
|
+
):
|
|
958
|
+
result.append(item)
|
|
959
|
+
pbar.update()
|
|
960
|
+
else:
|
|
961
|
+
result = process_map(
|
|
962
|
+
partial(func, **kwargs),
|
|
963
|
+
names,
|
|
964
|
+
max_workers=max_workers,
|
|
965
|
+
chunksize=chunksize,
|
|
966
|
+
desc=desc,
|
|
967
|
+
total=len(names),
|
|
968
|
+
)
|
|
853
969
|
else:
|
|
854
970
|
with ProcessPoolExecutor(max_workers=max_workers) as executor:
|
|
855
971
|
result = executor.map(
|
|
856
972
|
partial(func, **kwargs), names, chunksize=chunksize
|
|
857
973
|
)
|
|
858
|
-
|
|
974
|
+
|
|
975
|
+
# update links if models were stored
|
|
976
|
+
self._trigger_links_update_if_needed(modelnames=names)
|
|
977
|
+
|
|
978
|
+
return result
|
|
859
979
|
|
|
860
980
|
def _list_symbols(self, libname: AllLibs) -> List[str]:
|
|
861
981
|
"""List symbols in a library (internal method).
|
|
@@ -872,3 +992,9 @@ class PasConnector(BaseConnector, ParallelUtil):
|
|
|
872
992
|
"""
|
|
873
993
|
lib = self._get_library(libname)
|
|
874
994
|
return [i.stem for i in lib.glob("*.pas") if not i.stem.endswith("_meta")]
|
|
995
|
+
|
|
996
|
+
def _item_exists(self, libname: str, name: str) -> bool:
|
|
997
|
+
"""Check if item exists without scanning directory."""
|
|
998
|
+
lib = self._get_library(libname)
|
|
999
|
+
path = lib / f"{name}.pas"
|
|
1000
|
+
return path.exists()
|
pastastore/datasets.py
CHANGED
|
@@ -104,11 +104,7 @@ def example_pastastore(conn="DictConnector"):
|
|
|
104
104
|
kind="riv",
|
|
105
105
|
metadata={"x": 200_000, "y": 450_000.0},
|
|
106
106
|
)
|
|
107
|
-
# TODO: temporary fix for older version of hydropandas that does not
|
|
108
|
-
# read Menyanthes time series names correctly.
|
|
109
|
-
# multiwell notebook data
|
|
110
107
|
fname = datadir / "MenyanthesTest.men"
|
|
111
|
-
# meny = ps.read.MenyData(fname)
|
|
112
108
|
meny = ObsCollection.from_menyanthes(fname, Obs)
|
|
113
109
|
|
|
114
110
|
oseries = meny.loc["Obsevation well", "obs"]
|
pastastore/store.py
CHANGED
|
@@ -6,7 +6,7 @@ import os
|
|
|
6
6
|
import warnings
|
|
7
7
|
from functools import partial
|
|
8
8
|
from pathlib import Path
|
|
9
|
-
from typing import Dict, Iterable, List, Optional, Tuple, Union
|
|
9
|
+
from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union
|
|
10
10
|
|
|
11
11
|
import numpy as np
|
|
12
12
|
import pandas as pd
|
|
@@ -76,7 +76,7 @@ class PastaStore:
|
|
|
76
76
|
self.name = name if name is not None else self.conn.name
|
|
77
77
|
self._register_connector_methods()
|
|
78
78
|
|
|
79
|
-
# register map, plot and
|
|
79
|
+
# register map, plot, yaml and zip classes
|
|
80
80
|
self.maps = Maps(self)
|
|
81
81
|
self.plots = Plots(self)
|
|
82
82
|
self.yaml = PastastoreYAML(self)
|
|
@@ -1739,6 +1739,8 @@ class PastaStore:
|
|
|
1739
1739
|
parallel: bool = False,
|
|
1740
1740
|
max_workers: Optional[int] = None,
|
|
1741
1741
|
fancy_output: bool = True,
|
|
1742
|
+
initializer: Callable = None,
|
|
1743
|
+
initargs: Optional[tuple] = None,
|
|
1742
1744
|
) -> Union[dict, pd.Series, pd.DataFrame]:
|
|
1743
1745
|
"""Apply function to items in library.
|
|
1744
1746
|
|
|
@@ -1767,6 +1769,10 @@ class PastaStore:
|
|
|
1767
1769
|
fancy_output : bool, optional
|
|
1768
1770
|
if True, try returning result as pandas Series or DataFrame, by default
|
|
1769
1771
|
False
|
|
1772
|
+
initializer : Callable, optional
|
|
1773
|
+
function to initialize each worker process, only used if parallel is True
|
|
1774
|
+
initargs : tuple, optional
|
|
1775
|
+
arguments to pass to initializer, only used if parallel is True
|
|
1770
1776
|
|
|
1771
1777
|
Returns
|
|
1772
1778
|
-------
|
|
@@ -1802,16 +1808,22 @@ class PastaStore:
|
|
|
1802
1808
|
progressbar=progressbar,
|
|
1803
1809
|
max_workers=max_workers,
|
|
1804
1810
|
chunksize=None,
|
|
1805
|
-
desc=f"Computing {func
|
|
1811
|
+
desc=f"Computing {getattr(func, '__name__', '')} (parallel)",
|
|
1812
|
+
initializer=initializer,
|
|
1813
|
+
initargs=initargs,
|
|
1806
1814
|
)
|
|
1807
1815
|
else:
|
|
1808
1816
|
result = []
|
|
1809
1817
|
for n in tqdm(
|
|
1810
|
-
names,
|
|
1818
|
+
names,
|
|
1819
|
+
desc=f"Computing {getattr(func, '__name__', '')}",
|
|
1820
|
+
disable=not progressbar,
|
|
1811
1821
|
):
|
|
1812
1822
|
result.append(func(n, **kwargs))
|
|
1813
1823
|
if fancy_output:
|
|
1814
|
-
return PastaStore._fancy_output(
|
|
1824
|
+
return PastaStore._fancy_output(
|
|
1825
|
+
result, names, getattr(func, "__name__", "")
|
|
1826
|
+
)
|
|
1815
1827
|
else:
|
|
1816
1828
|
return result
|
|
1817
1829
|
|
pastastore/styling.py
CHANGED
|
@@ -55,16 +55,18 @@ def boolean_styler(b):
|
|
|
55
55
|
|
|
56
56
|
>>> df.style.map(boolean_styler, subset=["some column"])
|
|
57
57
|
"""
|
|
58
|
-
if b:
|
|
58
|
+
if b is True or b is np.True_:
|
|
59
59
|
return (
|
|
60
60
|
f"background-color: {rgb2hex((231 / 255, 255 / 255, 239 / 255))}; "
|
|
61
61
|
"color: darkgreen"
|
|
62
62
|
)
|
|
63
|
-
|
|
63
|
+
elif b is False or b is np.False_:
|
|
64
64
|
return (
|
|
65
65
|
f"background-color: {rgb2hex((255 / 255, 238 / 255, 238 / 255))}; "
|
|
66
66
|
"color: darkred"
|
|
67
67
|
)
|
|
68
|
+
else:
|
|
69
|
+
return "background-color: White; color: Black"
|
|
68
70
|
|
|
69
71
|
|
|
70
72
|
def boolean_row_styler(row, column):
|
pastastore/util.py
CHANGED
|
@@ -312,6 +312,7 @@ def delete_arcticdb_connector(
|
|
|
312
312
|
elif name is None or uri is None:
|
|
313
313
|
raise ValueError("Provide 'name' and 'uri' OR 'conn'!")
|
|
314
314
|
|
|
315
|
+
# connect to arcticdb
|
|
315
316
|
arc = arcticdb.Arctic(uri)
|
|
316
317
|
|
|
317
318
|
logger.info("Deleting ArcticDBConnector database: '%s' ... ", name)
|
|
@@ -566,6 +567,8 @@ def compare_models(
|
|
|
566
567
|
counter += 1
|
|
567
568
|
|
|
568
569
|
for p in ml.parameters.index:
|
|
570
|
+
df.loc[f"param: {p} (pmin)", f"model {i}"] = ml.parameters.loc[p, "pmin"]
|
|
571
|
+
df.loc[f"param: {p} (pmax)", f"model {i}"] = ml.parameters.loc[p, "pmax"]
|
|
569
572
|
df.loc[f"param: {p} (init)", f"model {i}"] = ml.parameters.loc[p, "initial"]
|
|
570
573
|
df.loc[f"param: {p} (opt)", f"model {i}"] = ml.parameters.loc[p, "optimal"]
|
|
571
574
|
|
pastastore/validator.py
CHANGED
|
@@ -56,6 +56,9 @@ class Validator:
|
|
|
56
56
|
# whether to validate time series according to pastas rules
|
|
57
57
|
USE_PASTAS_VALIDATE_SERIES = True
|
|
58
58
|
|
|
59
|
+
# whether to validate metadata keys
|
|
60
|
+
VALIDATE_METADATA = True
|
|
61
|
+
|
|
59
62
|
# protect series in models from being deleted or modified
|
|
60
63
|
PROTECT_SERIES_IN_MODELS = True
|
|
61
64
|
|
|
@@ -79,6 +82,7 @@ class Validator:
|
|
|
79
82
|
return {
|
|
80
83
|
"CHECK_MODEL_SERIES_VALUES": self.CHECK_MODEL_SERIES_VALUES,
|
|
81
84
|
"USE_PASTAS_VALIDATE_SERIES": self.USE_PASTAS_VALIDATE_SERIES,
|
|
85
|
+
"VALIDATE_METADATA": self.VALIDATE_METADATA,
|
|
82
86
|
"PROTECT_SERIES_IN_MODELS": self.PROTECT_SERIES_IN_MODELS,
|
|
83
87
|
"SERIES_EQUALITY_ABSOLUTE_TOLERANCE": (
|
|
84
88
|
self.SERIES_EQUALITY_ABSOLUTE_TOLERANCE
|
|
@@ -132,6 +136,51 @@ class Validator:
|
|
|
132
136
|
self.USE_PASTAS_VALIDATE_SERIES = b
|
|
133
137
|
logger.info("Pastas time series validation set to: %s.", b)
|
|
134
138
|
|
|
139
|
+
def set_validate_metadata(self, b: bool):
|
|
140
|
+
"""Turn VALIDATE_METADATA option on (True) or off (False).
|
|
141
|
+
|
|
142
|
+
Parameters
|
|
143
|
+
----------
|
|
144
|
+
b : bool
|
|
145
|
+
boolean indicating whether option should be turned on (True) or
|
|
146
|
+
off (False). Option is on by default.
|
|
147
|
+
"""
|
|
148
|
+
self.VALIDATE_METADATA = b
|
|
149
|
+
logger.info("Metadata validation set to: %s.", b)
|
|
150
|
+
|
|
151
|
+
def validate_metadata(self, metadata: dict):
|
|
152
|
+
"""Validate metadata.
|
|
153
|
+
|
|
154
|
+
Checks if metadata keys 'tmin', 'tmax', 'date_modified', and
|
|
155
|
+
'date_created' are valid Pandas Timestamps (or convertible to them)
|
|
156
|
+
or None.
|
|
157
|
+
|
|
158
|
+
Parameters
|
|
159
|
+
----------
|
|
160
|
+
metadata : dict
|
|
161
|
+
metadata dictionary
|
|
162
|
+
"""
|
|
163
|
+
if not self.VALIDATE_METADATA or metadata is None:
|
|
164
|
+
return
|
|
165
|
+
|
|
166
|
+
for key in ["tmin", "tmax", "date_modified", "date_created"]:
|
|
167
|
+
if key in metadata:
|
|
168
|
+
val = metadata[key]
|
|
169
|
+
if val is None:
|
|
170
|
+
continue
|
|
171
|
+
if isinstance(val, bool):
|
|
172
|
+
raise ValueError(
|
|
173
|
+
f"Metadata key '{key}' has boolean value {val}, "
|
|
174
|
+
"expected Timestamp, None, or convertible string."
|
|
175
|
+
)
|
|
176
|
+
try:
|
|
177
|
+
pd.Timestamp(val)
|
|
178
|
+
except (ValueError, TypeError) as e:
|
|
179
|
+
raise ValueError(
|
|
180
|
+
f"Metadata key '{key}' has value {val} which is "
|
|
181
|
+
f"not convertible to Timestamp: {e}"
|
|
182
|
+
) from e
|
|
183
|
+
|
|
135
184
|
def set_protect_series_in_models(self, b: bool):
|
|
136
185
|
"""Turn PROTECT_SERIES_IN_MODELS option on (True) or off (False).
|
|
137
186
|
|
|
@@ -246,7 +295,6 @@ class Validator:
|
|
|
246
295
|
"""Check if all stressmodels in the model are supported."""
|
|
247
296
|
supported_stressmodels = [
|
|
248
297
|
"StressModel",
|
|
249
|
-
"StressModel2",
|
|
250
298
|
"RechargeModel",
|
|
251
299
|
"WellModel",
|
|
252
300
|
"TarsoModel",
|
|
@@ -461,13 +509,15 @@ class Validator:
|
|
|
461
509
|
"delete model(s) first, or use force=True."
|
|
462
510
|
)
|
|
463
511
|
if libname == "oseries":
|
|
464
|
-
|
|
512
|
+
self.connector._trigger_links_update_if_needed() # trigger update if needed
|
|
513
|
+
if self.connector._item_exists("oseries_models", name):
|
|
465
514
|
n_models = len(self.connector.oseries_models[name])
|
|
466
515
|
raise SeriesUsedByModel(
|
|
467
516
|
msg.format(libname=libname, name=name, n_models=n_models)
|
|
468
517
|
)
|
|
469
518
|
elif libname == "stresses":
|
|
470
|
-
|
|
519
|
+
self.connector._trigger_links_update_if_needed() # trigger update if needed
|
|
520
|
+
if self.connector._item_exists("stresses_models", name):
|
|
471
521
|
n_models = len(self.connector.stresses_models[name])
|
|
472
522
|
raise SeriesUsedByModel(
|
|
473
523
|
msg.format(libname=libname, name=name, n_models=n_models)
|
pastastore/version.py
CHANGED
|
@@ -8,7 +8,7 @@ from packaging.version import parse as parse_version
|
|
|
8
8
|
PASTAS_VERSION = parse_version(ps.__version__)
|
|
9
9
|
PASTAS_GEQ_150 = PASTAS_VERSION >= parse_version("1.5.0")
|
|
10
10
|
|
|
11
|
-
__version__ = "1.
|
|
11
|
+
__version__ = "1.12.1"
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
def show_versions(optional=False) -> None:
|
|
@@ -31,7 +31,7 @@ def show_versions(optional=False) -> None:
|
|
|
31
31
|
msg += "\nArcticDB version : "
|
|
32
32
|
try:
|
|
33
33
|
import_module("arcticdb")
|
|
34
|
-
msg += f"{metadata.version('
|
|
34
|
+
msg += f"{metadata.version('arcticdb')}"
|
|
35
35
|
except ImportError:
|
|
36
36
|
msg += "Not Installed"
|
|
37
37
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pastastore
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.12.1
|
|
4
4
|
Summary: Tools for managing Pastas time series models.
|
|
5
5
|
Author: D.A. Brakenhoff
|
|
6
6
|
Maintainer-email: "D.A. Brakenhoff" <d.brakenhoff@artesia-water.nl>, "R. Calje" <r.calje@artesia-water.nl>, "M.A. Vonk" <m.vonk@artesia-water.nl>
|
|
@@ -46,13 +46,14 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
46
46
|
Classifier: Programming Language :: Python :: 3.13
|
|
47
47
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
48
48
|
Classifier: Topic :: Scientific/Engineering :: Hydrology
|
|
49
|
-
Requires-Python: >=3.
|
|
49
|
+
Requires-Python: >=3.11
|
|
50
50
|
Description-Content-Type: text/markdown
|
|
51
51
|
License-File: LICENSE
|
|
52
52
|
Requires-Dist: pastas>=0.13
|
|
53
53
|
Requires-Dist: tqdm>=4.36
|
|
54
54
|
Requires-Dist: pyyaml
|
|
55
55
|
Requires-Dist: colorama
|
|
56
|
+
Requires-Dist: pandas<3.0
|
|
56
57
|
Provides-Extra: full
|
|
57
58
|
Requires-Dist: pastastore[arcticdb,optional]; extra == "full"
|
|
58
59
|
Requires-Dist: hydropandas; extra == "full"
|
|
@@ -79,8 +80,7 @@ Requires-Dist: pastastore[optional]; extra == "docs"
|
|
|
79
80
|
Requires-Dist: sphinx_rtd_theme; extra == "docs"
|
|
80
81
|
Requires-Dist: Ipython; extra == "docs"
|
|
81
82
|
Requires-Dist: ipykernel; extra == "docs"
|
|
82
|
-
Requires-Dist:
|
|
83
|
-
Requires-Dist: nbsphinx_link; extra == "docs"
|
|
83
|
+
Requires-Dist: myst_nb; extra == "docs"
|
|
84
84
|
Provides-Extra: test
|
|
85
85
|
Requires-Dist: pastastore[arcticdb,optional,pytest]; extra == "test"
|
|
86
86
|
Requires-Dist: hydropandas[full]; extra == "test"
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
docs/conf.py,sha256=RFWVsGSGtqXc-N0ett9Z5XVrTKDzj3ukbXPQZM9wQIM,4186
|
|
2
|
+
pastastore/__init__.py,sha256=It_5KcVu7tMdo0QL9DXUNWtMKUDSLNfjzcuomkRdnUE,547
|
|
3
|
+
pastastore/base.py,sha256=57S57mful28-pOfpoeeMZe8vhtG2dAI6GeUOcw8J-x0,72408
|
|
4
|
+
pastastore/connectors.py,sha256=ArR053kp8ogwbtRgSCOC8f5wyMw16WNSLRzb_Kx1b6k,35291
|
|
5
|
+
pastastore/datasets.py,sha256=0ZjeXRX5uRTAbZPfAVoLjfR2sh0MDpMSatfKu7Imeg0,6007
|
|
6
|
+
pastastore/plotting.py,sha256=Zvesn7yY_z4LXdTLNT5TDui1YJ1wcm_yTvgqw6h2iQg,55697
|
|
7
|
+
pastastore/store.py,sha256=ymXP9UyXPqARlJ6lNPasucFe1qURf3bn5Sc1oBZx6xY,70004
|
|
8
|
+
pastastore/styling.py,sha256=aw7rTrllLyL574EcLqMSwNTuQaT3jpFAK92MyJzhLL8,2423
|
|
9
|
+
pastastore/typing.py,sha256=wsn50wH_LV3J6WXNpAUcRnSJppmGqYsX6kGCg49uGzY,376
|
|
10
|
+
pastastore/util.py,sha256=sytCqmtJf4CNpg6wMoo9eDBOUnn3SpDc7DY46gKuxuI,35750
|
|
11
|
+
pastastore/validator.py,sha256=Kwl0QoLCOYk1eLbBhaaZJNq0XMtsRwM2Uo1qlWu_f5w,20236
|
|
12
|
+
pastastore/version.py,sha256=Kfg-WfLu01rVG9F0K1kxod60TFH_72qHN11cnKS6jB8,1148
|
|
13
|
+
pastastore/yaml_interface.py,sha256=x05edO_N-F9ia-nn5LlKzGdGATFHaGs9rwQAGvuDrnM,30577
|
|
14
|
+
pastastore/extensions/__init__.py,sha256=pHZQha6yhq3fwsoDWvXW-lYEbUUmlfCcHMrYoK_1Hxs,505
|
|
15
|
+
pastastore/extensions/accessor.py,sha256=kftQM6dqMDoySbyTKcvmkjC5gJRp465KA18G4NVXUO0,367
|
|
16
|
+
pastastore/extensions/hpd.py,sha256=bTX3UebPKAVZikIgDfHh6xrK7sHVAT2EbiImh1e7_OE,29728
|
|
17
|
+
pastastore-1.12.1.dist-info/licenses/LICENSE,sha256=MB_6p4kXDCUsYNjslcMByBu6i7wMNRKPC36JnhzpN4o,1087
|
|
18
|
+
tests/conftest.py,sha256=5fko9oJ7vsFkxhvldHSRHa0XA6TzUgYOHZMSSn9f6ho,7943
|
|
19
|
+
tests/test_001_import.py,sha256=o31MWpMNxngnytJeEnPz_0UTcT-XbG1JqggvrWLvSUY,238
|
|
20
|
+
tests/test_002_connectors.py,sha256=o5uXrylGUiHvOAaQ-IqhM8LJ4WC1ktHBecmZ0TUreqg,9442
|
|
21
|
+
tests/test_003_pastastore.py,sha256=7Ebtk-f963b4QAOG8TAJxipM5IEpxfFuLbaG0i0MMlA,11456
|
|
22
|
+
tests/test_004_yaml.py,sha256=3hMNjb9s0S2rbmpyEjW6FDRAxfUZS_U1qoPl4wB-cCo,4440
|
|
23
|
+
tests/test_005_maps_plots.py,sha256=HVfkTQhtpONl8N8Uv39nJMrkLYOmMYy9T2AjpzYmUp4,2152
|
|
24
|
+
tests/test_006_benchmark.py,sha256=hEk2kpYKpghI7RDQaSuWkSzkF9lNphm8VgBxouVtWAU,4940
|
|
25
|
+
tests/test_007_hpdextension.py,sha256=QKXct8sBKIypAYmI7URypYPaAtCJgM464u7D3yJMNow,4304
|
|
26
|
+
tests/test_008_stressmodels.py,sha256=733fyCvuzjKcaLjvSMt5dTTLp-T4alzNJAToSxTIUug,4003
|
|
27
|
+
tests/test_009_parallel.py,sha256=aDFEcLSz9cwbgKXpbRmNwIKA2IQ3rAPLYnA-U1nDZ1o,15039
|
|
28
|
+
pastastore-1.12.1.dist-info/METADATA,sha256=1_EBH9lfjSeTCqDH7DxoKL2OOsLNG-3hprdEmUoUKK0,7696
|
|
29
|
+
pastastore-1.12.1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
30
|
+
pastastore-1.12.1.dist-info/top_level.txt,sha256=1bgyMk1p23f04RK83Jju2_YAQBwyoQD_fInxoPB4YRw,22
|
|
31
|
+
pastastore-1.12.1.dist-info/RECORD,,
|