pastastore 1.10.2__py3-none-any.whl → 1.11.0__py3-none-any.whl

pastastore/connectors.py

@@ -4,650 +4,39 @@ import json
 import logging
 import os
 import warnings
-from collections.abc import Iterable
 from concurrent.futures import ProcessPoolExecutor
 from copy import deepcopy
 from functools import partial
+from pathlib import Path
 
 # import weakref
 from typing import Callable, Dict, List, Optional, Tuple, Union
 
 import pandas as pd
-import pastas as ps
-from numpy import isin
-from packaging.version import parse as parse_version
-from pandas.testing import assert_series_equal
 from pastas.io.pas import PastasEncoder, pastas_hook
 from tqdm.auto import tqdm
 from tqdm.contrib.concurrent import process_map
 
 from pastastore.base import BaseConnector, ModelAccessor
-from pastastore.util import _custom_warning
-from pastastore.version import PASTAS_LEQ_022
+from pastastore.typing import AllLibs, FrameOrSeriesUnion, TimeSeriesLibs
+from pastastore.util import _custom_warning, metadata_from_json, series_from_json
+from pastastore.validator import Validator
 
-FrameorSeriesUnion = Union[pd.DataFrame, pd.Series]
 warnings.showwarning = _custom_warning
 
 logger = logging.getLogger(__name__)
 
+# Global connector for multiprocessing workaround
+# This is required for connectors (like ArcticDBConnector) that cannot be pickled.
+# The initializer function in _parallel() sets this global variable in each worker
+# process, allowing unpicklable connectors to be used with multiprocessing.
+# See: https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor
+# Note: Using simple None type to avoid circular import issues
+conn = None
 
-class ConnectorUtil:
-    """Mix-in class for general Connector helper functions.
 
-    Only for internal methods, and not methods that are related to CRUD operations on
-    database.
-    """
-
-    def _parse_names(
-        self,
-        names: Optional[Union[list, str]] = None,
-        libname: Optional[str] = "oseries",
-    ) -> list:
-        """Parse names kwarg, returns iterable with name(s) (internal method).
-
-        Parameters
-        ----------
-        names : Union[list, str], optional
-            str or list of str or None or 'all' (last two options
-            retrieves all names)
-        libname : str, optional
-            name of library, default is 'oseries'
-
-        Returns
-        -------
-        list
-            list of names
-        """
-        if not isinstance(names, str) and isinstance(names, Iterable):
-            return names
-        elif isinstance(names, str) and names != "all":
-            return [names]
-        elif names is None or names == "all":
-            if libname == "oseries":
-                return self.oseries_names
-            elif libname == "stresses":
-                return self.stresses_names
-            elif libname == "models":
-                return self.model_names
-            elif libname == "oseries_models":
-                return self.oseries_with_models
-            else:
-                raise ValueError(f"No library '{libname}'!")
-        else:
-            raise NotImplementedError(f"Cannot parse 'names': {names}")
-
-    @staticmethod
-    def _meta_list_to_frame(metalist: list, names: list):
-        """Convert list of metadata dictionaries to DataFrame.
-
-        Parameters
-        ----------
-        metalist : list
-            list of metadata dictionaries
-        names : list
-            list of names corresponding to data in metalist
-
-        Returns
-        -------
-        pandas.DataFrame
-            DataFrame containing overview of metadata
-        """
-        # convert to dataframe
-        if len(metalist) > 1:
-            meta = pd.DataFrame(metalist)
-            if len({"x", "y"}.difference(meta.columns)) == 0:
-                meta["x"] = meta["x"].astype(float)
-                meta["y"] = meta["y"].astype(float)
-        elif len(metalist) == 1:
-            meta = pd.DataFrame(metalist)
-        elif len(metalist) == 0:
-            meta = pd.DataFrame()
-
-        meta.index = names
-        meta.index.name = "name"
-        return meta
-
-    def _parse_model_dict(self, mdict: dict, update_ts_settings: bool = False):
-        """Parse dictionary describing pastas models (internal method).
-
-        Parameters
-        ----------
-        mdict : dict
-            dictionary describing pastas.Model
-        update_ts_settings : bool, optional
-            update stored tmin and tmax in time series settings
-            based on time series loaded from store.
-
-        Returns
-        -------
-        ml : pastas.Model
-            time series analysis model
-        """
-        PASFILE_LEQ_022 = parse_version(
-            mdict["file_info"]["pastas_version"]
-        ) <= parse_version("0.22.0")
-
-        # oseries
-        if "series" not in mdict["oseries"]:
-            name = str(mdict["oseries"]["name"])
-            if name not in self.oseries.index:
-                msg = "oseries '{}' not present in library".format(name)
-                raise LookupError(msg)
-            mdict["oseries"]["series"] = self.get_oseries(name).squeeze()
-            # update tmin/tmax from time series
-            if update_ts_settings:
-                mdict["oseries"]["settings"]["tmin"] = mdict["oseries"]["series"].index[
-                    0
-                ]
-                mdict["oseries"]["settings"]["tmax"] = mdict["oseries"]["series"].index[
-                    -1
-                ]
-
-        # StressModel, WellModel
-        for ts in mdict["stressmodels"].values():
-            if "stress" in ts.keys():
-                # WellModel
-                classkey = "stressmodel" if PASFILE_LEQ_022 else "class"
-                if ts[classkey] == "WellModel":
-                    for stress in ts["stress"]:
-                        if "series" not in stress:
-                            name = str(stress["name"])
-                            if name in self.stresses.index:
-                                stress["series"] = self.get_stresses(name).squeeze()
-                                # update tmin/tmax from time series
-                                if update_ts_settings:
-                                    stress["settings"]["tmin"] = stress["series"].index[
-                                        0
-                                    ]
-                                    stress["settings"]["tmax"] = stress["series"].index[
-                                        -1
-                                    ]
-                # StressModel
-                else:
-                    for stress in ts["stress"] if PASFILE_LEQ_022 else [ts["stress"]]:
-                        if "series" not in stress:
-                            name = str(stress["name"])
-                            if name in self.stresses.index:
-                                stress["series"] = self.get_stresses(name).squeeze()
-                                # update tmin/tmax from time series
-                                if update_ts_settings:
-                                    stress["settings"]["tmin"] = stress["series"].index[
-                                        0
-                                    ]
-                                    stress["settings"]["tmax"] = stress["series"].index[
-                                        -1
-                                    ]
-
-            # RechargeModel, TarsoModel
-            if ("prec" in ts.keys()) and ("evap" in ts.keys()):
-                for stress in [ts["prec"], ts["evap"]]:
-                    if "series" not in stress:
-                        name = str(stress["name"])
-                        if name in self.stresses.index:
-                            stress["series"] = self.get_stresses(name).squeeze()
-                            # update tmin/tmax from time series
-                            if update_ts_settings:
-                                stress["settings"]["tmin"] = stress["series"].index[0]
-                                stress["settings"]["tmax"] = stress["series"].index[-1]
-                        else:
-                            msg = "stress '{}' not present in library".format(name)
-                            raise KeyError(msg)
-
-        # hack for pcov w dtype object (when filled with NaNs on store?)
-        if "fit" in mdict:
-            if "pcov" in mdict["fit"]:
-                pcov = mdict["fit"]["pcov"]
-                if pcov.dtypes.apply(lambda dtyp: isinstance(dtyp, object)).any():
-                    mdict["fit"]["pcov"] = pcov.astype(float)
-
-        # check pastas version vs pas-file version
-        file_version = mdict["file_info"]["pastas_version"]
-
-        # check file version and pastas version
-        # if file<0.23  and pastas>=1.0 --> error
-        PASTAS_GT_023 = parse_version(ps.__version__) > parse_version("0.23.1")
-        if PASFILE_LEQ_022 and PASTAS_GT_023:
-            raise UserWarning(
-                f"This file was created with Pastas v{file_version} "
-                f"and cannot be loaded with Pastas v{ps.__version__} Please load and "
-                "save the file with Pastas 0.23 first to update the file "
-                "format."
-            )
-
-        try:
-            # pastas>=0.15.0
-            ml = ps.io.base._load_model(mdict)
-        except AttributeError:
-            # pastas<0.15.0
-            ml = ps.io.base.load_model(mdict)
-        return ml
-
-    @staticmethod
-    def _validate_input_series(series):
-        """Check if series is pandas.DataFrame or pandas.Series.
-
-        Parameters
-        ----------
-        series : object
-            object to validate
-
-        Raises
-        ------
-        TypeError
-            if object is not of type pandas.DataFrame or pandas.Series
-        """
-        if not (isinstance(series, pd.DataFrame) or isinstance(series, pd.Series)):
-            raise TypeError("Please provide pandas.DataFrame or pandas.Series!")
-        if isinstance(series, pd.DataFrame):
-            if series.columns.size > 1:
-                raise ValueError("Only DataFrames with one column are supported!")
-
-    @staticmethod
-    def _set_series_name(series, name):
-        """Set series name to match user defined name in store.
-
-        Parameters
-        ----------
-        series : pandas.Series or pandas.DataFrame
-            set name for this time series
-        name : str
-            name of the time series (used in the pastastore)
-        """
-        if isinstance(series, pd.Series):
-            series.name = name
-            # empty string on index name causes trouble when reading
-            # data from ArcticDB: TODO: check if still an issue?
-            if series.index.name == "":
-                series.index.name = None
-
-        if isinstance(series, pd.DataFrame):
-            series.columns = [name]
-            # check for hydropandas objects which are instances of DataFrame but
-            # do have a name attribute
-            if hasattr(series, "name"):
-                series.name = name
-        return series
-
-    @staticmethod
-    def _check_stressmodels_supported(ml):
-        supported_stressmodels = [
-            "StressModel",
-            "StressModel2",
-            "RechargeModel",
-            "WellModel",
-            "TarsoModel",
-            "Constant",
-            "LinearTrend",
-            "StepModel",
-        ]
-        if isinstance(ml, ps.Model):
-            smtyps = [sm._name for sm in ml.stressmodels.values()]
-        elif isinstance(ml, dict):
-            classkey = "stressmodel" if PASTAS_LEQ_022 else "class"
-            smtyps = [sm[classkey] for sm in ml["stressmodels"].values()]
-        check = isin(smtyps, supported_stressmodels)
-        if not all(check):
-            unsupported = set(smtyps) - set(supported_stressmodels)
-            raise NotImplementedError(
-                "PastaStore does not support storing models with the "
-                f"following stressmodels: {unsupported}"
-            )
-
-    @staticmethod
-    def _check_model_series_names_for_store(ml):
-        prec_evap_model = ["RechargeModel", "TarsoModel"]
-
-        if isinstance(ml, ps.Model):
-            series_names = [
-                istress.series.name
-                for sm in ml.stressmodels.values()
-                for istress in sm.stress
-            ]
-
-        elif isinstance(ml, dict):
-            # non RechargeModel, Tarsomodel, WellModel stressmodels
-            classkey = "stressmodel" if PASTAS_LEQ_022 else "class"
-            if PASTAS_LEQ_022:
-                series_names = [
-                    istress["name"]
-                    for sm in ml["stressmodels"].values()
-                    if sm[classkey] not in (prec_evap_model + ["WellModel"])
-                    for istress in sm["stress"]
-                ]
-            else:
-                series_names = [
-                    sm["stress"]["name"]
-                    for sm in ml["stressmodels"].values()
-                    if sm[classkey] not in (prec_evap_model + ["WellModel"])
-                ]
-
-            # WellModel
-            if isin(
-                ["WellModel"],
-                [i[classkey] for i in ml["stressmodels"].values()],
-            ).any():
-                series_names += [
-                    istress["name"]
-                    for sm in ml["stressmodels"].values()
-                    if sm[classkey] in ["WellModel"]
-                    for istress in sm["stress"]
-                ]
-
-            # RechargeModel, TarsoModel
-            if isin(
-                prec_evap_model,
-                [i[classkey] for i in ml["stressmodels"].values()],
-            ).any():
-                series_names += [
-                    istress["name"]
-                    for sm in ml["stressmodels"].values()
-                    if sm[classkey] in prec_evap_model
-                    for istress in [sm["prec"], sm["evap"]]
-                ]
-
-        else:
-            raise TypeError("Expected pastas.Model or dict!")
-        if len(series_names) - len(set(series_names)) > 0:
-            msg = (
-                "There are multiple stresses series with the same name! "
-                "Each series name must be unique for the PastaStore!"
-            )
-            raise ValueError(msg)
-
-    def _check_oseries_in_store(self, ml: Union[ps.Model, dict]):
-        """Check if Model oseries are contained in PastaStore (internal method).
-
-        Parameters
-        ----------
-        ml : Union[ps.Model, dict]
-            pastas Model
-        """
-        if isinstance(ml, ps.Model):
-            name = ml.oseries.name
-        elif isinstance(ml, dict):
-            name = str(ml["oseries"]["name"])
-        else:
-            raise TypeError("Expected pastas.Model or dict!")
-        if name not in self.oseries.index:
-            msg = (
-                f"Cannot add model because oseries '{name}' is not contained in store."
-            )
-            raise LookupError(msg)
-        # expensive check
-        if self.CHECK_MODEL_SERIES_VALUES and isinstance(ml, ps.Model):
-            s_org = self.get_oseries(name).squeeze().dropna()
-            if PASTAS_LEQ_022:
-                so = ml.oseries.series_original
-            else:
-                so = ml.oseries._series_original
-            try:
-                assert_series_equal(
-                    so.dropna(),
-                    s_org,
-                    atol=self.SERIES_EQUALITY_ABSOLUTE_TOLERANCE,
-                    rtol=self.SERIES_EQUALITY_RELATIVE_TOLERANCE,
-                )
-            except AssertionError as e:
-                raise ValueError(
-                    f"Cannot add model because model oseries '{name}'"
-                    " is different from stored oseries! See stacktrace for differences."
-                ) from e
-
-    def _check_stresses_in_store(self, ml: Union[ps.Model, dict]):
-        """Check if stresses time series are contained in PastaStore (internal method).
-
-        Parameters
-        ----------
-        ml : Union[ps.Model, dict]
-            pastas Model
-        """
-        prec_evap_model = ["RechargeModel", "TarsoModel"]
-        if isinstance(ml, ps.Model):
-            for sm in ml.stressmodels.values():
-                if sm._name in prec_evap_model:
-                    stresses = [sm.prec, sm.evap]
-                else:
-                    stresses = sm.stress
-                for s in stresses:
-                    if str(s.name) not in self.stresses.index:
-                        msg = (
-                            f"Cannot add model because stress '{s.name}' "
-                            "is not contained in store."
-                        )
-                        raise LookupError(msg)
-                    if self.CHECK_MODEL_SERIES_VALUES:
-                        s_org = self.get_stresses(s.name).squeeze()
-                        if PASTAS_LEQ_022:
-                            so = s.series_original
-                        else:
-                            so = s._series_original
-                        try:
-                            assert_series_equal(
-                                so,
-                                s_org,
-                                atol=self.SERIES_EQUALITY_ABSOLUTE_TOLERANCE,
-                                rtol=self.SERIES_EQUALITY_RELATIVE_TOLERANCE,
-                            )
-                        except AssertionError as e:
-                            raise ValueError(
-                                f"Cannot add model because model stress "
-                                f"'{s.name}' is different from stored stress! "
-                                "See stacktrace for differences."
-                            ) from e
-        elif isinstance(ml, dict):
-            for sm in ml["stressmodels"].values():
-                classkey = "stressmodel" if PASTAS_LEQ_022 else "class"
-                if sm[classkey] in prec_evap_model:
-                    stresses = [sm["prec"], sm["evap"]]
-                elif sm[classkey] in ["WellModel"]:
-                    stresses = sm["stress"]
-                else:
-                    stresses = sm["stress"] if PASTAS_LEQ_022 else [sm["stress"]]
-                for s in stresses:
-                    if str(s["name"]) not in self.stresses.index:
-                        msg = (
-                            f"Cannot add model because stress '{s['name']}' "
-                            "is not contained in store."
-                        )
-                        raise LookupError(msg)
-        else:
-            raise TypeError("Expected pastas.Model or dict!")
-
-    def _stored_series_to_json(
-        self,
-        libname: str,
-        names: Optional[Union[list, str]] = None,
-        squeeze: bool = True,
-        progressbar: bool = False,
-    ):
-        """Write stored series to JSON.
-
-        Parameters
-        ----------
-        libname : str
-            library name
-        names : Optional[Union[list, str]], optional
-            names of series, by default None
-        squeeze : bool, optional
-            return single entry as json string instead
-            of list, by default True
-        progressbar : bool, optional
-            show progressbar, by default False
-
-        Returns
-        -------
-        files : list or str
-            list of series converted to JSON string or single string
-            if single entry is returned and squeeze is True
-        """
-        names = self._parse_names(names, libname=libname)
-        files = []
-        for n in tqdm(names, desc=libname) if progressbar else names:
-            s = self._get_series(libname, n, progressbar=False)
-            if isinstance(s, pd.Series):
-                s = s.to_frame()
-            try:
-                sjson = s.to_json(orient="columns")
-            except ValueError as e:
-                msg = (
-                    f"DatetimeIndex of '{n}' probably contains NaT "
-                    "or duplicate timestamps!"
-                )
-                raise ValueError(msg) from e
-            files.append(sjson)
-        if len(files) == 1 and squeeze:
-            return files[0]
-        else:
-            return files
-
-    def _stored_metadata_to_json(
-        self,
-        libname: str,
-        names: Optional[Union[list, str]] = None,
-        squeeze: bool = True,
-        progressbar: bool = False,
-    ):
-        """Write metadata from stored series to JSON.
-
-        Parameters
-        ----------
-        libname : str
-            library containing series
-        names : Optional[Union[list, str]], optional
-            names to parse, by default None
-        squeeze : bool, optional
-            return single entry as json string instead of list, by default True
-        progressbar : bool, optional
-            show progressbar, by default False
-
-        Returns
-        -------
-        files : list or str
-            list of json string
-        """
-        names = self._parse_names(names, libname=libname)
-        files = []
-        for n in tqdm(names, desc=libname) if progressbar else names:
-            meta = self.get_metadata(libname, n, as_frame=False)
-            meta_json = json.dumps(meta, cls=PastasEncoder, indent=4)
-            files.append(meta_json)
-        if len(files) == 1 and squeeze:
-            return files[0]
-        else:
-            return files
-
-    def _series_to_archive(
-        self,
-        archive,
-        libname: str,
-        names: Optional[Union[list, str]] = None,
-        progressbar: bool = True,
-    ):
-        """Write DataFrame or Series to zipfile (internal method).
-
-        Parameters
-        ----------
-        archive : zipfile.ZipFile
-            reference to an archive to write data to
-        libname : str
-            name of the library to write to zipfile
-        names : str or list of str, optional
-            names of the time series to write to archive, by default None,
-            which writes all time series to archive
-        progressbar : bool, optional
-            show progressbar, by default True
-        """
-        names = self._parse_names(names, libname=libname)
-        for n in tqdm(names, desc=libname) if progressbar else names:
-            sjson = self._stored_series_to_json(
-                libname, names=n, progressbar=False, squeeze=True
-            )
-            meta_json = self._stored_metadata_to_json(
-                libname, names=n, progressbar=False, squeeze=True
-            )
-            archive.writestr(f"{libname}/{n}.pas", sjson)
-            archive.writestr(f"{libname}/{n}_meta.pas", meta_json)
-
-    def _models_to_archive(self, archive, names=None, progressbar=True):
-        """Write pastas.Model to zipfile (internal method).
-
-        Parameters
-        ----------
-        archive : zipfile.ZipFile
-            reference to an archive to write data to
-        names : str or list of str, optional
-            names of the models to write to archive, by default None,
-            which writes all models to archive
-        progressbar : bool, optional
-            show progressbar, by default True
-        """
-        names = self._parse_names(names, libname="models")
-        for n in tqdm(names, desc="models") if progressbar else names:
-            m = self.get_models(n, return_dict=True)
-            jsondict = json.dumps(m, cls=PastasEncoder, indent=4)
-            archive.writestr(f"models/{n}.pas", jsondict)
-
-    @staticmethod
-    def _series_from_json(fjson: str, squeeze: bool = True):
-        """Load time series from JSON.
-
-        Parameters
-        ----------
-        fjson : str
-            path to file
-        squeeze : bool, optional
-            squeeze time series object to obtain pandas Series
-
-        Returns
-        -------
-        s : pd.DataFrame
-            DataFrame containing time series
-        """
-        s = pd.read_json(fjson, orient="columns", precise_float=True, dtype=False)
-        if not isinstance(s.index, pd.DatetimeIndex):
-            s.index = pd.to_datetime(s.index, unit="ms")
-        s = s.sort_index()  # needed for some reason ...
-        if squeeze:
-            return s.squeeze(axis="columns")
-        return s
-
-    @staticmethod
-    def _metadata_from_json(fjson: str):
-        """Load metadata dictionary from JSON.
-
-        Parameters
-        ----------
-        fjson : str
-            path to file
-
-        Returns
-        -------
-        meta : dict
-            dictionary containing metadata
-        """
-        with open(fjson, "r") as f:
-            meta = json.load(f)
-        return meta
-
-    def _get_model_orphans(self):
-        """Get models whose oseries no longer exist in database.
-
-        Returns
-        -------
-        dict
-            dictionary with oseries names as keys and lists of model names
-            as values
-        """
-        d = {}
-        for mlnam in tqdm(self.model_names, desc="Identifying model orphans"):
-            mdict = self.get_models(mlnam, return_dict=True)
-            onam = mdict["oseries"]["name"]
-            if onam not in self.oseries_names:
-                if onam in d:
-                    d[onam] = d[onam].append(mlnam)
-                else:
-                    d[onam] = [mlnam]
-        return d
+class ParallelUtil:
+    """Mix-in class for storing parallelizable methods."""
 
     @staticmethod
     def _solve_model(
@@ -675,11 +64,11 @@ class ConnectorUtil:
             arguments are passed to the solve method.
         """
         if connector is not None:
-            conn = connector
+            _conn = connector
         else:
-            conn = globals()["conn"]
+            _conn = globals()["conn"]
 
-        ml = conn.get_models(ml_name)
+        ml = _conn.get_models(ml_name)
         m_kwargs = {}
         for key, value in kwargs.items():
             if isinstance(value, pd.Series):
@@ -693,14 +82,14 @@ class ConnectorUtil:
 
         try:
             ml.solve(report=report, **m_kwargs)
-        except Exception as e:
+        except Exception as e:  # pylint: disable=broad-except
             if ignore_solve_errors:
-                warning = "Solve error ignored for '%s': %s " % (ml.name, e)
+                warning = f"Solve error ignored for '{ml.name}': {e}"
                 logger.warning(warning)
             else:
                 raise e
-
-        conn.add_model(ml, overwrite=True)
+        # store the updated model back in the database
+        _conn.add_model(ml, overwrite=True)
 
     @staticmethod
     def _get_statistics(
@@ -717,13 +106,14 @@ class ConnectorUtil:
         of the apply method.
         """
         if connector is not None:
-            conn = connector
+            _conn = connector
         else:
-            conn = globals()["conn"]
+            _conn = globals()["conn"]
 
-        ml = conn.get_model(name)
+        ml = _conn.get_model(name)
         series = pd.Series(index=statistics, dtype=float)
         for stat in statistics:
+            # Note: ml.stats is part of pastas.Model public API
             series.loc[stat] = getattr(ml.stats, stat)(**kwargs)
         return series
 
@@ -739,15 +129,18 @@ class ConnectorUtil:
             min(32, os.cpu_count() + 4) if max_workers is None else max_workers
         )
         if chunksize is None:
-            num_chunks = max_workers * 14
+            # 14 chunks per worker balances overhead vs granularity
+            # from stackoverflow link posted in docstring.
+            CHUNKS_PER_WORKER = 14
+            num_chunks = max_workers * CHUNKS_PER_WORKER
             chunksize = max(njobs // num_chunks, 1)
         return max_workers, chunksize
 
 
-class ArcticDBConnector(BaseConnector, ConnectorUtil):
+class ArcticDBConnector(BaseConnector, ParallelUtil):
     """ArcticDBConnector object using ArcticDB to store data."""
 
-    conn_type = "arcticdb"
+    _conn_type = "arcticdb"
 
     def __init__(self, name: str, uri: str, verbose: bool = True):
         """Create an ArcticDBConnector object using ArcticDB to store data.
@@ -759,39 +152,48 @@ class ArcticDBConnector(BaseConnector, ConnectorUtil):
         uri : str
             URI connection string (e.g. 'lmdb://<your path here>')
         verbose : bool, optional
-            whether to print message when database is initialized, by default True
+            whether to log messages when database is initialized, by default True
         """
         try:
             import arcticdb
 
         except ModuleNotFoundError as e:
-            print("Please install arcticdb with `pip install arcticdb`!")
+            logger.error("Please install arcticdb with `pip install arcticdb`!")
             raise e
+        super().__init__()
         self.uri = uri
         self.name = name
 
+        # initialize validator class to check inputs
+        self._validator = Validator(self)
+
+        # create libraries
         self.libs: dict = {}
         self.arc = arcticdb.Arctic(uri)
         self._initialize(verbose=verbose)
         self.models = ModelAccessor(self)
         # for older versions of PastaStore, if oseries_models library is empty
         # populate oseries - models database
-        self._update_all_oseries_model_links()
+        self._update_time_series_model_links()
         # write pstore file to store database info that can be used to load pstore
         if "lmdb" in self.uri:
             self.write_pstore_config_file()
 
     def _initialize(self, verbose: bool = True) -> None:
         """Initialize the libraries (internal method)."""
+        if "lmdb" in self.uri.lower():  # only check for LMDB
+            self.validator.check_config_connector_type(
+                Path(self.uri.split("://")[1]) / self.name
+            )
         for libname in self._default_library_names:
             if self._library_name(libname) not in self.arc.list_libraries():
                 self.arc.create_library(self._library_name(libname))
             else:
                 if verbose:
-                    print(
-                        f"ArcticDBConnector: library "
-                        f"'{self._library_name(libname)}'"
-                        " already exists. Linking to existing library."
+                    logger.info(
+                        "ArcticDBConnector: library '%s' already exists. "
+                        "Linking to existing library.",
+                        self._library_name(libname),
                     )
             self.libs[libname] = self._get_library(libname)
 
@@ -809,20 +211,21 @@ class ArcticDBConnector(BaseConnector, ConnectorUtil):
             "uri": self.uri,
         }
         if path is None and "lmdb" in self.uri:
-            path = self.uri.split("://")[1]
+            path = Path(self.uri.split("://")[1])
         elif path is None and "lmdb" not in self.uri:
             raise ValueError("Please provide a path to write the pastastore file!")
 
-        with open(
-            os.path.join(path, f"{self.name}.pastastore"), "w", encoding="utf-8"
+        with (path / self.name / f"{self.name}.pastastore").open(
+            "w",
+            encoding="utf-8",
         ) as f:
             json.dump(config, f)
 
-    def _library_name(self, libname: str) -> str:
+    def _library_name(self, libname: AllLibs) -> str:
         """Get full library name according to ArcticDB (internal method)."""
         return ".".join([self.name, libname])
 
-    def _get_library(self, libname: str):
+    def _get_library(self, libname: AllLibs):
         """Get ArcticDB library handle.
 
         Parameters
@@ -836,13 +239,15 @@ class ArcticDBConnector(BaseConnector, ConnectorUtil):
             handle to the library
         """
         # get library handle
-        lib = self.arc.get_library(self._library_name(libname))
-        return lib
+        if libname in self.libs:
+            return self.libs[libname]
+        else:
+            return self.arc.get_library(self._library_name(libname))
 
     def _add_item(
         self,
-        libname: str,
-        item: Union[FrameorSeriesUnion, Dict],
+        libname: AllLibs,
+        item: Union[FrameOrSeriesUnion, Dict],
         name: str,
         metadata: Optional[Dict] = None,
         **_,
@@ -861,6 +266,10 @@ class ArcticDBConnector(BaseConnector, ConnectorUtil):
             dictionary containing metadata, by default None
         """
         lib = self._get_library(libname)
+
+        # check file name for illegal characters
+        name = self.validator.check_filename_illegal_chars(libname, name)
+
         # only normalizable datatypes can be written with write, else use write_pickle
         # normalizable: Series, DataFrames, Numpy Arrays
         if isinstance(item, (dict, list)):
@@ -868,7 +277,7 @@ class ArcticDBConnector(BaseConnector, ConnectorUtil):
         else:
             lib.write(name, item, metadata=metadata)
 
-    def _get_item(self, libname: str, name: str) -> Union[FrameorSeriesUnion, Dict]:
+    def _get_item(self, libname: AllLibs, name: str) -> Union[FrameOrSeriesUnion, Dict]:
         """Retrieve item from library (internal method).
 
         Parameters
@@ -886,7 +295,7 @@ class ArcticDBConnector(BaseConnector, ConnectorUtil):
         lib = self._get_library(libname)
         return lib.read(name).data
 
-    def _del_item(self, libname: str, name: str) -> None:
+    def _del_item(self, libname: AllLibs, name: str, force: bool = False) -> None:
         """Delete items (series or models) (internal method).
 
         Parameters
@@ -895,11 +304,15 @@ class ArcticDBConnector(BaseConnector, ConnectorUtil):
             name of library to delete item from
         name : str
             name of item to delete
+        force : bool, optional
+            force deletion even if series is used in models, by default False
         """
         lib = self._get_library(libname)
+        if self.validator.PROTECT_SERIES_IN_MODELS and not force:
+            self.validator.check_series_in_models(libname, name)
         lib.delete(name)
 
-    def _get_metadata(self, libname: str, name: str) -> dict:
+    def _get_metadata(self, libname: TimeSeriesLibs, name: str) -> dict:
         """Retrieve metadata for an item (internal method).
 
         Parameters
@@ -931,6 +344,20 @@ class ArcticDBConnector(BaseConnector, ConnectorUtil):
 
         Does not return results, so function must store results in database.
 
+        Note
+        ----
+        ArcticDB connection objects cannot be pickled, which is required for
+        multiprocessing. This implementation uses an initializer function that
+        creates a new ArcticDBConnector instance in each worker process and stores
+        it in the global `conn` variable. User-provided functions can access this
+        connector via the global `conn` variable.
+
+        This is the standard Python multiprocessing pattern for unpicklable objects.
+        See: https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor
+
+        For a connector that supports direct method passing (no global variable
+        required), use PasConnector instead.
+
         Parameters
         ----------
         func : function
@@ -948,13 +375,13 @@ class ArcticDBConnector(BaseConnector, ConnectorUtil):
         desc : str, optional
             description for progressbar, by default ""
         """
-        max_workers, chunksize = ConnectorUtil._get_max_workers_and_chunksize(
+        max_workers, chunksize = self._get_max_workers_and_chunksize(
             max_workers, len(names), chunksize
         )
 
         def initializer(*args):
-            global conn
-            conn = ArcticDBConnector(*args)
+            # assign to module-level variable without using 'global' statement
+            globals()["conn"] = ArcticDBConnector(*args)
 
         initargs = (self.name, self.uri, False)
 
@@ -981,49 +408,26 @@ class ArcticDBConnector(BaseConnector, ConnectorUtil):
                 )
         return result
 
-    @property
-    def oseries_names(self):
-        """List of oseries names.
-
-        Returns
-        -------
-        list
-            list of oseries in library
-        """
-        return self._get_library("oseries").list_symbols()
-
-    @property
-    def stresses_names(self):
-        """List of stresses names.
+    def _list_symbols(self, libname: AllLibs) -> List[str]:
+        """List symbols in a library (internal method).
 
-        Returns
-        -------
-        list
-            list of stresses in library
-        """
-        return self._get_library("stresses").list_symbols()
-
-    @property
-    def model_names(self):
-        """List of model names.
+        Parameters
+        ----------
+        libname : str
+            name of the library
 
         Returns
         -------
         list
-            list of models in library
+            list of symbols in the library
         """
-        return self._get_library("models").list_symbols()
+        return self._get_library(libname).list_symbols()
 
-    @property
-    def oseries_with_models(self):
-        """List of oseries with models."""
-        return self._get_library("oseries_models").list_symbols()
 
-
-class DictConnector(BaseConnector, ConnectorUtil):
+class DictConnector(BaseConnector, ParallelUtil):
     """DictConnector object that stores timeseries and models in dictionaries."""
 
-    conn_type = "dict"
+    _conn_type = "dict"
 
     def __init__(self, name: str = "pastas_db"):
         """Create DictConnector object that stores data in dictionaries.
@@ -1033,17 +437,19 @@ class DictConnector(BaseConnector, ConnectorUtil):
         name : str, optional
             user-specified name of the connector
         """
+        super().__init__()
         self.name = name
 
         # create empty dictionaries for series and models
         for val in self._default_library_names:
             setattr(self, "lib_" + val, {})
+        self._validator = Validator(self)
         self.models = ModelAccessor(self)
         # for older versions of PastaStore, if oseries_models library is empty
         # populate oseries - models database
-        self._update_all_oseries_model_links()
+        self._update_time_series_model_links()
 
-    def _get_library(self, libname: str):
+    def _get_library(self, libname: AllLibs):
         """Get reference to dictionary holding data.
 
         Parameters
@@ -1061,7 +467,7 @@ class DictConnector(BaseConnector, ConnectorUtil):
     def _add_item(
         self,
         libname: str,
-        item: Union[FrameorSeriesUnion, Dict],
+        item: Union[FrameOrSeriesUnion, Dict],
         name: str,
         metadata: Optional[Dict] = None,
         **_,
@@ -1080,12 +486,16 @@ class DictConnector(BaseConnector, ConnectorUtil):
             dictionary containing metadata, by default None
         """
         lib = self._get_library(libname)
-        if libname in ["models", "oseries_models"]:
+
+        # check file name for illegal characters
+        name = self.validator.check_filename_illegal_chars(libname, name)
+
+        if libname in ["models", "oseries_models", "stresses_models"]:
             lib[name] = item
         else:
             lib[name] = (metadata, item)
 
-    def _get_item(self, libname: str, name: str) -> Union[FrameorSeriesUnion, Dict]:
+    def _get_item(self, libname: AllLibs, name: str) -> Union[FrameOrSeriesUnion, Dict]:
         """Retrieve item from database (internal method).
 
         Parameters
@@ -1098,16 +508,18 @@ class DictConnector(BaseConnector, ConnectorUtil):
         Returns
         -------
         item : Union[FrameorSeriesUnion, Dict]
-            time series or model dictionary
+            time series or model dictionary, modifying the returned object will not
+            affect the stored data, like in a real database
         """
         lib = self._get_library(libname)
-        if libname in ["models", "oseries_models"]:
+        # deepcopy calls are needed to ensure users cannot change "stored" items
+        if libname in ["models", "oseries_models", "stresses_models"]:
             item = deepcopy(lib[name])
         else:
             item = deepcopy(lib[name][1])
         return item
 
-    def _del_item(self, libname: str, name: str) -> None:
+    def _del_item(self, libname: AllLibs, name: str, force: bool = False) -> None:
         """Delete items (series or models) (internal method).
 
         Parameters
@@ -1116,11 +528,16 @@ class DictConnector(BaseConnector, ConnectorUtil):
             name of library to delete item from
         name : str
             name of item to delete
+        force : bool, optional
+            if True, force delete item and do not perform check if series
+            is used in a model, by default False
         """
+        if self.validator.PROTECT_SERIES_IN_MODELS and not force:
+            self.validator.check_series_in_models(libname, name)
         lib = self._get_library(libname)
         _ = lib.pop(name)
 
-    def _get_metadata(self, libname: str, name: str) -> dict:
+    def _get_metadata(self, libname: TimeSeriesLibs, name: str) -> dict:
         """Read metadata (internal method).
 
         Parameters
@@ -1140,40 +557,40 @@ class DictConnector(BaseConnector, ConnectorUtil):
         return imeta
 
     def _parallel(self, *args, **kwargs) -> None:
+        """Parallel implementation method.
+
+        Raises
+        ------
+        NotImplementedError
+            DictConnector uses in-memory storage that cannot be shared across
+            processes. Use PasConnector or ArcticDBConnector for parallel operations.
+        """
         raise NotImplementedError(
             "DictConnector does not support parallel processing,"
             " use PasConnector or ArcticDBConnector."
         )
 
-    @property
-    def oseries_names(self):
-        """List of oseries names."""
-        lib = self._get_library("oseries")
-        return list(lib.keys())
-
-    @property
-    def stresses_names(self):
-        """List of stresses names."""
-        lib = self._get_library("stresses")
-        return list(lib.keys())
+    def _list_symbols(self, libname: AllLibs) -> List[str]:
+        """List symbols in a library (internal method).
 
-    @property
-    def model_names(self):
-        """List of model names."""
-        lib = self._get_library("models")
-        return list(lib.keys())
+        Parameters
+        ----------
+        libname : str
+            name of the library
 
-    @property
-    def oseries_with_models(self):
-        """List of oseries with models."""
-        lib = self._get_library("oseries_models")
+        Returns
+        -------
+        list
+            list of symbols in the library
+        """
+        lib = self._get_library(libname)
         return list(lib.keys())
 
 
-class PasConnector(BaseConnector, ConnectorUtil):
+class PasConnector(BaseConnector, ParallelUtil):
     """PasConnector object that stores time series and models as JSON files on disk."""
 
-    conn_type = "pas"
+    _conn_type = "pas"
 
     def __init__(self, name: str, path: str, verbose: bool = True):
         """Create PasConnector object that stores data as JSON files on disk.
@@ -1190,47 +607,52 @@ class PasConnector(BaseConnector, ConnectorUtil):
         verbose : bool, optional
             whether to print message when database is initialized, by default True
         """
+        # super().__init__()
         self.name = name
-        self.parentdir = path
-        self.path = os.path.abspath(os.path.join(path, self.name))
-        self.relpath = os.path.relpath(self.path)
+        self.parentdir = Path(path)
+        self.path = (self.parentdir / self.name).absolute()
+        self.relpath = os.path.relpath(self.parentdir)
+        self._validator = Validator(self)
         self._initialize(verbose=verbose)
         self.models = ModelAccessor(self)
         # for older versions of PastaStore, if oseries_models library is empty
         # populate oseries_models library
-        self._update_all_oseries_model_links()
+        self._update_time_series_model_links()
         # write pstore file to store database info that can be used to load pstore
         self._write_pstore_config_file()
 
     def _initialize(self, verbose: bool = True) -> None:
         """Initialize the libraries (internal method)."""
+        self.validator.check_config_connector_type(self.path)
         for val in self._default_library_names:
-            libdir = os.path.join(self.path, val)
-            if not os.path.exists(libdir):
+            libdir = self.path / val
+            if not libdir.exists():
                 if verbose:
-                    print(f"PasConnector: library '{val}' created in '{libdir}'")
-                os.makedirs(libdir)
+                    logger.info(
+                        "PasConnector: library '%s' created in '%s'", val, libdir
+                    )
+                libdir.mkdir(parents=True, exist_ok=False)
             else:
                 if verbose:
-                    print(
-                        f"PasConnector: library '{val}' already exists. "
-                        f"Linking to existing directory: '{libdir}'"
+                    logger.info(
+                        "PasConnector: library '%s' already exists. "
+                        "Linking to existing directory: '%s'",
+                        val,
+                        libdir,
                     )
-            setattr(self, f"lib_{val}", os.path.join(self.path, val))
+            setattr(self, f"lib_{val}", self.path / val)
 
     def _write_pstore_config_file(self):
         """Write pstore configuration file to store database info."""
         config = {
             "connector_type": self.conn_type,
             "name": self.name,
-            "path": os.path.abspath(self.parentdir),
+            "path": str(self.parentdir.absolute()),
         }
-        with open(
-            os.path.join(self.path, f"{self.name}.pastastore"), "w", encoding="utf-8"
-        ) as f:
+        with (self.path / f"{self.name}.pastastore").open("w", encoding="utf-8") as f:
             json.dump(config, f)
 
-    def _get_library(self, libname: str):
+    def _get_library(self, libname: AllLibs) -> Path:
         """Get path to directory holding data.
 
         Parameters
@@ -1243,12 +665,12 @@ class PasConnector(BaseConnector, ConnectorUtil):
         lib : str
             path to library
         """
-        return getattr(self, "lib_" + libname)
+        return Path(getattr(self, "lib_" + libname))
 
     def _add_item(
         self,
         libname: str,
-        item: Union[FrameorSeriesUnion, Dict],
+        item: Union[FrameOrSeriesUnion, Dict],
         name: str,
         metadata: Optional[Dict] = None,
         **_,
@@ -1268,33 +690,41 @@ class PasConnector(BaseConnector, ConnectorUtil):
         """
         lib = self._get_library(libname)
 
+        # check file name for illegal characters
+        name = self.validator.check_filename_illegal_chars(libname, name)
+
         # time series
         if isinstance(item, pd.Series):
             item = item.to_frame()
         if isinstance(item, pd.DataFrame):
             sjson = item.to_json(orient="columns")
-            fname = os.path.join(lib, f"{name}.pas")
-            with open(fname, "w") as f:
+            if name.endswith("_meta"):
+                raise ValueError(
+                    "Time series name cannot end with '_meta'. "
+                    "Please use a different name for your time series."
+                )
+            fname = lib / f"{name}.pas"
+            with fname.open("w", encoding="utf-8") as f:
                 f.write(sjson)
             if metadata is not None:
                 mjson = json.dumps(metadata, cls=PastasEncoder, indent=4)
-                fname_meta = os.path.join(lib, f"{name}_meta.pas")
-                with open(fname_meta, "w") as m:
+                fname_meta = lib / f"{name}_meta.pas"
+                with fname_meta.open("w", encoding="utf-8") as m:
                     m.write(mjson)
         # pastas model dict
         elif isinstance(item, dict):
             jsondict = json.dumps(item, cls=PastasEncoder, indent=4)
-            fmodel = os.path.join(lib, f"{name}.pas")
-            with open(fmodel, "w") as fm:
+            fmodel = lib / f"{name}.pas"
+            with fmodel.open("w", encoding="utf-8") as fm:
                 fm.write(jsondict)
-        # oseries_models list
+        # oseries_models or stresses_models list
         elif isinstance(item, list):
             jsondict = json.dumps(item)
-            fname = os.path.join(lib, f"{name}.pas")
-            with open(fname, "w") as fm:
+            fname = lib / f"{name}.pas"
+            with fname.open("w", encoding="utf-8") as fm:
                 fm.write(jsondict)
 
-    def _get_item(self, libname: str, name: str) -> Union[FrameorSeriesUnion, Dict]:
+    def _get_item(self, libname: AllLibs, name: str) -> Union[FrameOrSeriesUnion, Dict]:
         """Retrieve item (internal method).
 
         Parameters
@@ -1310,24 +740,24 @@ class PasConnector(BaseConnector, ConnectorUtil):
             time series or model dictionary
         """
         lib = self._get_library(libname)
-        fjson = os.path.join(lib, f"{name}.pas")
-        if not os.path.exists(fjson):
+        fjson = lib / f"{name}.pas"
+        if not fjson.exists():
             msg = f"Item '{name}' not in '{libname}' library."
             raise FileNotFoundError(msg)
         # model
         if libname == "models":
-            with open(fjson, "r") as ml_json:
+            with fjson.open("r", encoding="utf-8") as ml_json:
                 item = json.load(ml_json, object_hook=pastas_hook)
         # list of models per oseries
-        elif libname == "oseries_models":
-            with open(fjson, "r") as f:
+        elif libname in ["oseries_models", "stresses_models"]:
+            with fjson.open("r", encoding="utf-8") as f:
                 item = json.load(f)
         # time series
         else:
-            item = self._series_from_json(fjson)
+            item = series_from_json(fjson)
         return item
 
-    def _del_item(self, libname: str, name: str) -> None:
+    def _del_item(self, libname: AllLibs, name: str, force: bool = False) -> None:
         """Delete items (series or models) (internal method).
 
         Parameters
@@ -1336,18 +766,23 @@ class PasConnector(BaseConnector, ConnectorUtil):
             name of library to delete item from
         name : str
             name of item to delete
+        force : bool, optional
+            if True, force delete item and do not perform check if series
+            is used in a model, by default False
         """
         lib = self._get_library(libname)
-        os.remove(os.path.join(lib, f"{name}.pas"))
+        if self.validator.PROTECT_SERIES_IN_MODELS and not force:
+            self.validator.check_series_in_models(libname, name)
+        (lib / f"{name}.pas").unlink()
         # remove metadata for time series
-        if libname != "models":
+        if libname in ["oseries", "stresses"]:
             try:
-                os.remove(os.path.join(lib, f"{name}_meta.pas"))
+                (lib / f"{name}_meta.pas").unlink()
             except FileNotFoundError:
                 # Nothing to delete
                 pass
 
-    def _get_metadata(self, libname: str, name: str) -> dict:
+    def _get_metadata(self, libname: TimeSeriesLibs, name: str) -> dict:
         """Read metadata (internal method).
 
         Parameters
@@ -1363,9 +798,9 @@ class PasConnector(BaseConnector, ConnectorUtil):
             dictionary containing metadata
         """
         lib = self._get_library(libname)
-        mjson = os.path.join(lib, f"{name}_meta.pas")
-        if os.path.isfile(mjson):
-            imeta = self._metadata_from_json(mjson)
+        mjson = lib / f"{name}_meta.pas"
+        if mjson.is_file():
+            imeta = metadata_from_json(mjson)
         else:
             imeta = {}
         return imeta
@@ -1399,7 +834,7 @@ class PasConnector(BaseConnector, ConnectorUtil):
         desc : str, optional
             description for progressbar, by default ""
         """
-        max_workers, chunksize = ConnectorUtil._get_max_workers_and_chunksize(
+        max_workers, chunksize = self._get_max_workers_and_chunksize(
             max_workers, len(names), chunksize
         )
 
@@ -1422,36 +857,18 @@ class PasConnector(BaseConnector, ConnectorUtil):
                 )
             return result
 
-    @property
-    def oseries_names(self):
-        """List of oseries names."""
-        lib = self._get_library("oseries")
-        return [
-            i[:-4]
-            for i in os.listdir(lib)
-            if i.endswith(".pas")
-            if not i.endswith("_meta.pas")
-        ]
-
-    @property
-    def stresses_names(self):
-        """List of stresses names."""
-        lib = self._get_library("stresses")
-        return [
-            i[:-4]
-            for i in os.listdir(lib)
-            if i.endswith(".pas")
-            if not i.endswith("_meta.pas")
-        ]
-
-    @property
-    def model_names(self):
-        """List of model names."""
-        lib = self._get_library("models")
-        return [i[:-4] for i in os.listdir(lib) if i.endswith(".pas")]
-
-    @property
-    def oseries_with_models(self):
-        """List of oseries with models."""
-        lib = self._get_library("oseries_models")
-        return [i[:-4] for i in os.listdir(lib) if i.endswith(".pas")]
+    def _list_symbols(self, libname: AllLibs) -> List[str]:
+        """List symbols in a library (internal method).
+
+        Parameters
+        ----------
+        libname : str
+            name of the library
+
+        Returns
+        -------
+        list
+            list of symbols in the library
+        """
+        lib = self._get_library(libname)
+        return [i.stem for i in lib.glob("*.pas") if not i.stem.endswith("_meta")]