pastastore 1.7.1__py3-none-any.whl → 1.7.2__py3-none-any.whl

docs/conf.py

@@ -0,0 +1,215 @@
+# ruff: noqa: D100
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath("."))
+from pastastore import __version__  # noqa: E402
+
+# -- Project information -----------------------------------------------------
+
+project = "pastastore"
+copyright = "2020, D.A. Brakenhoff"
+author = "D.A. Brakenhoff"
+
+# The short X.Y version
+version = __version__
+# The full version, including alpha/beta/rc tags
+release = __version__
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.doctest",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.todo",
+    "sphinx.ext.coverage",
+    "sphinx.ext.mathjax",
+    "sphinx.ext.ifconfig",
+    "sphinx.ext.viewcode",
+    "IPython.sphinxext.ipython_console_highlighting",  # lowercase didn't work
+    "sphinx.ext.autosectionlabel",
+    "nbsphinx",
+    "nbsphinx_link",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = ".rst"
+
+# The master toctree document.
+master_doc = "index"
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = "en"
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "sphinx_rtd_theme"
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+html_theme_options = {
+    "display_version": True,
+    "prev_next_buttons_location": "bottom",
+    # 'style_external_links': False,
+    # 'vcs_pageview_mode': '',
+    # 'style_nav_header_background': 'white',
+    # Toc options
+    "collapse_navigation": False,
+    "sticky_navigation": False,
+    "navigation_depth": 4,
+    "includehidden": True,
+    "titles_only": False,
+    "github_url": "https://github.com/pastas/pastastore",
+}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = "pastastoredoc"
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #
+    "preamble": r"""\makeatletter
+\def\UTFviii@defined#1{%
+  \ifx#1\relax
+      -%
+  \else\expandafter
+    #1%
+  \fi
+}
+
+\makeatother""",
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (
+        master_doc,
+        "pastastore.tex",
+        "pastastore Documentation",
+        "D.A. Brakenhoff",
+        "manual",
+    ),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [(master_doc, "pastastore", "pastastore Documentation", [author], 1)]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (
+        master_doc,
+        "pastastore",
+        "pastastore Documentation",
+        author,
+        "pastastore",
+        "Tools for managing time series and Pastas models",
+        "Miscellaneous",
+    ),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ["search.html"]

pastastore/__init__.py

@@ -1,5 +1,5 @@
 # ruff: noqa: F401 D104
-from pastastore import connectors, extensions, styling, util
+from pastastore import connectors, styling, util
 from pastastore.connectors import (
     ArcticDBConnector,
     DictConnector,
@@ -7,3 +7,8 @@ from pastastore.connectors import (
 )
 from pastastore.store import PastaStore
 from pastastore.version import __version__, show_versions
+
+try:
+    from pastastore import extensions
+except ModuleNotFoundError:
+    print("Could not import extensions module. Update pastas to >=1.3.0!")

pastastore/base.py

@@ -15,6 +15,7 @@ import pandas as pd
 import pastas as ps
 from numpy import isin
 from packaging.version import parse as parse_version
+from pandas.testing import assert_series_equal
 from pastas.io.pas import PastasEncoder
 from tqdm.auto import tqdm
 
@@ -30,7 +31,7 @@ class BaseConnector(ABC):
 
     Class holds base logic for dealing with time series and Pastas Models. Create your
     own Connector to a data source by writing a a class that inherits from this
-    BaseConnector. Your class has to override each abstractmethod and abstractproperty.
+    BaseConnector. Your class has to override each abstractmethod and property.
     """
 
     _default_library_names = [
@@ -47,6 +48,10 @@ class BaseConnector(ABC):
     # True for pastas>=0.23.0 and False for pastas<=0.22.0
     USE_PASTAS_VALIDATE_SERIES = False if PASTAS_LEQ_022 else True
 
+    # set series equality comparison settings (using assert_series_equal)
+    SERIES_EQUALITY_ABSOLUTE_TOLERANCE = 1e-10
+    SERIES_EQUALITY_RELATIVE_TOLERANCE = 0.0
+
     def __repr__(self):
         """Representation string of the object."""
         return (
@@ -670,22 +675,27 @@ class BaseConnector(ABC):
         metadata["kind"] = kind
         self._upsert_series("stresses", series, name, metadata=metadata)
 
-    def del_models(self, names: Union[list, str]) -> None:
+    def del_models(self, names: Union[list, str], verbose: bool = True) -> None:
         """Delete model(s) from the database.
 
         Parameters
         ----------
         names : str or list of str
             name(s) of the model to delete
+        verbose : bool, optional
+            print information about deleted models, by default True
         """
-        for n in self._parse_names(names, libname="models"):
+        names = self._parse_names(names, libname="models")
+        for n in names:
             mldict = self.get_models(n, return_dict=True)
             oname = mldict["oseries"]["name"]
             self._del_item("models", n)
             self._del_oseries_model_link(oname, n)
         self._clear_cache("_modelnames_cache")
+        if verbose:
+            print(f"Deleted {len(names)} model(s) from database.")
 
-    def del_model(self, names: Union[list, str]) -> None:
+    def del_model(self, names: Union[list, str], verbose: bool = True) -> None:
         """Delete model(s) from the database.
 
         Alias for del_models().
@@ -694,10 +704,14 @@ class BaseConnector(ABC):
         ----------
         names : str or list of str
             name(s) of the model to delete
+        verbose : bool, optional
+            print information about deleted models, by default True
         """
-        self.del_models(names=names)
+        self.del_models(names=names, verbose=verbose)
 
-    def del_oseries(self, names: Union[list, str], remove_models: bool = False):
+    def del_oseries(
+        self, names: Union[list, str], remove_models: bool = False, verbose: bool = True
+    ):
         """Delete oseries from the database.
 
         Parameters
@@ -706,29 +720,38 @@ class BaseConnector(ABC):
             name(s) of the oseries to delete
         remove_models : bool, optional
             also delete models for deleted oseries, default is False
+        verbose : bool, optional
+            print information about deleted oseries, by default True
         """
         names = self._parse_names(names, libname="oseries")
         for n in names:
             self._del_item("oseries", n)
         self._clear_cache("oseries")
+        if verbose:
+            print(f"Deleted {len(names)} oseries from database.")
         # remove associated models from database
         if remove_models:
             modelnames = list(
                 chain.from_iterable([self.oseries_models.get(n, []) for n in names])
             )
-            self.del_models(modelnames)
+            self.del_models(modelnames, verbose=verbose)
 
-    def del_stress(self, names: Union[list, str]):
+    def del_stress(self, names: Union[list, str], verbose: bool = True):
         """Delete stress from the database.
 
         Parameters
         ----------
         names : str or list of str
             name(s) of the stress to delete
+        verbose : bool, optional
+            print information about deleted stresses, by default True
         """
-        for n in self._parse_names(names, libname="stresses"):
+        names = self._parse_names(names, libname="stresses")
+        for n in names:
             self._del_item("stresses", n)
         self._clear_cache("stresses")
+        if verbose:
+            print(f"Deleted {len(names)} stress(es) from database.")
 
     def _get_series(
         self,
@@ -1665,11 +1688,18 @@ class ConnectorUtil:
                 so = ml.oseries.series_original
             else:
                 so = ml.oseries._series_original
-            if not so.dropna().equals(s_org):
+            try:
+                assert_series_equal(
+                    so.dropna(),
+                    s_org,
+                    atol=self.SERIES_EQUALITY_ABSOLUTE_TOLERANCE,
+                    rtol=self.SERIES_EQUALITY_RELATIVE_TOLERANCE,
+                )
+            except AssertionError as e:
                 raise ValueError(
                     f"Cannot add model because model oseries '{name}'"
-                    " is different from stored oseries!"
-                )
+                    " is different from stored oseries! See stacktrace for differences."
+                ) from e
 
     def _check_stresses_in_store(self, ml: Union[ps.Model, dict]):
         """Check if stresses time series are contained in PastaStore (internal method).
@@ -1699,11 +1729,19 @@ class ConnectorUtil:
                             so = s.series_original
                         else:
                             so = s._series_original
-                        if not so.equals(s_org):
+                        try:
+                            assert_series_equal(
+                                so,
+                                s_org,
+                                atol=self.SERIES_EQUALITY_ABSOLUTE_TOLERANCE,
+                                rtol=self.SERIES_EQUALITY_RELATIVE_TOLERANCE,
+                            )
+                        except AssertionError as e:
                             raise ValueError(
                                 f"Cannot add model because model stress "
-                                f"'{s.name}' is different from stored stress!"
-                            )
+                                f"'{s.name}' is different from stored stress! "
+                                "See stacktrace for differences."
+                            ) from e
         elif isinstance(ml, dict):
             for sm in ml["stressmodels"].values():
                 classkey = "stressmodel" if PASTAS_LEQ_022 else "class"

pastastore/extensions/hpd.py

@@ -164,10 +164,10 @@ class HydroPandasExtension:
         metadata.pop("name", None)
         metadata.pop("meta", None)
         unit = metadata.get("unit", None)
-        if unit == "m" and unit_multiplier == 1e3:
+        if unit == "m" and np.allclose(unit_multiplier, 1e-3):
             metadata["unit"] = "mm"
         elif unit_multiplier != 1.0:
-            metadata["unit"] = f"{unit_multiplier:e}*{unit}"
+            metadata["unit"] = f"{unit_multiplier:.1e}*{unit}"
 
         source = metadata.get("source", "")
         if len(source) > 0:
@@ -199,6 +199,60 @@ class HydroPandasExtension:
         else:
             raise ValueError("libname must be 'oseries' or 'stresses'.")
 
+    def _get_tmin_tmax(self, tmin, tmax, oseries=None):
+        """Get tmin and tmax from store if not specified.
+
+        Parameters
+        ----------
+        tmin : TimeType
+            start time
+        tmax : TimeType
+            end time
+        oseries : str, optional
+            name of the observation series to get tmin/tmax for, by default None
+
+        Returns
+        -------
+        tmin, tmax : TimeType, TimeType
+            tmin and tmax
+        """
+        # get tmin/tmax if not specified
+        if tmin is None or tmax is None:
+            tmintmax = self._store.get_tmin_tmax(
+                "oseries", names=[oseries] if oseries else None
+            )
+        if tmin is None:
+            tmin = tmintmax.loc[:, "tmin"].min() - Timedelta(days=10 * 365)
+        if tmax is None:
+            tmax = tmintmax.loc[:, "tmax"].max()
+        return tmin, tmax
+
+    @staticmethod
+    def _normalize_datetime_index(obs):
+        """Normalize observation datetime index (i.e. set observation time to midnight).
+
+        Parameters
+        ----------
+        obs : pandas.Series
+            observation series to normalize
+
+        Returns
+        -------
+        hpd.Obs
+            observation series with normalized datetime index
+        """
+        if isinstance(obs, hpd.Obs):
+            metadata = {k: getattr(obs, k) for k in obs._metadata}
+        else:
+            metadata = {}
+        return obs.__class__(
+            timestep_weighted_resample(
+                obs,
+                obs.index.normalize(),
+            ).rename(obs.name),
+            **metadata,
+        )
+
     def download_knmi_precipitation(
         self,
         stns: Optional[list[int]] = None,
@@ -303,7 +357,7 @@ class HydroPandasExtension:
             variable to download, by default "RH", valid options are
             e.g. ["RD", "RH", "EV24", "T", "Q"].
         kind : str
-            kind identifier for observations, usually "prec" or "evap".
+            kind identifier for observations in pastastore, usually "prec" or "evap".
         stns : list of int/str, optional
             list of station numbers to download data for, by default None
         tmin : TimeType, optional
@@ -320,12 +374,7 @@ class HydroPandasExtension:
             if True, normalize the datetime so stress value at midnight represents
             the daily total, by default True.
         """
-        # get tmin/tmax if not specified
-        tmintmax = self._store.get_tmin_tmax("oseries")
-        if tmin is None:
-            tmin = tmintmax.loc[:, "tmin"].min() - Timedelta(days=10 * 365)
-        if tmax is None:
-            tmax = tmintmax.loc[:, "tmax"].max()
+        tmin, tmax = self._get_tmin_tmax(tmin, tmax)
 
         if stns is None:
             locations = self._store.oseries.loc[:, ["x", "y"]]
@@ -354,6 +403,155 @@ class HydroPandasExtension:
             normalize_datetime_index=normalize_datetime_index,
         )
 
+    def download_nearest_knmi_precipitation(
+        self,
+        oseries: str,
+        meteo_var: str = "RD",
+        tmin: Optional[TimeType] = None,
+        tmax: Optional[TimeType] = None,
+        unit_multiplier: float = 1e-3,
+        normalize_datetime_index: bool = True,
+        fill_missing_obs: bool = True,
+        **kwargs,
+    ):
+        """Download precipitation time series data from nearest KNMI station.
+
+        Parameters
+        ----------
+        oseries : str
+            download nearest precipitation information for this observation well
+        meteo_var : str, optional
+            variable to download, by default "RD", valid options are ["RD", "RH"].
+        tmin : TimeType
+            start time
+        tmax : TimeType
+            end time
+        unit_multiplier : float, optional
+            multiply unit by this value before saving it in the store,
+            by default 1.0 (no conversion)
+        fill_missing_obs : bool, optional
+            if True, fill missing observations by getting observations from nearest
+            station with data.
+        fill_missing_obs : bool, optional
+            if True, fill missing observations by getting observations from nearest
+            station with data.
+        """
+        self.download_nearest_knmi_meteo(
+            oseries=oseries,
+            meteo_var=meteo_var,
+            kind="prec",
+            tmin=tmin,
+            tmax=tmax,
+            unit_multiplier=unit_multiplier,
+            normalize_datetime_index=normalize_datetime_index,
+            fill_missing_obs=fill_missing_obs,
+            **kwargs,
+        )
+
+    def download_nearest_knmi_evaporation(
+        self,
+        oseries: str,
+        meteo_var: str = "EV24",
+        tmin: Optional[TimeType] = None,
+        tmax: Optional[TimeType] = None,
+        unit_multiplier: float = 1e-3,
+        normalize_datetime_index: bool = True,
+        fill_missing_obs: bool = True,
+        **kwargs,
+    ):
+        """Download evaporation time series data from nearest KNMI station.
+
+        Parameters
+        ----------
+        oseries : str
+            download nearest evaporation information for this observation well
+        meteo_var : str, optional
+            variable to download, by default "EV24", valid options are:
+            ["EV24", "penman", "hargreaves", "makkink"].
+        tmin : TimeType
+            start time
+        tmax : TimeType
+            end time
+        unit_multiplier : float, optional
+            multiply unit by this value before saving it in the store,
+            by default 1.0 (no conversion)
+        fill_missing_obs : bool, optional
+            if True, fill missing observations by getting observations from nearest
+            station with data.
+        fill_missing_obs : bool, optional
+            if True, fill missing observations by getting observations from nearest
+            station with data.
+        """
+        self.download_nearest_knmi_meteo(
+            oseries=oseries,
+            meteo_var=meteo_var,
+            kind="evap",
+            tmin=tmin,
+            tmax=tmax,
+            unit_multiplier=unit_multiplier,
+            normalize_datetime_index=normalize_datetime_index,
+            fill_missing_obs=fill_missing_obs,
+            **kwargs,
+        )
+
+    def download_nearest_knmi_meteo(
+        self,
+        oseries: str,
+        meteo_var: str,
+        kind: str,
+        tmin: Optional[TimeType] = None,
+        tmax: Optional[TimeType] = None,
+        unit_multiplier: float = 1.0,
+        normalize_datetime_index: bool = True,
+        fill_missing_obs: bool = True,
+        **kwargs,
+    ):
+        """Download meteorological data from nearest KNMI station.
+
+        Parameters
+        ----------
+        oseries : str
+            download nearest meteorological information for this observation well
+        meteo_var : str
+            meteorological variable to download, e.g. "RD", "RH", "EV24", "T", "Q"
+        kind : str
+            kind identifier for observations in pastastore, usually "prec" or "evap".
+        tmin : TimeType
+            start time
+        tmax : TimeType
+            end time
+        unit_multiplier : float, optional
+            multiply unit by this value before saving it in the store,
+            by default 1.0 (no conversion)
+        fill_missing_obs : bool, optional
+            if True, fill missing observations by getting observations from nearest
+            station with data.
+        fill_missing_obs : bool, optional
+            if True, fill missing observations by getting observations from nearest
+            station with data.
+        """
+        xy = self._store.oseries.loc[[oseries], ["x", "y"]].to_numpy()
+        # download data
+        tmin, tmax = self._get_tmin_tmax(tmin, tmax, oseries=oseries)
+        knmi = hpd.read_knmi(
+            xy=xy,
+            meteo_vars=[meteo_var],
+            starts=tmin,
+            ends=tmax,
+            fill_missing_obs=fill_missing_obs,
+            **kwargs,
+        )
+        # add to store
+        self.add_obscollection(
+            libname="stresses",
+            oc=knmi,
+            kind=kind,
+            data_column=meteo_var,
+            unit_multiplier=unit_multiplier,
+            update=False,
+            normalize_datetime_index=normalize_datetime_index,
+        )
+
     def update_knmi_meteo(
         self,
         names: Optional[List[str]] = None,
@@ -386,6 +584,17 @@ class HydroPandasExtension:
         **kwargs : dict, optional
             Additional keyword arguments to pass to `hpd.read_knmi()`
         """
+        if "source" not in self._store.stresses.columns:
+            msg = (
+                "Cannot update KNMI stresses! "
+                "KNMI stresses cannot be identified if 'source' column is not defined."
+            )
+            logger.error(msg)
+            if raise_on_error:
+                raise ValueError(msg)
+            else:
+                return
+
         if names is None:
             names = self._store.stresses.loc[
                 self._store.stresses["source"] == "KNMI"
@@ -497,32 +706,6 @@ class HydroPandasExtension:
                 if raise_on_error:
                     raise e
 
-    @staticmethod
-    def _normalize_datetime_index(obs):
-        """Normalize observation datetime index (i.e. set observation time to midnight).
-
-        Parameters
-        ----------
-        obs : pandas.Series
-            observation series to normalize
-
-        Returns
-        -------
-        hpd.Obs
-            observation series with normalized datetime index
-        """
-        if isinstance(obs, hpd.Obs):
-            metadata = {k: getattr(obs, k) for k in obs._metadata}
-        else:
-            metadata = {}
-        return obs.__class__(
-            timestep_weighted_resample(
-                obs,
-                obs.index.normalize(),
-            ).rename(obs.name),
-            **metadata,
-        )
-
     def download_bro_gmw(
         self,
         extent: Optional[List[float]] = None,