rtc-tools 2.7.3__py3-none-any.whl

rtctools/data/rtc.py

@@ -0,0 +1,228 @@
+import logging
+import os
+from collections import namedtuple
+
+import defusedxml.ElementTree as DefusedElementTree
+
+ts_ids = namedtuple("ids", "location_id parameter_id qualifier_id")
+p_ids = namedtuple("ids", "model_id location_id parameter_id")
+
+ns = {"fews": "http://www.wldelft.nl/fews", "pi": "http://www.wldelft.nl/fews/PI"}
+
+logger = logging.getLogger("rtctools")
+
+
+class DataConfig:
+    """
+    rtcDataConfig wrapper.
+
+    Used to map PI timeseries to RTC-Tools variable names.
+    """
+
+    def __init__(self, folder):
+        """
+        Parse rtcDataConfig file
+
+        :param folder: Folder in which rtcDataConfig.xml is located.
+        """
+        self.__variable_map = {}
+        self.__location_parameter_ids = {}
+        self.__parameter_map = {}
+        self.__model_parameter_ids = {}
+
+        path = os.path.join(folder, "rtcDataConfig.xml")
+        try:
+            tree = DefusedElementTree.parse(path)
+            root = tree.getroot()
+
+            timeseriess1 = root.findall("./*/fews:timeSeries", ns)
+            timeseriess2 = root.findall("./fews:timeSeries", ns)
+            timeseriess1.extend(timeseriess2)
+
+            for timeseries in timeseriess1:
+                pi_timeseries = timeseries.find("fews:PITimeSeries", ns)
+                if pi_timeseries is not None:
+                    internal_id = timeseries.get("id")
+                    external_id = self.__pi_timeseries_id(pi_timeseries, "fews")
+
+                    if internal_id in self.__location_parameter_ids:
+                        message = (
+                            "Found more than one external timeseries "
+                            "mapped to internal id {} in {}."
+                        ).format(internal_id, path)
+                        logger.error(message)
+                        raise Exception(message)
+                    elif external_id in self.__variable_map:
+                        message = (
+                            "Found more than one internal timeseries "
+                            "mapped to external id {} in {}."
+                        ).format(external_id, path)
+                        logger.error(message)
+                        raise Exception(message)
+                    else:
+                        self.__location_parameter_ids[internal_id] = (
+                            self.__pi_location_parameter_id(pi_timeseries, "fews")
+                        )
+                        self.__variable_map[external_id] = internal_id
+
+            for k in ["import", "export"]:
+                res = root.find("./fews:%s/fews:PITimeSeriesFile/fews:timeSeriesFile" % k, ns)
+                if res is not None:
+                    setattr(self, "basename_%s" % k, os.path.splitext(res.text)[0])
+
+            parameters = root.findall("./fews:parameter", ns)
+            if parameters is not None:
+                for parameter in parameters:
+                    pi_parameter = parameter.find("fews:PIParameter", ns)
+                    if pi_parameter is not None:
+                        internal_id = parameter.get("id")
+                        external_id = self.__pi_parameter_id(pi_parameter, "fews")
+
+                        if internal_id in self.__model_parameter_ids:
+                            message = (
+                                "Found more than one external parameter mapped "
+                                "to internal id {} in {}."
+                            ).format(internal_id, path)
+                            logger.error(message)
+                            raise Exception(message)
+                        if external_id in self.__parameter_map:
+                            message = (
+                                "Found more than one interal parameter mapped to external "
+                                "modelId {}, locationId {}, parameterId {} in {}."
+                            ).format(
+                                external_id.model_id,
+                                external_id.location_id,
+                                external_id.parameter_id,
+                                path,
+                            )
+                            logger.error(message)
+                            raise Exception(message)
+                        else:
+                            self.__model_parameter_ids[internal_id] = self.__pi_model_parameter_id(
+                                pi_parameter, "fews"
+                            )
+                            self.__parameter_map[external_id] = internal_id
+
+        except IOError:
+            logger.error('No rtcDataConfig.xml file was found in "{}".'.format(folder))
+            raise
+
+    def __pi_timeseries_id(self, el, namespace):
+        location_id = el.find(namespace + ":locationId", ns).text
+        parameter_id = el.find(namespace + ":parameterId", ns).text
+
+        timeseries_id = location_id + ":" + parameter_id
+
+        qualifiers = el.findall(namespace + ":qualifierId", ns)
+        qualifier_ids = []
+        for qualifier in qualifiers:
+            qualifier_ids.append(qualifier.text)
+
+        if len(qualifier_ids) > 0:
+            qualifier_ids.sort()
+
+            return timeseries_id + ":" + ":".join(qualifier_ids)
+        else:
+            return timeseries_id
+
+    def __pi_location_parameter_id(self, el, namespace):
+        qualifier_ids = []
+        qualifiers = el.findall(namespace + ":qualifierId", ns)
+        for qualifier in qualifiers:
+            qualifier_ids.append(qualifier.text)
+
+        location_parameter_ids = ts_ids(
+            location_id=el.find(namespace + ":locationId", ns).text,
+            parameter_id=el.find(namespace + ":parameterId", ns).text,
+            qualifier_id=qualifier_ids,
+        )
+        return location_parameter_ids
+
+    def __pi_parameter_id(self, el, namespace):
+        model_id = el.find(namespace + ":modelId", ns).text
+        location_id = el.find(namespace + ":locationId", ns).text
+        parameter_id = el.find(namespace + ":parameterId", ns).text
+
+        return self.__long_parameter_id(parameter_id, location_id, model_id)
+
+    def __pi_model_parameter_id(self, el, namespace):
+        model_id = el.find(namespace + ":modelId", ns).text
+        location_id = el.find(namespace + ":locationId", ns).text
+        parameter_id = el.find(namespace + ":parameterId", ns).text
+
+        model_parameter_ids = p_ids(
+            model_id=(model_id if model_id is not None else ""),
+            location_id=(location_id if location_id is not None else ""),
+            parameter_id=(parameter_id if parameter_id is not None else ""),
+        )
+
+        return model_parameter_ids
+
+    def __long_parameter_id(self, parameter_id, location_id=None, model_id=None):
+        """
+        Convert a model, location and parameter combination to a single parameter id
+        of the form model:location:parameter.
+        """
+        if location_id is not None:
+            parameter_id = location_id + ":" + parameter_id
+        if model_id is not None:
+            parameter_id = model_id + ":" + parameter_id
+        return parameter_id
+
+    def variable(self, pi_header):
+        """
+        Map a PI timeseries header to an RTC-Tools timeseries ID.
+
+        :param pi_header: XML ElementTree node containing a PI timeseries header.
+
+        :returns: A timeseries ID.
+        :rtype: string
+        """
+        series_id = self.__pi_timeseries_id(pi_header, "pi")
+        try:
+            return self.__variable_map[series_id]
+        except KeyError:
+            return series_id
+
+    def pi_variable_ids(self, variable):
+        """
+        Map an RTC-Tools timeseries ID to a named tuple of location, parameter
+        and qualifier ID's.
+
+        :param variable: A timeseries ID.
+
+        :returns: A named tuple with fields location_id, parameter_id and qualifier_id.
+        :rtype: namedtuple
+        :raises KeyError: If the timeseries ID has no mapping in rtcDataConfig.
+        """
+        return self.__location_parameter_ids[variable]
+
+    def parameter(self, parameter_id, location_id=None, model_id=None):
+        """
+        Map a combination of parameter ID, location ID, model ID to an
+        RTC-Tools parameter ID.
+
+        :param parameter_id: String with parameter ID
+        :param location_id: String with location ID
+        :param model_id: String with model ID
+
+        :returns: A parameter ID.
+        :rtype: string
+        :raises KeyError: If the combination has no mapping in rtcDataConfig.
+        """
+        parameter_id_long = self.__long_parameter_id(parameter_id, location_id, model_id)
+
+        return self.__parameter_map[parameter_id_long]
+
+    def pi_parameter_ids(self, parameter):
+        """
+        Map an RTC-Tools model parameter ID to a named tuple of model, location
+        and parameter ID's.
+
+        :param parameter: A model parameter ID.
+
+        :returns: A named tuple with fields model_id, location_id and parameter_id.
+        :rtype: namedtuple
+        :raises KeyError: If the parameter ID has no mapping in rtcDataConfig.
+        """
+        return self.__model_parameter_ids[parameter]

rtctools/data/storage.py

@@ -0,0 +1,343 @@
+import logging
+from abc import ABCMeta, abstractmethod
+from datetime import datetime, timedelta
+from typing import Iterable, List, Tuple, Union
+
+import numpy as np
+
+from rtctools._internal.alias_tools import AliasDict, AliasRelation
+
+logger = logging.getLogger("rtctools")
+
+
+class DataStoreAccessor(metaclass=ABCMeta):
+    """
+    Base class for all problems.
+    Adds an internal data store where timeseries and parameters can be stored.
+    Access to the internal data store is always done through the io accessor.
+
+    :cvar timeseries_import_basename:
+        Import file basename. Default is ``timeseries_import``.
+    :cvar timeseries_export_basename:
+        Export file basename. Default is ``timeseries_export``.
+    """
+
+    #: Import file basename
+    timeseries_import_basename = "timeseries_import"
+    #: Export file basename
+    timeseries_export_basename = "timeseries_export"
+
+    def __init__(self, **kwargs):
+        # Save arguments
+        self._input_folder = kwargs["input_folder"] if "input_folder" in kwargs else "input"
+        self._output_folder = kwargs["output_folder"] if "output_folder" in kwargs else "output"
+
+        if logger.getEffectiveLevel() == logging.DEBUG:
+            logger.debug("Expecting input files to be located in '" + self._input_folder + "'.")
+            logger.debug("Writing output files to '" + self._output_folder + "'.")
+
+        self.io = DataStore(self)
+
+    @property
+    @abstractmethod
+    def alias_relation(self) -> AliasRelation:
+        raise NotImplementedError
+
+
+class DataStore:
+    """
+    DataStore class used by the DataStoreAccessor.
+    Contains all methods needed to access the internal data store.
+    """
+
+    def __init__(self, accessor):
+        self.__accessor = accessor
+
+        # Should all be set by subclass via setters
+        self.__reference_datetime = None
+        self.__timeseries_datetimes = None
+        self.__timeseries_times_sec = None
+        self.__timeseries_values = [AliasDict(self.__accessor.alias_relation)]
+        self.__parameters = [AliasDict(self.__accessor.alias_relation)]
+
+        self.__reference_datetime_fixed = False
+
+        self.__ensemble_size = 1
+
+    @property
+    def reference_datetime(self):
+        return self.__reference_datetime
+
+    @reference_datetime.setter
+    def reference_datetime(self, value):
+        if self.__reference_datetime_fixed and value != self.__reference_datetime:
+            raise RuntimeError(
+                "Cannot change reference datetime after times in seconds has been requested."
+            )
+        self.__reference_datetime = value
+
+    @property
+    def ensemble_size(self):
+        return self.__ensemble_size
+
+    def __update_ensemble_size(self, ensemble_size):
+        while ensemble_size > len(self.__timeseries_values):
+            self.__timeseries_values.append(AliasDict(self.__accessor.alias_relation))
+
+        while ensemble_size > len(self.__parameters):
+            self.__parameters.append(AliasDict(self.__accessor.alias_relation))
+
+        assert len(self.__parameters) == len(self.__timeseries_values)
+        assert len(self.__parameters) == ensemble_size
+
+        self.__ensemble_size = ensemble_size
+
+    @property
+    def datetimes(self) -> List[datetime]:
+        """
+        Returns the timeseries times in seconds.
+
+        :returns: timeseries datetimes, or None if there has been no call
+                  to :py:meth:`set_timeseries`.
+        """
+        return self.__timeseries_datetimes.copy()
+
+    @property
+    def times_sec(self) -> np.ndarray:
+        """
+        Returns the timeseries times in seconds.
+
+        Note that once this method is called, it is no longer allowed to
+        change :py:attr:`reference_datetime`.
+
+        :returns: timeseries times in seconds.
+        """
+        self._datetimes_to_seconds()
+
+        return self.__timeseries_times_sec
+
+    def _datetimes_to_seconds(self):
+        if self.__reference_datetime_fixed:
+            pass
+        else:
+            # Currently we only allow a reference datetime that exists in the
+            # timeseries datetimes. That way we can guarantee that we have
+            # "0.0" as one of our times in seconds. This restriction may be
+            # loosened in the future.
+            if self.reference_datetime not in self.__timeseries_datetimes:
+                raise Exception(
+                    "Reference datetime {} should be equal to "
+                    "one of the timeseries datetimes {}".format(
+                        self.reference_datetime, self.__timeseries_datetimes
+                    )
+                )
+            self.__timeseries_times_sec = self.datetime_to_sec(
+                self.__timeseries_datetimes, self.reference_datetime
+            )
+            self.__timeseries_times_sec.flags.writeable = False
+            self.__reference_datetime_fixed = True
+
+    def set_timeseries(
+        self,
+        variable: str,
+        datetimes: Iterable[datetime],
+        values: np.ndarray,
+        ensemble_member: int = 0,
+        check_duplicates: bool = False,
+    ) -> None:
+        """
+        Stores input time series values in the internal data store.
+
+        :param variable:         Variable name.
+        :param datetimes:        Times as datetime objects.
+        :param values:           The values to be stored.
+        :param ensemble_member:  The ensemble member index.
+        :param check_duplicates: If True, a warning will be given when overwriting values.
+                                 If False, existing values are silently overwritten with new values.
+        """
+        datetimes = list(datetimes)
+
+        if not isinstance(datetimes[0], datetime):
+            raise TypeError("DateStore.set_timeseries() only support datetimes")
+
+        if self.__timeseries_datetimes is not None and datetimes != self.__timeseries_datetimes:
+            raise RuntimeError(
+                "Attempting to overwrite the input time series datetimes with different values. "
+                "Please ensure all input time series have the same datetimes."
+            )
+        self.__timeseries_datetimes = datetimes
+
+        if len(self.__timeseries_datetimes) != len(values):
+            raise ValueError(
+                "Length of values ({}) must be the same as length of datetimes ({})".format(
+                    len(values), len(self.__timeseries_datetimes)
+                )
+            )
+
+        if ensemble_member >= self.__ensemble_size:
+            self.__update_ensemble_size(ensemble_member + 1)
+
+        if check_duplicates and variable in self.__timeseries_values[ensemble_member].keys():
+            logger.warning(
+                "Time series values for ensemble member {} and variable {} set twice. "
+                "Overwriting old values.".format(ensemble_member, variable)
+            )
+
+        self.__timeseries_values[ensemble_member][variable] = values
+
+    def get_timeseries(
+        self, variable: str, ensemble_member: int = 0
+    ) -> Tuple[List[datetime], np.ndarray]:
+        """
+        Looks up the time series in the internal data store.
+
+        :return a tuple (datetimes, values)
+        """
+        if ensemble_member >= self.__ensemble_size:
+            raise KeyError("ensemble_member {} does not exist".format(ensemble_member))
+        return self.__timeseries_datetimes, self.__timeseries_values[ensemble_member][variable]
+
+    def get_timeseries_names(self, ensemble_member: int = 0) -> Iterable[str]:
+        return self.__timeseries_values[ensemble_member].keys()
+
+    def set_timeseries_sec(
+        self,
+        variable: str,
+        times_in_sec: np.ndarray,
+        values: np.ndarray,
+        ensemble_member: int = 0,
+        check_duplicates: bool = False,
+    ) -> None:
+        """
+        Stores input time series values in the internal data store.
+
+        Note that once this method is called, it is no longer allowed to
+        change :py:attr:`reference_datetime`.
+
+        :param variable:         Variable name.
+        :param times_in_sec:     The times in seconds.
+        :param values:           The values to be stored.
+        :param ensemble_member:  The ensemble member index.
+        :param check_duplicates: If True, a warning will be given when overwriting values.
+                                 If False, existing values are silently overwritten with new values.
+        """
+        self._datetimes_to_seconds()
+
+        if self.reference_datetime is None:
+            raise RuntimeError("Cannot use times in seconds before reference datetime is set.")
+
+        if self.__timeseries_times_sec is not None and not np.array_equal(
+            times_in_sec, self.__timeseries_times_sec
+        ):
+            raise RuntimeError(
+                "Attempting to overwrite the input time series times with different values. "
+                "Please ensure all input time series have the same times."
+            )
+
+        if len(self.__timeseries_datetimes) != len(values):
+            raise ValueError(
+                "Length of values ({}) must be the same as length of times ({})".format(
+                    len(values), len(self.__timeseries_datetimes)
+                )
+            )
+
+        if ensemble_member >= self.__ensemble_size:
+            self.__update_ensemble_size(ensemble_member + 1)
+
+        if check_duplicates and variable in self.__timeseries_values[ensemble_member].keys():
+            logger.warning(
+                "Time series values for ensemble member {} and variable {} set twice. "
+                "Overwriting old values.".format(ensemble_member, variable)
+            )
+
+        self.__timeseries_values[ensemble_member][variable] = values
+
+    def get_timeseries_sec(
+        self, variable: str, ensemble_member: int = 0
+    ) -> Tuple[np.ndarray, np.ndarray]:
+        """
+        Looks up the time series in the internal data store.
+
+        Note that once this method is called, it is no longer allowed to
+        change :py:attr:`reference_datetime`.
+
+        :return a tuple (times, values)
+        """
+        self._datetimes_to_seconds()
+
+        if ensemble_member >= self.__ensemble_size:
+            raise KeyError("ensemble_member {} does not exist".format(ensemble_member))
+        return self.__timeseries_times_sec, self.__timeseries_values[ensemble_member][variable]
+
+    def set_parameter(
+        self,
+        parameter_name: str,
+        value: float,
+        ensemble_member: int = 0,
+        check_duplicates: bool = False,
+    ) -> None:
+        """
+        Stores the parameter value in the internal data store.
+
+        :param parameter_name:   Parameter name.
+        :param value:            The values to be stored.
+        :param ensemble_member:  The ensemble member index.
+        :param check_duplicates: If True, a warning will be given when overwriting values.
+                                 If False, existing values are silently overwritten with new values.
+        """
+        if ensemble_member >= self.__ensemble_size:
+            self.__update_ensemble_size(ensemble_member + 1)
+
+        if check_duplicates and parameter_name in self.__parameters[ensemble_member].keys():
+            logger.warning(
+                "Attempting to set parameter value for ensemble member {} and name {} twice. "
+                "Using new value of {}.".format(ensemble_member, parameter_name, value)
+            )
+
+        self.__parameters[ensemble_member][parameter_name] = value
+
+    def get_parameter(self, parameter_name: str, ensemble_member: int = 0) -> float:
+        """
+        Looks up the parameter value in the internal data store.
+        """
+        if ensemble_member >= self.__ensemble_size:
+            raise KeyError("ensemble_member {} does not exist".format(ensemble_member))
+        return self.__parameters[ensemble_member][parameter_name]
+
+    def parameters(self, ensemble_member: int = 0) -> AliasDict:
+        """
+        Returns an AliasDict of parameters to its values for the specified ensemble member.
+        """
+        if ensemble_member >= self.__ensemble_size:
+            raise KeyError("ensemble_member {} does not exist".format(ensemble_member))
+        return self.__parameters[ensemble_member]
+
+    @staticmethod
+    def datetime_to_sec(
+        d: Union[Iterable[datetime], datetime], t0: datetime
+    ) -> Union[np.ndarray, float]:
+        """
+        Returns the date/timestamps in seconds since t0.
+
+        :param d:  Iterable of datetimes or a single datetime object.
+        :param t0: Reference datetime.
+        """
+        if hasattr(d, "__iter__"):
+            return np.array([(t - t0).total_seconds() for t in d])
+        else:
+            return (d - t0).total_seconds()
+
+    @staticmethod
+    def sec_to_datetime(
+        s: Union[Iterable[float], float], t0: datetime
+    ) -> Union[List[datetime], datetime]:
+        """
+        Return the date/timestamps in seconds since t0 as datetime objects.
+
+        :param s:  Iterable of ints or a single int (number of seconds before or after t0).
+        :param t0: Reference datetime.
+        """
+        if hasattr(s, "__iter__"):
+            return [t0 + timedelta(seconds=t) for t in s]
+        else:
+            return t0 + timedelta(seconds=s)