rtc-tools 2.5.2rc4__py3-none-any.whl → 2.6.0__py3-none-any.whl

rtctools/data/rtc.py

@@ -3,11 +3,10 @@ import os
 import xml.etree.ElementTree as ET
 from collections import namedtuple
 
-ts_ids = namedtuple('ids', 'location_id parameter_id qualifier_id')
-p_ids = namedtuple('ids', 'model_id location_id parameter_id')
+ts_ids = namedtuple("ids", "location_id parameter_id qualifier_id")
+p_ids = namedtuple("ids", "model_id location_id parameter_id")
 
-ns = {'fews': 'http://www.wldelft.nl/fews',
-      'pi': 'http://www.wldelft.nl/fews/PI'}
+ns = {"fews": "http://www.wldelft.nl/fews", "pi": "http://www.wldelft.nl/fews/PI"}
 
 logger = logging.getLogger("rtctools")
 
@@ -35,77 +34,85 @@ class DataConfig:
             tree = ET.parse(path)
             root = tree.getroot()
 
-            timeseriess1 = root.findall('./*/fews:timeSeries', ns)
-            timeseriess2 = root.findall('./fews:timeSeries', ns)
+            timeseriess1 = root.findall("./*/fews:timeSeries", ns)
+            timeseriess2 = root.findall("./fews:timeSeries", ns)
             timeseriess1.extend(timeseriess2)
 
             for timeseries in timeseriess1:
-                pi_timeseries = timeseries.find('fews:PITimeSeries', ns)
+                pi_timeseries = timeseries.find("fews:PITimeSeries", ns)
                 if pi_timeseries is not None:
-                    internal_id = timeseries.get('id')
-                    external_id = self.__pi_timeseries_id(pi_timeseries, 'fews')
+                    internal_id = timeseries.get("id")
+                    external_id = self.__pi_timeseries_id(pi_timeseries, "fews")
 
                     if internal_id in self.__location_parameter_ids:
                         message = (
-                            'Found more than one external timeseries '
-                            'mapped to internal id {} in {}.').format(internal_id, path)
+                            "Found more than one external timeseries "
+                            "mapped to internal id {} in {}."
+                        ).format(internal_id, path)
                         logger.error(message)
                         raise Exception(message)
                     elif external_id in self.__variable_map:
                         message = (
-                            'Found more than one internal timeseries '
-                            'mapped to external id {} in {}.').format(external_id, path)
+                            "Found more than one internal timeseries "
+                            "mapped to external id {} in {}."
+                        ).format(external_id, path)
                         logger.error(message)
                         raise Exception(message)
                     else:
-                        self.__location_parameter_ids[internal_id] = \
-                            self.__pi_location_parameter_id(pi_timeseries, 'fews')
+                        self.__location_parameter_ids[
+                            internal_id
+                        ] = self.__pi_location_parameter_id(pi_timeseries, "fews")
                         self.__variable_map[external_id] = internal_id
 
-            for k in ['import', 'export']:
-                res = root.find(
-                    './fews:%s/fews:PITimeSeriesFile/fews:timeSeriesFile' % k, ns)
+            for k in ["import", "export"]:
+                res = root.find("./fews:%s/fews:PITimeSeriesFile/fews:timeSeriesFile" % k, ns)
                 if res is not None:
-                    setattr(self, 'basename_%s' %
-                            k, os.path.splitext(res.text)[0])
+                    setattr(self, "basename_%s" % k, os.path.splitext(res.text)[0])
 
-            parameters = root.findall('./fews:parameter', ns)
+            parameters = root.findall("./fews:parameter", ns)
             if parameters is not None:
                 for parameter in parameters:
-                    pi_parameter = parameter.find('fews:PIParameter', ns)
+                    pi_parameter = parameter.find("fews:PIParameter", ns)
                     if pi_parameter is not None:
-                        internal_id = parameter.get('id')
-                        external_id = self.__pi_parameter_id(pi_parameter, 'fews')
+                        internal_id = parameter.get("id")
+                        external_id = self.__pi_parameter_id(pi_parameter, "fews")
 
                         if internal_id in self.__model_parameter_ids:
                             message = (
-                                'Found more than one external parameter mapped '
-                                'to internal id {} in {}.').format(internal_id, path)
+                                "Found more than one external parameter mapped "
+                                "to internal id {} in {}."
+                            ).format(internal_id, path)
                             logger.error(message)
                             raise Exception(message)
                         if external_id in self.__parameter_map:
                             message = (
-                                'Found more than one interal parameter mapped to external '
-                                'modelId {}, locationId {}, parameterId {} in {}.').format(
-                                external_id.model_id, external_id.location_id, external_id.parameter_id, path)
+                                "Found more than one interal parameter mapped to external "
+                                "modelId {}, locationId {}, parameterId {} in {}."
+                            ).format(
+                                external_id.model_id,
+                                external_id.location_id,
+                                external_id.parameter_id,
+                                path,
+                            )
                             logger.error(message)
                             raise Exception(message)
                         else:
-                            self.__model_parameter_ids[internal_id] = self.__pi_model_parameter_id(pi_parameter, 'fews')
+                            self.__model_parameter_ids[internal_id] = self.__pi_model_parameter_id(
+                                pi_parameter, "fews"
+                            )
                             self.__parameter_map[external_id] = internal_id
 
         except IOError:
-            logger.error(
-                'No rtcDataConfig.xml file was found in "{}".'.format(folder))
+            logger.error('No rtcDataConfig.xml file was found in "{}".'.format(folder))
             raise
 
     def __pi_timeseries_id(self, el, namespace):
-        location_id = el.find(namespace + ':locationId', ns).text
-        parameter_id = el.find(namespace + ':parameterId', ns).text
+        location_id = el.find(namespace + ":locationId", ns).text
+        parameter_id = el.find(namespace + ":parameterId", ns).text
 
-        timeseries_id = location_id + ':' + parameter_id
+        timeseries_id = location_id + ":" + parameter_id
 
-        qualifiers = el.findall(namespace + ':qualifierId', ns)
+        qualifiers = el.findall(namespace + ":qualifierId", ns)
         qualifier_ids = []
         for qualifier in qualifiers:
             qualifier_ids.append(qualifier.text)
@@ -113,36 +120,40 @@ class DataConfig:
         if len(qualifier_ids) > 0:
             qualifier_ids.sort()
 
-            return timeseries_id + ':' + ':'.join(qualifier_ids)
+            return timeseries_id + ":" + ":".join(qualifier_ids)
         else:
             return timeseries_id
 
     def __pi_location_parameter_id(self, el, namespace):
         qualifier_ids = []
-        qualifiers = el.findall(namespace + ':qualifierId', ns)
+        qualifiers = el.findall(namespace + ":qualifierId", ns)
         for qualifier in qualifiers:
             qualifier_ids.append(qualifier.text)
 
-        location_parameter_ids = ts_ids(location_id=el.find(namespace + ':locationId', ns).text,
-                                        parameter_id=el.find(namespace + ':parameterId', ns).text,
-                                        qualifier_id=qualifier_ids)
+        location_parameter_ids = ts_ids(
+            location_id=el.find(namespace + ":locationId", ns).text,
+            parameter_id=el.find(namespace + ":parameterId", ns).text,
+            qualifier_id=qualifier_ids,
+        )
         return location_parameter_ids
 
     def __pi_parameter_id(self, el, namespace):
-        model_id = el.find(namespace + ':modelId', ns).text
-        location_id = el.find(namespace + ':locationId', ns).text
-        parameter_id = el.find(namespace + ':parameterId', ns).text
+        model_id = el.find(namespace + ":modelId", ns).text
+        location_id = el.find(namespace + ":locationId", ns).text
+        parameter_id = el.find(namespace + ":parameterId", ns).text
 
         return self.__long_parameter_id(parameter_id, location_id, model_id)
 
     def __pi_model_parameter_id(self, el, namespace):
-        model_id = el.find(namespace + ':modelId', ns).text
-        location_id = el.find(namespace + ':locationId', ns).text
-        parameter_id = el.find(namespace + ':parameterId', ns).text
+        model_id = el.find(namespace + ":modelId", ns).text
+        location_id = el.find(namespace + ":locationId", ns).text
+        parameter_id = el.find(namespace + ":parameterId", ns).text
 
-        model_parameter_ids = p_ids(model_id=(model_id if model_id is not None else ""),
-                                    location_id=(location_id if location_id is not None else ""),
-                                    parameter_id=(parameter_id if parameter_id is not None else ""))
+        model_parameter_ids = p_ids(
+            model_id=(model_id if model_id is not None else ""),
+            location_id=(location_id if location_id is not None else ""),
+            parameter_id=(parameter_id if parameter_id is not None else ""),
+        )
 
         return model_parameter_ids
 
@@ -152,9 +163,9 @@ class DataConfig:
         of the form model:location:parameter.
         """
         if location_id is not None:
-            parameter_id = location_id + ':' + parameter_id
+            parameter_id = location_id + ":" + parameter_id
         if model_id is not None:
-            parameter_id = model_id + ':' + parameter_id
+            parameter_id = model_id + ":" + parameter_id
         return parameter_id
 
     def variable(self, pi_header):
@@ -166,7 +177,7 @@ class DataConfig:
         :returns: A timeseries ID.
         :rtype: string
         """
-        series_id = self.__pi_timeseries_id(pi_header, 'pi')
+        series_id = self.__pi_timeseries_id(pi_header, "pi")
         try:
             return self.__variable_map[series_id]
         except KeyError:

rtctools/data/storage.py

@@ -23,14 +23,14 @@ class DataStoreAccessor(metaclass=ABCMeta):
     """
 
     #: Import file basename
-    timeseries_import_basename = 'timeseries_import'
+    timeseries_import_basename = "timeseries_import"
     #: Export file basename
-    timeseries_export_basename = 'timeseries_export'
+    timeseries_export_basename = "timeseries_export"
 
     def __init__(self, **kwargs):
         # Save arguments
-        self._input_folder = kwargs['input_folder'] if 'input_folder' in kwargs else 'input'
-        self._output_folder = kwargs['output_folder'] if 'output_folder' in kwargs else 'output'
+        self._input_folder = kwargs["input_folder"] if "input_folder" in kwargs else "input"
+        self._output_folder = kwargs["output_folder"] if "output_folder" in kwargs else "output"
 
         if logger.getEffectiveLevel() == logging.DEBUG:
             logger.debug("Expecting input files to be located in '" + self._input_folder + "'.")
@@ -71,7 +71,9 @@ class DataStore:
     @reference_datetime.setter
     def reference_datetime(self, value):
         if self.__reference_datetime_fixed and value != self.__reference_datetime:
-            raise RuntimeError("Cannot change reference datetime after times in seconds has been requested.")
+            raise RuntimeError(
+                "Cannot change reference datetime after times in seconds has been requested."
+            )
         self.__reference_datetime = value
 
     @property
@@ -92,7 +94,7 @@ class DataStore:
 
     @property
     def datetimes(self) -> List[datetime]:
-        """"
+        """
         Returns the timeseries times in seconds.
 
         :returns: timeseries datetimes, or None if there has been no call
@@ -102,7 +104,7 @@ class DataStore:
 
     @property
     def times_sec(self) -> np.ndarray:
-        """"
+        """
         Returns the timeseries times in seconds.
 
         Note that once this method is called, it is no longer allowed to
@@ -123,18 +125,26 @@ class DataStore:
             # "0.0" as one of our times in seconds. This restriction may be
             # loosened in the future.
             if self.reference_datetime not in self.__timeseries_datetimes:
-                raise Exception("Reference datetime {} should be equal to one of the timeseries datetimes {}".format(
-                    self.reference_datetime, self.__timeseries_datetimes))
-            self.__timeseries_times_sec = self.datetime_to_sec(self.__timeseries_datetimes, self.reference_datetime)
+                raise Exception(
+                    "Reference datetime {} should be equal to "
+                    "one of the timeseries datetimes {}".format(
+                        self.reference_datetime, self.__timeseries_datetimes
+                    )
+                )
+            self.__timeseries_times_sec = self.datetime_to_sec(
+                self.__timeseries_datetimes, self.reference_datetime
+            )
             self.__timeseries_times_sec.flags.writeable = False
             self.__reference_datetime_fixed = True
 
-    def set_timeseries(self,
-                       variable: str,
-                       datetimes: Iterable[datetime],
-                       values: np.ndarray,
-                       ensemble_member: int = 0,
-                       check_duplicates: bool = False) -> None:
+    def set_timeseries(
+        self,
+        variable: str,
+        datetimes: Iterable[datetime],
+        values: np.ndarray,
+        ensemble_member: int = 0,
+        check_duplicates: bool = False,
+    ) -> None:
         """
         Stores input time series values in the internal data store.
 
@@ -143,7 +153,7 @@ class DataStore:
         :param values:           The values to be stored.
         :param ensemble_member:  The ensemble member index.
         :param check_duplicates: If True, a warning will be given when overwriting values.
-                                 If False, existing values can be silently overwritten with new values.
+                                 If False, existing values are silently overwritten with new values.
         """
         datetimes = list(datetimes)
 
@@ -151,24 +161,33 @@ class DataStore:
             raise TypeError("DateStore.set_timeseries() only support datetimes")
 
         if self.__timeseries_datetimes is not None and datetimes != self.__timeseries_datetimes:
-            raise RuntimeError("Attempting to overwrite the input time series datetimes with different values. "
-                               "Please ensure all input time series have the same datetimes.")
+            raise RuntimeError(
+                "Attempting to overwrite the input time series datetimes with different values. "
+                "Please ensure all input time series have the same datetimes."
+            )
         self.__timeseries_datetimes = datetimes
 
         if len(self.__timeseries_datetimes) != len(values):
-            raise ValueError("Length of values ({}) must be the same as length of datetimes ({})"
-                             .format(len(values), len(self.__timeseries_datetimes)))
+            raise ValueError(
+                "Length of values ({}) must be the same as length of datetimes ({})".format(
+                    len(values), len(self.__timeseries_datetimes)
+                )
+            )
 
         if ensemble_member >= self.__ensemble_size:
             self.__update_ensemble_size(ensemble_member + 1)
 
         if check_duplicates and variable in self.__timeseries_values[ensemble_member].keys():
-            logger.warning("Time series values for ensemble member {} and variable {} set twice. "
-                           "Overwriting old values.".format(ensemble_member, variable))
+            logger.warning(
+                "Time series values for ensemble member {} and variable {} set twice. "
+                "Overwriting old values.".format(ensemble_member, variable)
+            )
 
         self.__timeseries_values[ensemble_member][variable] = values
 
-    def get_timeseries(self, variable: str, ensemble_member: int = 0) -> Tuple[List[datetime], np.ndarray]:
+    def get_timeseries(
+        self, variable: str, ensemble_member: int = 0
+    ) -> Tuple[List[datetime], np.ndarray]:
         """
         Looks up the time series in the internal data store.
 
@@ -181,12 +200,14 @@ class DataStore:
     def get_timeseries_names(self, ensemble_member: int = 0) -> Iterable[str]:
         return self.__timeseries_values[ensemble_member].keys()
 
-    def set_timeseries_sec(self,
-                           variable: str,
-                           times_in_sec: np.ndarray,
-                           values: np.ndarray,
-                           ensemble_member: int = 0,
-                           check_duplicates: bool = False) -> None:
+    def set_timeseries_sec(
+        self,
+        variable: str,
+        times_in_sec: np.ndarray,
+        values: np.ndarray,
+        ensemble_member: int = 0,
+        check_duplicates: bool = False,
+    ) -> None:
         """
         Stores input time series values in the internal data store.
 
@@ -198,31 +219,42 @@ class DataStore:
         :param values:           The values to be stored.
         :param ensemble_member:  The ensemble member index.
         :param check_duplicates: If True, a warning will be given when overwriting values.
-                                 If False, existing values can be silently overwritten with new values.
+                                 If False, existing values are silently overwritten with new values.
         """
         self._datetimes_to_seconds()
 
         if self.reference_datetime is None:
             raise RuntimeError("Cannot use times in seconds before reference datetime is set.")
 
-        if self.__timeseries_times_sec is not None and not np.array_equal(times_in_sec, self.__timeseries_times_sec):
-            raise RuntimeError("Attempting to overwrite the input time series times with different values. "
-                               "Please ensure all input time series have the same times.")
+        if self.__timeseries_times_sec is not None and not np.array_equal(
+            times_in_sec, self.__timeseries_times_sec
+        ):
+            raise RuntimeError(
+                "Attempting to overwrite the input time series times with different values. "
+                "Please ensure all input time series have the same times."
+            )
 
         if len(self.__timeseries_datetimes) != len(values):
-            raise ValueError("Length of values ({}) must be the same as length of times ({})"
-                             .format(len(values), len(self.__timeseries_datetimes)))
+            raise ValueError(
+                "Length of values ({}) must be the same as length of times ({})".format(
+                    len(values), len(self.__timeseries_datetimes)
+                )
+            )
 
         if ensemble_member >= self.__ensemble_size:
             self.__update_ensemble_size(ensemble_member + 1)
 
         if check_duplicates and variable in self.__timeseries_values[ensemble_member].keys():
-            logger.warning("Time series values for ensemble member {} and variable {} set twice. "
-                           "Overwriting old values.".format(ensemble_member, variable))
+            logger.warning(
+                "Time series values for ensemble member {} and variable {} set twice. "
+                "Overwriting old values.".format(ensemble_member, variable)
+            )
 
         self.__timeseries_values[ensemble_member][variable] = values
 
-    def get_timeseries_sec(self, variable: str, ensemble_member: int = 0) -> Tuple[np.ndarray, np.ndarray]:
+    def get_timeseries_sec(
+        self, variable: str, ensemble_member: int = 0
+    ) -> Tuple[np.ndarray, np.ndarray]:
         """
         Looks up the time series in the internal data store.
 
@@ -237,11 +269,13 @@ class DataStore:
             raise KeyError("ensemble_member {} does not exist".format(ensemble_member))
         return self.__timeseries_times_sec, self.__timeseries_values[ensemble_member][variable]
 
-    def set_parameter(self,
-                      parameter_name: str,
-                      value: float,
-                      ensemble_member: int = 0,
-                      check_duplicates: bool = False) -> None:
+    def set_parameter(
+        self,
+        parameter_name: str,
+        value: float,
+        ensemble_member: int = 0,
+        check_duplicates: bool = False,
+    ) -> None:
         """
         Stores the parameter value in the internal data store.
 
@@ -249,14 +283,16 @@ class DataStore:
         :param value:            The values to be stored.
         :param ensemble_member:  The ensemble member index.
         :param check_duplicates: If True, a warning will be given when overwriting values.
-                                 If False, existing values can be silently overwritten with new values.
+                                 If False, existing values are silently overwritten with new values.
         """
         if ensemble_member >= self.__ensemble_size:
             self.__update_ensemble_size(ensemble_member + 1)
 
         if check_duplicates and parameter_name in self.__parameters[ensemble_member].keys():
-            logger.warning("Attempting to set parameter value for ensemble member {} and name {} twice. "
-                           "Using new value of {}.".format(ensemble_member, parameter_name, value))
+            logger.warning(
+                "Attempting to set parameter value for ensemble member {} and name {} twice. "
+                "Using new value of {}.".format(ensemble_member, parameter_name, value)
+            )
 
         self.__parameters[ensemble_member][parameter_name] = value
 
@@ -277,27 +313,31 @@ class DataStore:
         return self.__parameters[ensemble_member]
 
     @staticmethod
-    def datetime_to_sec(d: Union[Iterable[datetime], datetime], t0: datetime) -> Union[np.ndarray, float]:
+    def datetime_to_sec(
+        d: Union[Iterable[datetime], datetime], t0: datetime
+    ) -> Union[np.ndarray, float]:
         """
         Returns the date/timestamps in seconds since t0.
 
         :param d:  Iterable of datetimes or a single datetime object.
         :param t0: Reference datetime.
         """
-        if hasattr(d, '__iter__'):
+        if hasattr(d, "__iter__"):
             return np.array([(t - t0).total_seconds() for t in d])
         else:
             return (d - t0).total_seconds()
 
     @staticmethod
-    def sec_to_datetime(s: Union[Iterable[float], float], t0: datetime) -> Union[List[datetime], datetime]:
+    def sec_to_datetime(
+        s: Union[Iterable[float], float], t0: datetime
+    ) -> Union[List[datetime], datetime]:
         """
         Return the date/timestamps in seconds since t0 as datetime objects.
 
         :param s:  Iterable of ints or a single int (number of seconds before or after t0).
         :param t0: Reference datetime.
         """
-        if hasattr(s, '__iter__'):
+        if hasattr(s, "__iter__"):
             return [t0 + timedelta(seconds=t) for t in s]
         else:
             return t0 + timedelta(seconds=s)