pthelma 1.0.0__cp310-cp310-win_amd64.whl

evaporation/evaporation.py

@@ -0,0 +1,437 @@
+import datetime as dt
+import math
+import warnings
+from math import cos, pi, sin, tan
+
+import numpy as np
+
+# Note about RuntimeWarning
+#
+# When numpy makes calculations with masked arrays, it sometimes emits spurious
+# RuntimeWarnings. This is because it occasionally does use the masked part of
+# the array during the calculations (but masks the result). This is a known
+# numpy bug (e.g. https://github.com/numpy/numpy/issues/4269). The numpy
+# documentation, section "Operations on masked arrays", also has a related
+# warning there.
+#
+# In order to avoid these spurious warnings, we have used, at various places in
+# the code, "with warnings.catch_warnings()". We have attempted to unit test
+# it, but sometimes it's hard to make the bug appear. A large array in
+# production may cause the bug, but a small array in the unit test might not
+# cause it, despite same python and numpy version. So the locations in which
+# a fix was needed were largely located in production.
+
+
+class PenmanMonteith(object):
+    # Stefan-Boltzmann constant (Allen et al., 1998, p. 52)
+    sigma = 4.903e-9
+
+    def __init__(
+        self,
+        albedo,
+        elevation,
+        latitude,
+        time_step,
+        longitude=None,
+        nighttime_solar_radiation_ratio=None,
+        unit_converters={},
+    ):
+        self.albedo = albedo
+        self.nighttime_solar_radiation_ratio = nighttime_solar_radiation_ratio
+        self.elevation = elevation
+        self.latitude = latitude
+        self.longitude = longitude
+        self.time_step = time_step
+        self.unit_converters = unit_converters
+
+    def calculate(self, **kwargs):
+        if self.time_step == "H":
+            return self.calculate_hourly(**kwargs)
+        elif self.time_step == "D":
+            return self.calculate_daily(**kwargs)
+        else:
+            raise NotImplementedError(
+                "Evaporation for time steps other than hourly and daily "
+                "has not been implemented."
+            )
+
+    def calculate_daily(
+        self,
+        temperature_max,
+        temperature_min,
+        humidity_max,
+        humidity_min,
+        wind_speed,
+        adatetime,
+        sunshine_duration=None,
+        pressure=None,
+        solar_radiation=None,
+    ):
+        if pressure is None:
+            # Eq. 7 p. 31
+            pressure = 101.3 * ((293 - 0.0065 * self.elevation) / 293) ** 5.26
+        variables = self.convert_units(
+            temperature_max=temperature_max,
+            temperature_min=temperature_min,
+            humidity_max=humidity_max,
+            humidity_min=humidity_min,
+            wind_speed=wind_speed,
+            sunshine_duration=sunshine_duration,
+            pressure=pressure,
+        )
+
+        # Radiation
+        r_a, N = self.get_extraterrestrial_radiation(adatetime)
+        if solar_radiation is None:
+            solar_radiation = (
+                0.25 + 0.50 * variables["sunshine_duration"] / N
+            ) * r_a  # Eq.35 p. 50
+        r_so = r_a * (0.75 + 2e-5 * self.elevation)  # Eq. 37, p. 51
+        variables.update(self.convert_units(solar_radiation=solar_radiation))
+
+        with warnings.catch_warnings():
+            # See comment about RuntimeWarning on top of the file
+            warnings.simplefilter("ignore", RuntimeWarning)
+            temperature_mean = (
+                variables["temperature_max"] + variables["temperature_min"]
+            ) / 2
+        variables["temperature_mean"] = temperature_mean
+        gamma = self.get_psychrometric_constant(temperature_mean, pressure)
+        return self.penman_monteith_daily(
+            incoming_solar_radiation=variables["solar_radiation"],
+            clear_sky_solar_radiation=r_so,
+            psychrometric_constant=gamma,
+            mean_wind_speed=variables["wind_speed"],
+            temperature_max=variables["temperature_max"],
+            temperature_min=variables["temperature_min"],
+            temperature_mean=variables["temperature_mean"],
+            humidity_max=variables["humidity_max"],
+            humidity_min=variables["humidity_min"],
+            adate=adatetime,
+        )
+
+    def calculate_hourly(
+        self,
+        temperature,
+        humidity,
+        wind_speed,
+        solar_radiation,
+        adatetime,
+        pressure=None,
+    ):
+        if pressure is None:
+            # Eq. 7 p. 31
+            pressure = 101.3 * ((293 - 0.0065 * self.elevation) / 293) ** 5.26
+        variables = self.convert_units(
+            temperature=temperature,
+            humidity=humidity,
+            wind_speed=wind_speed,
+            pressure=pressure,
+            solar_radiation=solar_radiation,
+        )
+        gamma = self.get_psychrometric_constant(
+            variables["temperature"], variables["pressure"]
+        )
+        r_so = self.get_extraterrestrial_radiation(adatetime) * (
+            0.75 + 2e-5 * self.elevation
+        )  # Eq. 37, p. 51
+        return self.penman_monteith_hourly(
+            incoming_solar_radiation=variables["solar_radiation"],
+            clear_sky_solar_radiation=r_so,
+            psychrometric_constant=gamma,
+            mean_wind_speed=variables["wind_speed"],
+            mean_temperature=variables["temperature"],
+            mean_relative_humidity=variables["humidity"],
+            adatetime=adatetime,
+        )
+
+    def convert_units(self, **kwargs):
+        result = {}
+        for item in kwargs:
+            varname = item
+            if item.endswith("_max") or item.endswith("_min"):
+                varname = item[:-4]
+            converter = self.unit_converters.get(varname, lambda x: x)
+            with warnings.catch_warnings():
+                # See comment about RuntimeWarning on top of the file
+                warnings.simplefilter("ignore", RuntimeWarning)
+                result[item] = converter(kwargs[item])
+        return result
+
+    def get_extraterrestrial_radiation(self, adatetime):
+        """
+        Calculates the solar radiation we would receive if there were no
+        atmosphere. This is a function of date, time and location.
+
+        If adatetime is a datetime object, it merely returns the
+        extraterrestrial radiation R_a; if it is a date object, it returns a
+        tuple, (R_a, N), where N is the daylight hours.
+        """
+        j = adatetime.timetuple().tm_yday  # Day of year
+
+        # Inverse relative distance Earth-Sun, eq. 23, p. 46.
+        dr = 1 + 0.033 * cos(2 * pi * j / 365)
+
+        # Solar declination, eq. 24, p. 46.
+        decl = 0.409 * sin(2 * pi * j / 365 - 1.39)
+
+        if self.time_step == "D":  # Daily?
+            phi = self.latitude / 180.0 * pi
+            omega_s = np.arccos(-np.tan(phi) * tan(decl))  # Eq. 25 p. 46
+
+            r_a = (
+                24
+                * 60
+                / pi
+                * 0.0820
+                * dr
+                * (
+                    omega_s * np.sin(phi) * sin(decl)
+                    + np.cos(phi) * cos(decl) * np.sin(omega_s)
+                )
+            )  # Eq. 21 p. 46
+            n = 24 / pi * omega_s  # Eq. 34 p. 48
+            return r_a, n
+
+        # Seasonal correction for solar time, eq. 32, p. 48.
+        b = 2 * pi * (j - 81) / 364
+        sc = 0.1645 * sin(2 * b) - 0.1255 * cos(b) - 0.025 * sin(b)
+
+        # Longitude at the centre of the local time zone
+        utc_offset = adatetime.utcoffset()
+        utc_offset_hours = utc_offset.days * 24 + utc_offset.seconds / 3600.0
+        lz = -utc_offset_hours * 15
+
+        # Solar time angle at midpoint of the time period, eq. 31, p. 48.
+        time_step_delta = (
+            self.time_step == "D" and dt.timedelta(days=1) or dt.timedelta(hours=1)
+        )
+        tm = adatetime - time_step_delta / 2
+        t = tm.hour + tm.minute / 60.0
+        omega = pi / 12 * ((t + 0.06667 * (lz + self.longitude) + sc) - 12)
+
+        # Solar time angles at beginning and end of the period, eqs. 29 and 30,
+        # p. 48.
+        t1 = time_step_delta.seconds / 3600.0
+        omega1 = omega - pi * t1 / 24
+        omega2 = omega + pi * t1 / 24
+
+        # Result: eq. 28, p. 47.
+        phi = self.latitude / 180.0 * pi
+        return (
+            12
+            * 60
+            / pi
+            * 0.0820
+            * dr
+            * (
+                (omega2 - omega1) * np.sin(phi) * sin(decl)
+                + np.cos(phi) * cos(decl) * (np.sin(omega2) - np.sin(omega1))
+            )
+        )
+
+    def get_psychrometric_constant(self, temperature, pressure):
+        """
+        Allen et al. (1998), eq. 8, p. 32.
+
+        This is called a "constant" because, although it is a function of
+        temperature and pressure, its variations are small, and therefore it
+        can be assumed constant for a location assuming standard pressure at
+        that elevation and 20 degrees C. However, here we actually calculate
+        it, so it isn't a constant.
+        """
+        lambda_ = 2.501 - (2.361e-3) * temperature  # eq. 3-1, p. 223
+        return 1.013e-3 * pressure / 0.622 / lambda_
+
+    def penman_monteith_daily(
+        self,
+        incoming_solar_radiation,
+        clear_sky_solar_radiation,
+        psychrometric_constant,
+        mean_wind_speed,
+        temperature_max,
+        temperature_min,
+        temperature_mean,
+        humidity_max,
+        humidity_min,
+        adate,
+    ):
+        """
+        Calculates and returns the reference evapotranspiration according
+        to Allen et al. (1998), eq. 6, p. 24 & 65.
+        """
+
+        # Saturation and actual vapour pressure
+        svp_max = self.get_saturation_vapour_pressure(temperature_max)
+        svp_min = self.get_saturation_vapour_pressure(temperature_min)
+        avp1 = svp_max * humidity_min / 100
+        avp2 = svp_min * humidity_max / 100
+        svp = (svp_max + svp_min) / 2  # Eq. 12 p. 36
+        avp = (avp1 + avp2) / 2  # Eq. 12 p. 36
+
+        # Saturation vapour pressure curve slope
+        delta = self.get_saturation_vapour_pressure_curve_slope(temperature_mean)
+
+        # Net incoming radiation; p. 51, eq. 38
+        albedo = (
+            self.albedo[adate.month - 1]
+            if self.albedo.__class__.__name__ in ("tuple", "list")
+            else self.albedo
+        )
+        rns = (1.0 - albedo) * incoming_solar_radiation
+
+        # Net outgoing radiation
+        rnl = self.get_net_outgoing_radiation(
+            (temperature_min, temperature_max),
+            incoming_solar_radiation,
+            clear_sky_solar_radiation,
+            avp,
+        )
+
+        # Net radiation at grass surface
+        rn = rns - rnl
+
+        # Soil heat flux
+        g_day = 0  # Eq. 42 p. 54
+
+        # Apply the formula
+        numerator_term1 = 0.408 * delta * (rn - g_day)
+        numerator_term2 = (
+            psychrometric_constant
+            * 900
+            / (temperature_mean + 273.16)
+            * mean_wind_speed
+            * (svp - avp)
+        )
+        denominator = delta + psychrometric_constant * (1 + 0.34 * mean_wind_speed)
+
+        return (numerator_term1 + numerator_term2) / denominator
+
+    def penman_monteith_hourly(
+        self,
+        incoming_solar_radiation,
+        clear_sky_solar_radiation,
+        psychrometric_constant,
+        mean_wind_speed,
+        mean_temperature,
+        mean_relative_humidity,
+        adatetime,
+    ):
+        """
+        Calculates and returns the reference evapotranspiration according
+        to Allen et al. (1998), eq. 53, p. 74.
+
+        As explained in Allen et al. (1998, p. 74), the function is
+        modified in relation to the original Penman-Monteith equation, so
+        that it is suitable for hourly data.
+        """
+
+        # Saturation and actual vapour pressure
+        svp = self.get_saturation_vapour_pressure(mean_temperature)
+        with warnings.catch_warnings():
+            # See comment about RuntimeWarning on top of the file
+            warnings.simplefilter("ignore", RuntimeWarning)
+            avp = svp * mean_relative_humidity / 100.0  # Eq. 54, p. 74
+
+        # Net incoming radiation; p. 51, eq. 38
+        albedo = (
+            self.albedo[adatetime.month - 1]
+            if self.albedo.__class__.__name__ in ("tuple", "list")
+            else self.albedo
+        )
+        rns = (1.0 - albedo) * incoming_solar_radiation
+
+        # Net outgoing radiation
+        rnl = self.get_net_outgoing_radiation(
+            mean_temperature, incoming_solar_radiation, clear_sky_solar_radiation, avp
+        )
+
+        # Net radiation at grass surface
+        rn = rns - rnl
+
+        # Saturation vapour pressure curve slope
+        delta = self.get_saturation_vapour_pressure_curve_slope(mean_temperature)
+
+        # Soil heat flux density
+        g = self.get_soil_heat_flux_density(incoming_solar_radiation, rn)
+
+        # Apply the formula
+        numerator_term1 = 0.408 * delta * (rn - g)
+        with warnings.catch_warnings():
+            # See comment about RuntimeWarning on top of the file
+            warnings.simplefilter("ignore", RuntimeWarning)
+            numerator_term2 = (
+                psychrometric_constant
+                * 37
+                / (mean_temperature + 273.16)
+                * mean_wind_speed
+                * (svp - avp)
+            )
+        denominator = delta + psychrometric_constant * (1 + 0.34 * mean_wind_speed)
+
+        return (numerator_term1 + numerator_term2) / denominator
+
+    def get_net_outgoing_radiation(
+        self,
+        temperature,
+        incoming_solar_radiation,
+        clear_sky_solar_radiation,
+        mean_actual_vapour_pressure,
+    ):
+        """
+        Allen et al. (1998), p. 52, eq. 39. Temperature can be a tuple (a pair)
+        of min and max, or a single value. If it is a single value, the
+        equation is modified according to end of page 74.
+        """
+        if temperature.__class__.__name__ in ("tuple", "list"):
+            with warnings.catch_warnings():
+                # See comment about RuntimeWarning on top of the file
+                warnings.simplefilter("ignore", RuntimeWarning)
+                factor1 = (
+                    self.sigma
+                    * ((temperature[0] + 273.16) ** 4 + (temperature[1] + 273.16) ** 4)
+                    / 2
+                )
+        else:
+            with warnings.catch_warnings():
+                # See comment about RuntimeWarning on top of the file
+                warnings.simplefilter("ignore", RuntimeWarning)
+                factor1 = self.sigma / 24 * (temperature + 273.16) ** 4
+        factor2 = 0.34 - 0.14 * (mean_actual_vapour_pressure**0.5)
+
+        # Solar radiation ratio Rs/Rs0 (Allen et al., 1998, top of p. 75).
+        with warnings.catch_warnings():
+            # See comment about RuntimeWarning on top of the file
+            warnings.simplefilter("ignore", RuntimeWarning)
+            solar_radiation_ratio = np.where(
+                clear_sky_solar_radiation > 0.05,
+                incoming_solar_radiation / clear_sky_solar_radiation,
+                self.nighttime_solar_radiation_ratio,
+            )
+        solar_radiation_ratio = np.maximum(solar_radiation_ratio, 0.3)
+        solar_radiation_ratio = np.minimum(solar_radiation_ratio, 1.0)
+
+        factor3 = 1.35 * solar_radiation_ratio - 0.35
+
+        return factor1 * factor2 * factor3
+
+    def get_saturation_vapour_pressure(self, temperature):
+        "Allen et al. (1998), p. 36, eq. 11."
+        with warnings.catch_warnings():
+            # See comment about RuntimeWarning on top of the file
+            warnings.simplefilter("ignore")
+            return 0.6108 * math.e ** (17.27 * temperature / (237.3 + temperature))
+
+    def get_soil_heat_flux_density(self, incoming_solar_radiation, rn):
+        "Allen et al. (1998), p. 55, eq. 45 & 46."
+        coefficient = np.where(incoming_solar_radiation > 0.05, 0.1, 0.5)
+        return coefficient * rn
+
+    def get_saturation_vapour_pressure_curve_slope(self, temperature):
+        "Allen et al. (1998), p. 37, eq. 13."
+        numerator = 4098 * self.get_saturation_vapour_pressure(temperature)
+        with warnings.catch_warnings():
+            # See comment about RuntimeWarning on top of the file
+            warnings.simplefilter("ignore", RuntimeWarning)
+            denominator = (temperature + 237.3) ** 2
+            return numerator / denominator

haggregate/__init__.py

@@ -0,0 +1,5 @@
+from .haggregate import *  # NOQA
+from .regularize import *  # NOQA
+
+__author__ = """Antonis Christofides"""
+__email__ = "antonis@antonischristofides.com"

haggregate/cli.py

@@ -0,0 +1,91 @@
+import configparser
+import datetime as dt
+import logging
+import os
+import sys
+import traceback
+
+import click
+
+from haggregate import RegularizationMode, aggregate
+from haggregate.regularize import regularize
+from htimeseries import HTimeseries
+
+
+@click.command()
+@click.argument("configfile")
+def main(configfile):
+    """Create lower-step timeseries from higher-step ones"""
+
+    # Start by setting logger to stdout; later we will switch it according to config
+    logger = logging.getLogger("haggregate")
+    stdout_handler = logging.StreamHandler()
+    logger.addHandler(stdout_handler)
+
+    try:
+        config = configparser.ConfigParser()
+        with open(configfile) as f:
+            config.read_file(f)
+
+        # Read the [General] section
+        logfile = config.get("General", "logfile", fallback="")
+        loglevel = config.get("General", "loglevel", fallback="warning")
+        base_dir = config.get("General", "base_dir", fallback=".")
+        target_step = config.get("General", "target_step")
+        min_count = config.getint("General", "min_count")
+        missing_flag = config.get("General", "missing_flag")
+        target_timestamp_offset = config.get(
+            "General", "target_timestamp_offset", fallback=None
+        )
+
+        # Remove [General] and make sure there are more sections
+        config.pop("General")
+        if not len(config.sections()):
+            raise configparser.NoSectionError("No time series have been specified")
+
+        # Setup logger
+        logger.setLevel(loglevel.upper())
+        if logfile:
+            logger.removeHandler(stdout_handler)
+            logger.addHandler(logging.FileHandler(logfile))
+
+        # Log start of execution
+        logger.info("Starting haggregate, " + dt.datetime.today().isoformat())
+
+        # Read each section and do the work for it
+        for section_name in config.sections():
+            section = config[section_name]
+            source_filename = os.path.join(base_dir, section.get("source_file"))
+            target_filename = os.path.join(base_dir, section.get("target_file"))
+            method = section.get("method")
+            with open(source_filename, newline="\n") as f:
+                ts = HTimeseries(
+                    f, format=HTimeseries.FILE, default_tzinfo=dt.timezone.utc
+                )
+            if method == "mean":
+                regularization_mode = RegularizationMode.INSTANTANEOUS
+            else:
+                regularization_mode = RegularizationMode.INTERVAL
+            regts = regularize(ts, new_date_flag="DATEINSERT", mode=regularization_mode)
+            aggts = aggregate(
+                regts,
+                target_step,
+                method,
+                min_count=min_count,
+                missing_flag=missing_flag,
+                target_timestamp_offset=target_timestamp_offset,
+            )
+            with open(target_filename, "w") as f:
+                aggts.write(f, format=HTimeseries.FILE)
+
+        # Log end of execution
+        logger.info("Finished haggregate, " + dt.datetime.today().isoformat())
+
+    except Exception as e:
+        logger.error(str(e))
+        logger.debug(traceback.format_exc())
+        raise click.ClickException(str(e))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

haggregate/haggregate.py

@@ -0,0 +1,155 @@
+import re
+from enum import Enum
+
+import numpy as np
+import pandas as pd
+
+from htimeseries import HTimeseries
+
+methods = {
+    "sum": pd.Series.sum,
+    "mean": pd.Series.mean,
+    "max": pd.Series.max,
+    "min": pd.Series.min,
+}
+
+
+class AggregateError(Exception):
+    pass
+
+
+def aggregate(
+    hts,
+    target_step,
+    method,
+    min_count=1,
+    missing_flag="MISS",
+    target_timestamp_offset=None,
+):
+    aggregation = Aggregation(
+        source_timeseries=hts,
+        target_step=target_step,
+        method=method,
+        min_count=min_count,
+        missing_flag=missing_flag,
+        target_timestamp_offset=target_timestamp_offset,
+    )
+    aggregation.execute()
+    return aggregation.result
+
+
+class Aggregation:
+    def __init__(self, **kwargs):
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+        self.source = SourceTimeseries(self.source_timeseries)
+        self.result = AggregatedTimeseries()
+        self.result.time_step = self.target_step
+
+    def execute(self):
+        self.result.set_metadata(self.source_timeseries)
+        try:
+            self.source.normalize(self.target_step)
+        except CannotInferFrequency:
+            return
+        self.do_aggregation()
+        self.result.remove_leading_and_trailing_nans()
+        self.result.add_timestamp_offset(self.target_timestamp_offset)
+
+    def do_aggregation(self):
+        self.create_resampler()
+        self.get_result_values()
+        self.get_result_flags()
+
+    def create_resampler(self):
+        self.resampler = self.source.data["value"].resample(
+            self.result.time_step, closed="right", label="right"
+        )
+
+    def get_result_values(self):
+        result_values = self.resampler.agg(methods[self.method])
+        values_count = self.resampler.count()
+        result_values[values_count < self.min_count] = np.nan
+        self.result.data["value"] = result_values
+
+    def get_result_flags(self):
+        max_count = int(pd.Timedelta(self.result.time_step) / self.source.freq)
+        values_count = self.resampler.count()
+        self.result.data["flags"] = (values_count < max_count).apply(
+            lambda x: self.missing_flag if x else ""
+        )
+
+
+class CannotInferFrequency(Exception):
+    pass
+
+
+attrs = ("unit", "timezone", "interval_type", "variable", "precision", "location")
+
+
+class SourceTimeseries(HTimeseries):
+    def __init__(self, s):
+        for attr in attrs:
+            setattr(self, attr, getattr(s, attr, None))
+        self.data = s.data
+
+    def normalize(self, target_step):
+        """Reindex so that it has no missing records but has NaNs instead, starting from
+        one before and ending in one after.
+        """
+        current_range = self.data.index
+        try:
+            self.freq = pd.tseries.frequencies.to_offset(pd.infer_freq(current_range))
+            if self.freq is None:
+                raise AggregateError(
+                    "Can't infer time series step; maybe it's not regularized"
+                )
+        except ValueError:
+            raise CannotInferFrequency()
+        first_timestamp = (current_range[0] - pd.Timedelta("1S")).floor(target_step)
+        end_timestamp = current_range[-1].ceil(target_step)
+        new_range = pd.date_range(first_timestamp, end_timestamp, freq=self.freq)
+        self.data = self.data.reindex(new_range)
+
+
+class AggregatedTimeseries(HTimeseries):
+    def set_metadata(self, source_timeseries):
+        for attr in attrs:
+            setattr(self, attr, getattr(source_timeseries, attr, None))
+        if self.time_step not in ("1H", "1D"):
+            raise AggregateError("The target step can currently only be 1H or 1D")
+        if hasattr(source_timeseries, "title"):
+            self.title = "Aggregated " + source_timeseries.title
+        if hasattr(source_timeseries, "comment"):
+            self.comment = (
+                "Created by aggregating the time series that had this comment:\n\n"
+                + source_timeseries.comment
+            )
+
+    def remove_leading_and_trailing_nans(self):
+        while len(self.data.index) > 0 and pd.isnull(self.data["value"]).iloc[0]:
+            self.data = self.data.drop(self.data.index[0])
+        while len(self.data.index) > 0 and pd.isnull(self.data["value"]).iloc[-1]:
+            self.data = self.data.drop(self.data.index[-1])
+
+    def add_timestamp_offset(self, target_timestamp_offset):
+        if target_timestamp_offset:
+            periods = target_timestamp_offset.startswith("-") and 1 or -1
+            freq = target_timestamp_offset.lstrip("-")
+            self.data = self.data.shift(periods, freq=freq)
+
+
+def _get_offset_in_minutes(timestamp_offset):
+    m = re.match(r"(-?)(\d*)(T|min)$", timestamp_offset)
+    if not m:
+        raise AggregateError(
+            "The target timestamp offset can currently only be a number of minutes "
+            "such as 1min"
+        )
+    sign = m.group(1) == "-" and -1 or 1
+    return sign * int(m.group(2))
+
+
+class RegularizationMode(Enum):
+    INSTANTANEOUS = 1
+    INTERVAL = 2