e-data 1.3.1__py3-none-any.whl

processors/base.py

@@ -0,0 +1,27 @@
+"""Base definitions for processors."""
+
+from abc import ABC, abstractmethod
+from copy import deepcopy
+from typing import Any
+
+
+class Processor(ABC):
+    """A base class for data processors."""
+
+    _LABEL = "Processor"
+
+    def __init__(self, input_data: Any, auto: bool = True) -> None:
+        """Init method."""
+        self._input = deepcopy(input_data)
+        self._output = None
+        if auto:
+            self.do_process()
+
+    @abstractmethod
+    def do_process(self):
+        """Process method."""
+
+    @property
+    def output(self):
+        """An output property."""
+        return deepcopy(self._output)

processors/billing.py

@@ -0,0 +1,215 @@
+"""Billing data processors."""
+
+import contextlib
+from datetime import datetime, timedelta
+import logging
+from typing import Optional, TypedDict
+
+from jinja2 import Environment
+import voluptuous
+
+from ..definitions import (
+    ConsumptionData,
+    ConsumptionSchema,
+    ContractData,
+    ContractSchema,
+    PricingAggData,
+    PricingData,
+    PricingRules,
+    PricingRulesSchema,
+    PricingSchema,
+)
+from ..processors import utils
+from ..processors.base import Processor
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class BillingOutput(TypedDict):
+    """A dict holding BillingProcessor output property."""
+
+    hourly: list[PricingAggData]
+    daily: list[PricingAggData]
+    monthly: list[PricingAggData]
+
+
+class BillingInput(TypedDict):
+    """A dict holding BillingProcessor input data."""
+
+    contracts: list[ContractData]
+    consumptions: list[ConsumptionData]
+    prices: Optional[list[PricingData]]
+    rules: PricingRules
+
+
+class BillingProcessor(Processor):
+    """A billing processor for edata."""
+
+    def do_process(self):
+        """Process billing and get hourly/daily/monthly metrics."""
+        self._output = BillingOutput(hourly=[], daily=[], monthly=[])
+
+        _schema = voluptuous.Schema(
+            {
+                voluptuous.Required("contracts"): [ContractSchema],
+                voluptuous.Required("consumptions"): [ConsumptionSchema],
+                voluptuous.Optional("prices", default=None): voluptuous.Union(
+                    [voluptuous.Union(PricingSchema)], None
+                ),
+                voluptuous.Required("rules"): PricingRulesSchema,
+            }
+        )
+        self._input = _schema(self._input)
+
+        self._cycle_offset = self._input["rules"]["cycle_start_day"] - 1
+
+        # joint data by datetime
+        _data = {
+            x["datetime"]: {
+                "datetime": x["datetime"],
+                "kwh": x["value_kWh"],
+                "surplus_kwh": x["surplus_kWh"] if x["surplus_kWh"] is not None else 0,
+            }
+            for x in self._input["consumptions"]
+        }
+
+        for contract in self._input["contracts"]:
+            start = contract["date_start"]
+            end = contract["date_end"]
+            finish = False
+            while not finish:
+                if start in _data:
+                    _data[start]["p1_kw"] = contract["power_p1"]
+                    _data[start]["p2_kw"] = contract["power_p2"]
+                start = start + timedelta(hours=1)
+                finish = not (end > start)
+
+        if self._input["prices"]:
+            for x in self._input["prices"]:
+                start = x["datetime"]
+                if start in _data:
+                    _data[start]["kwh_eur"] = x["value_eur_kWh"]
+
+        env = Environment()
+        energy_expr = env.compile_expression(
+            f'({self._input["rules"]["energy_formula"]})|float'
+        )
+        power_expr = env.compile_expression(
+            f'({self._input["rules"]["power_formula"]})|float'
+        )
+        others_expr = env.compile_expression(
+            f'({self._input["rules"]["others_formula"]})|float'
+        )
+        surplus_expr = env.compile_expression(
+            f'({self._input["rules"]["surplus_formula"]})|float'
+        )
+        main_expr = env.compile_expression(
+            f'({self._input["rules"]["main_formula"]})|float'
+        )
+
+        _data = sorted([_data[x] for x in _data], key=lambda x: x["datetime"])
+        hourly = []
+        for x in _data:
+            x.update(self._input["rules"])
+            tariff = utils.get_pvpc_tariff(x["datetime"])
+            if "kwh_eur" not in x:
+                if tariff == "p1":
+                    x["kwh_eur"] = x["p1_kwh_eur"]
+                elif tariff == "p2":
+                    x["kwh_eur"] = x["p2_kwh_eur"]
+                elif tariff == "p3":
+                    x["kwh_eur"] = x["p3_kwh_eur"]
+
+                if x["kwh_eur"] is None:
+                    continue
+
+            if tariff == "p1":
+                x["surplus_kwh_eur"] = x["surplus_p1_kwh_eur"]
+            elif tariff == "p2":
+                x["surplus_kwh_eur"] = x["surplus_p2_kwh_eur"]
+            elif tariff == "p3":
+                x["surplus_kwh_eur"] = x["surplus_p3_kwh_eur"]
+
+            _energy_term = 0
+            _power_term = 0
+            _others_term = 0
+            _surplus_term = 0
+
+            with contextlib.suppress(Exception):
+                _energy_term = round(energy_expr(**x), 6)
+                _power_term = round(power_expr(**x), 6)
+                _others_term = round(others_expr(**x), 6)
+                _surplus_term = round(surplus_expr(**x), 6)
+
+            new_item = PricingAggData(
+                datetime=x["datetime"],
+                energy_term=_energy_term,
+                power_term=_power_term,
+                others_term=_others_term,
+                surplus_term=_surplus_term,
+                value_eur=0,
+                delta_h=1,
+            )
+            hourly.append(new_item)
+
+        self._output["hourly"] = hourly
+
+        last_day_dt = None
+        last_month_dt = None
+        for hour in hourly:
+            curr_hour_dt: datetime = hour["datetime"]
+            curr_day_dt = curr_hour_dt.replace(hour=0, minute=0, second=0)
+            curr_month_dt = (curr_day_dt - timedelta(days=self._cycle_offset)).replace(
+                day=1
+            )
+
+            if last_day_dt is None or curr_day_dt != last_day_dt:
+                self._output["daily"].append(
+                    PricingAggData(
+                        datetime=curr_day_dt,
+                        energy_term=hour["energy_term"],
+                        power_term=hour["power_term"],
+                        others_term=hour["others_term"],
+                        surplus_term=hour["surplus_term"],
+                        value_eur=hour["value_eur"],
+                        delta_h=hour["delta_h"],
+                    )
+                )
+            else:
+                self._output["daily"][-1]["energy_term"] += hour["energy_term"]
+                self._output["daily"][-1]["power_term"] += hour["power_term"]
+                self._output["daily"][-1]["others_term"] += hour["others_term"]
+                self._output["daily"][-1]["surplus_term"] += hour["surplus_term"]
+                self._output["daily"][-1]["delta_h"] += hour["delta_h"]
+                self._output["daily"][-1]["value_eur"] += hour["value_eur"]
+
+            if last_month_dt is None or curr_month_dt != last_month_dt:
+                self._output["monthly"].append(
+                    PricingAggData(
+                        datetime=curr_month_dt,
+                        energy_term=hour["energy_term"],
+                        power_term=hour["power_term"],
+                        others_term=hour["others_term"],
+                        surplus_term=hour["surplus_term"],
+                        value_eur=hour["value_eur"],
+                        delta_h=hour["delta_h"],
+                    )
+                )
+            else:
+                self._output["monthly"][-1]["energy_term"] += hour["energy_term"]
+                self._output["monthly"][-1]["power_term"] += hour["power_term"]
+                self._output["monthly"][-1]["others_term"] += hour["others_term"]
+                self._output["monthly"][-1]["surplus_term"] += hour["surplus_term"]
+                self._output["monthly"][-1]["value_eur"] += hour["value_eur"]
+                self._output["monthly"][-1]["delta_h"] += hour["delta_h"]
+
+            last_day_dt = curr_day_dt
+            last_month_dt = curr_month_dt
+
+        for item in self._output:
+            for cost in self._output[item]:
+                cost["value_eur"] = round(main_expr(**cost, **self._input["rules"]), 6)
+                cost["energy_term"] = round(cost["energy_term"], 6)
+                cost["power_term"] = round(cost["power_term"], 6)
+                cost["others_term"] = round(cost["others_term"], 6)
+                cost["surplus_term"] = round(cost["surplus_term"], 6)

processors/consumption.py

@@ -0,0 +1,139 @@
+"""Consumption data processors."""
+
+import logging
+from collections.abc import Iterable
+from typing import TypedDict
+from datetime import datetime, timedelta
+
+import voluptuous
+
+from ..definitions import ConsumptionAggData, ConsumptionSchema
+from . import utils
+from .base import Processor
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class ConsumptionOutput(TypedDict):
+    """A dict holding ConsumptionProcessor output property."""
+
+    daily: Iterable[ConsumptionAggData]
+    monthly: Iterable[ConsumptionAggData]
+
+
+class ConsumptionProcessor(Processor):
+    """A consumptions processor."""
+
+    def do_process(self):
+        """Calculate daily and monthly consumption stats."""
+
+        self._output = ConsumptionOutput(daily=[], monthly=[])
+
+        last_day_dt = None
+        last_month_dt = None
+
+        _schema = voluptuous.Schema(
+            {
+                voluptuous.Required("consumptions"): [ConsumptionSchema],
+                voluptuous.Optional("cycle_start_day", default=1): voluptuous.Range(
+                    1, 30
+                ),
+            }
+        )
+        self._input = _schema(self._input)
+
+        self._cycle_offset = self._input["cycle_start_day"] - 1
+
+        for consumption in self._input["consumptions"]:
+            curr_hour_dt: datetime = consumption["datetime"]
+            curr_day_dt = curr_hour_dt.replace(hour=0, minute=0, second=0)
+            curr_month_dt = (curr_day_dt - timedelta(days=self._cycle_offset)).replace(
+                day=1
+            )
+
+            tariff = utils.get_pvpc_tariff(curr_hour_dt)
+            kwh = consumption["value_kWh"]
+            surplus_kwh = consumption["surplus_kWh"]
+            delta_h = consumption["delta_h"]
+
+            kwh_by_tariff = [0, 0, 0]
+            surplus_kwh_by_tariff = [0, 0, 0]
+
+            match tariff:
+                case "p1":
+                    kwh_by_tariff[0] = kwh
+                    surplus_kwh_by_tariff[0] = surplus_kwh
+                case "p2":
+                    kwh_by_tariff[1] = kwh
+                    surplus_kwh_by_tariff[1] = surplus_kwh
+                case "p3":
+                    kwh_by_tariff[2] = kwh
+                    surplus_kwh_by_tariff[2] = surplus_kwh
+
+            if last_day_dt is None or curr_day_dt != last_day_dt:
+                self._output["daily"].append(
+                    ConsumptionAggData(
+                        datetime=curr_day_dt,
+                        value_kWh=kwh,
+                        delta_h=delta_h,
+                        value_p1_kWh=kwh_by_tariff[0],
+                        value_p2_kWh=kwh_by_tariff[1],
+                        value_p3_kWh=kwh_by_tariff[2],
+                        surplus_kWh=surplus_kwh,
+                        surplus_p1_kWh=surplus_kwh_by_tariff[0],
+                        surplus_p2_kWh=surplus_kwh_by_tariff[1],
+                        surplus_p3_kWh=surplus_kwh_by_tariff[2],
+                    )
+                )
+            else:
+                self._output["daily"][-1]["value_kWh"] += kwh
+                self._output["daily"][-1]["value_p1_kWh"] += kwh_by_tariff[0]
+                self._output["daily"][-1]["value_p2_kWh"] += kwh_by_tariff[1]
+                self._output["daily"][-1]["value_p3_kWh"] += kwh_by_tariff[2]
+                self._output["daily"][-1]["surplus_kWh"] += surplus_kwh
+                self._output["daily"][-1]["surplus_p1_kWh"] += surplus_kwh_by_tariff[0]
+                self._output["daily"][-1]["surplus_p2_kWh"] += surplus_kwh_by_tariff[1]
+                self._output["daily"][-1]["surplus_p3_kWh"] += surplus_kwh_by_tariff[2]
+                self._output["daily"][-1]["delta_h"] += delta_h
+
+            if last_month_dt is None or curr_month_dt != last_month_dt:
+                self._output["monthly"].append(
+                    ConsumptionAggData(
+                        datetime=curr_month_dt,
+                        value_kWh=kwh,
+                        delta_h=delta_h,
+                        value_p1_kWh=kwh_by_tariff[0],
+                        value_p2_kWh=kwh_by_tariff[1],
+                        value_p3_kWh=kwh_by_tariff[2],
+                        surplus_kWh=surplus_kwh,
+                        surplus_p1_kWh=surplus_kwh_by_tariff[0],
+                        surplus_p2_kWh=surplus_kwh_by_tariff[1],
+                        surplus_p3_kWh=surplus_kwh_by_tariff[2],
+                    )
+                )
+            else:
+                self._output["monthly"][-1]["value_kWh"] += kwh
+                self._output["monthly"][-1]["value_p1_kWh"] += kwh_by_tariff[0]
+                self._output["monthly"][-1]["value_p2_kWh"] += kwh_by_tariff[1]
+                self._output["monthly"][-1]["value_p3_kWh"] += kwh_by_tariff[2]
+                self._output["monthly"][-1]["surplus_kWh"] += surplus_kwh
+                self._output["monthly"][-1]["surplus_p1_kWh"] += surplus_kwh_by_tariff[
+                    0
+                ]
+                self._output["monthly"][-1]["surplus_p2_kWh"] += surplus_kwh_by_tariff[
+                    1
+                ]
+                self._output["monthly"][-1]["surplus_p3_kWh"] += surplus_kwh_by_tariff[
+                    2
+                ]
+                self._output["monthly"][-1]["delta_h"] += delta_h
+
+            last_day_dt = curr_day_dt
+            last_month_dt = curr_month_dt
+
+        # Round to two decimals
+        for item in self._output:
+            for cons in self._output[item]:
+                for key in cons:
+                    if isinstance(cons[key], float):
+                        cons[key] = round(cons[key], 2)

processors/maximeter.py

@@ -0,0 +1,56 @@
+"""Maximeter data processors."""
+
+import logging
+from datetime import datetime
+from typing import TypedDict
+
+from dateparser import parse
+import voluptuous
+
+from edata.definitions import MaxPowerSchema
+from edata.processors import utils
+
+from .base import Processor
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class MaximeterStats(TypedDict):
+    """A dict holding MaximeterProcessor stats."""
+
+    value_max_kW: float
+    date_max: datetime
+    value_mean_kW: float
+    value_tile90_kW: float
+
+
+class MaximeterOutput(TypedDict):
+    """A dict holding MaximeterProcessor output property."""
+
+    stats: MaximeterStats
+
+
+class MaximeterProcessor(Processor):
+    """A processor for Maximeter data."""
+
+    def do_process(self):
+        """Calculate maximeter stats."""
+
+        self._output = MaximeterOutput(stats={})
+
+        _schema = voluptuous.Schema([MaxPowerSchema])
+        self._input = _schema(self._input)
+
+        _values = [x["value_kW"] for x in self._input]
+
+        _max_kW = max(_values)
+        _dt_max_kW = parse(str(self._input[_values.index(_max_kW)]["datetime"]))
+        _mean_kW = sum(_values) / len(_values)
+        _tile90_kW = utils.percentile(_values, 0.9)
+
+        self._output["stats"] = MaximeterOutput(
+            value_max_kW=round(_max_kW, 2),
+            date_max=_dt_max_kW,
+            value_mean_kW=round(_mean_kW, 2),
+            value_tile90_kW=round(_tile90_kW, 2),
+        )

processors/utils.py

@@ -0,0 +1,148 @@
+"""Generic utilities for processing data."""
+
+import json
+import logging
+from copy import deepcopy
+from datetime import date, datetime, timedelta
+from json import JSONEncoder
+
+import holidays
+
+import math
+import functools
+
+import contextlib
+
+_LOGGER = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+HOURS_P1 = [10, 11, 12, 13, 18, 19, 20, 21]
+HOURS_P2 = [8, 9, 14, 15, 16, 17, 22, 23]
+WEEKDAYS_P3 = [5, 6]
+
+
+def is_empty(lst):
+    """Check if a list is empty."""
+    return len(lst) == 0
+
+
+def extract_dt_ranges(lst, dt_from, dt_to, gap_interval=timedelta(hours=1)):
+    """Filter a list of dicts between two datetimes."""
+    new_lst = []
+    missing = []
+    oldest_dt = None
+    newest_dt = None
+    last_dt = None
+    if len(lst) > 0:
+        sorted_lst = sorted(lst, key=lambda i: i["datetime"])
+        last_dt = dt_from
+        for i in sorted_lst:
+            if dt_from <= i["datetime"] <= dt_to:
+                if (i["datetime"] - last_dt) > gap_interval:
+                    missing.append({"from": last_dt, "to": i["datetime"]})
+                if i.get("value_kWh", 1) > 0:
+                    if oldest_dt is None or i["datetime"] < oldest_dt:
+                        oldest_dt = i["datetime"]
+                    if newest_dt is None or i["datetime"] > newest_dt:
+                        newest_dt = i["datetime"]
+                if i["datetime"] != last_dt:  # remove duplicates
+                    new_lst.append(i)
+                    last_dt = i["datetime"]
+        if dt_to > last_dt:
+            missing.append({"from": last_dt, "to": dt_to})
+        _LOGGER.debug("found data from %s to %s", oldest_dt, newest_dt)
+    else:
+        missing.append({"from": dt_from, "to": dt_to})
+    return new_lst, missing
+
+
+def extend_by_key(old_lst, new_lst, key):
+    """Extend a list of dicts by key."""
+    lst = deepcopy(old_lst)
+    temp_list = []
+    for new_element in new_lst:
+        for old_element in lst:
+            if new_element[key] == old_element[key]:
+                for i in old_element:
+                    old_element[i] = new_element[i]
+                break
+        else:
+            temp_list.append(new_element)
+    lst.extend(temp_list)
+    return lst
+
+
+def extend_and_filter(old_lst, new_lst, key, dt_from, dt_to):
+    data = extend_by_key(old_lst, new_lst, key)
+    data, _ = extract_dt_ranges(
+        data,
+        dt_from,
+        dt_to,
+        gap_interval=timedelta(days=365),  # trick
+    )
+
+    return data
+
+
+def get_by_key(lst, key, value):
+    """Obtain an element of a list of dicts by key=value."""
+    for i in lst:
+        if i[key] == value:
+            return i
+    return None
+
+
+def get_pvpc_tariff(a_datetime):
+    """Evals the PVPC tariff for a given datetime."""
+    hdays = holidays.country_holidays("ES")
+    hour = a_datetime.hour
+    weekday = a_datetime.weekday()
+    if weekday in WEEKDAYS_P3 or a_datetime.date() in hdays:
+        return "p3"
+    elif hour in HOURS_P1:
+        return "p1"
+    elif hour in HOURS_P2:
+        return "p2"
+    else:
+        return "p3"
+
+
+def serialize_dict(data: dict) -> dict:
+    """Serialize dicts as json."""
+
+    class DateTimeEncoder(JSONEncoder):
+        """Replace datetime objects with ISO strings."""
+
+        def default(self, o):
+            if isinstance(o, (date, datetime)):
+                return o.isoformat()
+
+    return json.loads(json.dumps(data, cls=DateTimeEncoder))
+
+
+def deserialize_dict(serialized_dict: dict) -> dict:
+    """Deserializes a json replacing ISOTIME strings into datetime."""
+
+    def datetime_parser(json_dict):
+        """Parse JSON while converting ISO strings into datetime objects."""
+        for key, value in json_dict.items():
+            if "date" in key:
+                with contextlib.suppress(Exception):
+                    json_dict[key] = datetime.fromisoformat(value)
+        return json_dict
+
+    return json.loads(json.dumps(serialized_dict), object_hook=datetime_parser)
+
+
+def percentile(N, percent, key=lambda x: x):
+    """Find the percentile of a list of values."""
+    if not N:
+        return None
+    k = (len(N) - 1) * percent
+    f = math.floor(k)
+    c = math.ceil(k)
+    if f == c:
+        return key(N[int(k)])
+    d0 = key(N[int(f)]) * (c - k)
+    d1 = key(N[int(c)]) * (k - f)
+    return d0 + d1