e-data 1.3.1__tar.gz → 1.3.2__tar.gz

{e_data-1.3.1/edata/e_data.egg-info → e_data-1.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: e-data
-Version: 1.3.1
+Version: 1.3.2
 Summary: Python library for managing spanish energy data from various web providers
 Author-email: VMG <vmayorg@outlook.es>
 License:                     GNU GENERAL PUBLIC LICENSE

{e_data-1.3.1 → e_data-1.3.2/e_data.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: e-data
-Version: 1.3.1
+Version: 1.3.2
 Summary: Python library for managing spanish energy data from various web providers
 Author-email: VMG <vmayorg@outlook.es>
 License:                     GNU GENERAL PUBLIC LICENSE

{e_data-1.3.1/edata → e_data-1.3.2}/e_data.egg-info/SOURCES.txt

@@ -2,14 +2,19 @@ LICENSE
 MANIFEST.in
 README.md
 pyproject.toml
+e_data.egg-info/PKG-INFO
+e_data.egg-info/SOURCES.txt
+e_data.egg-info/dependency_links.txt
+e_data.egg-info/requires.txt
+e_data.egg-info/top_level.txt
+edata/__init__.py
+edata/const.py
+edata/definitions.py
+edata/helpers.py
+edata/storage.py
 edata/connectors/__init__.py
 edata/connectors/datadis.py
 edata/connectors/redata.py
-edata/e_data.egg-info/PKG-INFO
-edata/e_data.egg-info/SOURCES.txt
-edata/e_data.egg-info/dependency_links.txt
-edata/e_data.egg-info/requires.txt
-edata/e_data.egg-info/top_level.txt
 edata/processors/__init__.py
 edata/processors/base.py
 edata/processors/billing.py

e_data-1.3.2/e_data.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+dist
+edata

e_data-1.3.2/edata/const.py

@@ -0,0 +1,3 @@
+"""Constants file."""
+
+PROG_NAME = "edata"

e_data-1.3.2/edata/definitions.py

@@ -0,0 +1,329 @@
+"""Definitions for data structures."""
+
+import voluptuous as vol
+import datetime as dt
+from typing import TypedDict
+
+ATTRIBUTES = {
+    "cups": None,
+    "contract_p1_kW": "kW",
+    "contract_p2_kW": "kW",
+    "yesterday_kWh": "kWh",
+    "yesterday_hours": "h",
+    "yesterday_p1_kWh": "kWh",
+    "yesterday_p2_kWh": "kWh",
+    "yesterday_p3_kWh": "kWh",
+    "yesterday_surplus_kWh": "kWh",
+    "yesterday_surplus_p1_kWh": "kWh",
+    "yesterday_surplus_p2_kWh": "kWh",
+    "yesterday_surplus_p3_kWh": "kWh",
+    "last_registered_date": None,
+    "last_registered_day_kWh": "kWh",
+    "last_registered_day_hours": "h",
+    "last_registered_day_p1_kWh": "kWh",
+    "last_registered_day_p2_kWh": "kWh",
+    "last_registered_day_p3_kWh": "kWh",
+    "last_registered_day_surplus_kWh": "kWh",
+    "last_registered_day_surplus_p1_kWh": "kWh",
+    "last_registered_day_surplus_p2_kWh": "kWh",
+    "last_registered_day_surplus_p3_kWh": "kWh",
+    "month_kWh": "kWh",
+    "month_daily_kWh": "kWh",
+    "month_days": "d",
+    "month_p1_kWh": "kWh",
+    "month_p2_kWh": "kWh",
+    "month_p3_kWh": "kWh",
+    "month_surplus_kWh": "kWh",
+    "month_surplus_p1_kWh": "kWh",
+    "month_surplus_p2_kWh": "kWh",
+    "month_surplus_p3_kWh": "kWh",
+    "month_€": "€",
+    "last_month_kWh": "kWh",
+    "last_month_daily_kWh": "kWh",
+    "last_month_days": "d",
+    "last_month_p1_kWh": "kWh",
+    "last_month_p2_kWh": "kWh",
+    "last_month_p3_kWh": "kWh",
+    "last_month_surplus_kWh": "kWh",
+    "last_month_surplus_p1_kWh": "kWh",
+    "last_month_surplus_p2_kWh": "kWh",
+    "last_month_surplus_p3_kWh": "kWh",
+    "last_month_€": "€",
+    "max_power_kW": "kW",
+    "max_power_date": None,
+    "max_power_mean_kW": "kW",
+    "max_power_90perc_kW": "kW",
+}
+
+# Energy term with taxes
+DEFAULT_BILLING_ENERGY_FORMULA = "electricity_tax * iva_tax * kwh_eur * kwh"
+
+# Power term with taxes
+DEFAULT_BILLING_POWER_FORMULA = "electricity_tax * iva_tax * (p1_kw * (p1_kw_year_eur + market_kw_year_eur) + p2_kw * p2_kw_year_eur) / 365 / 24"
+
+# Others term with taxes
+DEFAULT_BILLING_OTHERS_FORMULA = "iva_tax * meter_month_eur / 30 / 24"
+
+# Surplus term with taxes
+DEFAULT_BILLING_SURPLUS_FORMULA = (
+    "electricity_tax * iva_tax * surplus_kwh * surplus_kwh_eur"
+)
+
+# Sum energy and power terms, and substract surplus until 0.
+# An alternative would be "[(energy_term + power_term - surplus_term), 0]|max + others_term"
+DEFAULT_BILLING_MAIN_FORMULA = "energy_term + power_term + others_term"
+
+
+class SupplyData(TypedDict):
+    """Data structure to represent a supply."""
+
+    cups: str
+    date_start: dt.datetime
+    date_end: dt.datetime
+    address: str | None
+    postal_code: str | None
+    province: str | None
+    municipality: str | None
+    distributor: str | None
+    pointType: int
+    distributorCode: str
+
+
+SupplySchema = vol.Schema(
+    {
+        vol.Required("cups"): str,
+        vol.Required("date_start"): dt.datetime,
+        vol.Required("date_end"): dt.datetime,
+        vol.Required("address"): vol.Union(str, None),
+        vol.Required("postal_code"): vol.Union(str, None),
+        vol.Required("province"): vol.Union(str, None),
+        vol.Required("municipality"): vol.Union(str, None),
+        vol.Required("distributor"): vol.Union(str, None),
+        vol.Required("pointType"): int,
+        vol.Required("distributorCode"): str,
+    }
+)
+
+
+class ContractData(TypedDict):
+    """Data structure to represent a contract."""
+
+    date_start: dt.datetime
+    date_end: dt.datetime
+    marketer: str
+    distributorCode: str
+    power_p1: float | None
+    power_p2: float | None
+
+
+ContractSchema = vol.Schema(
+    {
+        vol.Required("date_start"): dt.datetime,
+        vol.Required("date_end"): dt.datetime,
+        vol.Required("marketer"): str,
+        vol.Required("distributorCode"): str,
+        vol.Required("power_p1"): vol.Union(vol.Coerce(float), None),
+        vol.Required("power_p2"): vol.Union(vol.Coerce(float), None),
+    }
+)
+
+
+class ConsumptionData(TypedDict):
+    """Data structure to represent a consumption."""
+
+    datetime: dt.datetime
+    delta_h: float
+    value_kWh: float
+    surplus_kWh: float
+    real: bool
+
+
+ConsumptionSchema = vol.Schema(
+    {
+        vol.Required("datetime"): dt.datetime,
+        vol.Required("delta_h"): vol.Coerce(float),
+        vol.Required("value_kWh"): vol.Coerce(float),
+        vol.Optional("surplus_kWh", default=0): vol.Coerce(float),
+        vol.Required("real"): bool,
+    }
+)
+
+
+class MaxPowerData(TypedDict):
+    """Data structure to represent a MaxPower."""
+
+    datetime: dt.datetime
+    value_kW: float
+
+
+MaxPowerSchema = vol.Schema(
+    {
+        vol.Required("datetime"): dt.datetime,
+        vol.Required("value_kW"): vol.Coerce(float),
+    }
+)
+
+
+class PricingData(TypedDict):
+    """Data structure to represent pricing data."""
+
+    datetime: dt.datetime
+    value_eur_kWh: float
+    delta_h: float
+
+
+PricingSchema = vol.Schema(
+    {
+        vol.Required("datetime"): dt.datetime,
+        vol.Required("value_eur_kWh"): vol.Coerce(float),
+        vol.Required("delta_h"): vol.Coerce(float),
+    }
+)
+
+
+class PricingRules(TypedDict):
+    """Data structure to represent custom pricing rules."""
+
+    p1_kw_year_eur: float
+    p2_kw_year_eur: float
+    p1_kwh_eur: float | None
+    p2_kwh_eur: float | None
+    p3_kwh_eur: float | None
+    surplus_p1_kwh_eur: float | None
+    surplus_p2_kwh_eur: float | None
+    surplus_p3_kwh_eur: float | None
+    meter_month_eur: float
+    market_kw_year_eur: float
+    electricity_tax: float
+    iva_tax: float
+    energy_formula: str | None
+    power_formula: str | None
+    others_formula: str | None
+    surplus_formula: str | None
+    cycle_start_day: int | None
+
+
+PricingRulesSchema = vol.Schema(
+    {
+        vol.Required("p1_kw_year_eur"): vol.Coerce(float),
+        vol.Required("p2_kw_year_eur"): vol.Coerce(float),
+        vol.Optional("p1_kwh_eur", default=None): vol.Union(vol.Coerce(float), None),
+        vol.Optional("p2_kwh_eur", default=None): vol.Union(vol.Coerce(float), None),
+        vol.Optional("p3_kwh_eur", default=None): vol.Union(vol.Coerce(float), None),
+        vol.Optional("surplus_p1_kwh_eur", default=None): vol.Union(
+            vol.Coerce(float), None
+        ),
+        vol.Optional("surplus_p2_kwh_eur", default=None): vol.Union(
+            vol.Coerce(float), None
+        ),
+        vol.Optional("surplus_p3_kwh_eur", default=None): vol.Union(
+            vol.Coerce(float), None
+        ),
+        vol.Required("meter_month_eur"): vol.Coerce(float),
+        vol.Required("market_kw_year_eur"): vol.Coerce(float),
+        vol.Required("electricity_tax"): vol.Coerce(float),
+        vol.Required("iva_tax"): vol.Coerce(float),
+        vol.Optional("energy_formula", default=DEFAULT_BILLING_ENERGY_FORMULA): str,
+        vol.Optional("power_formula", default=DEFAULT_BILLING_POWER_FORMULA): str,
+        vol.Optional("others_formula", default=DEFAULT_BILLING_OTHERS_FORMULA): str,
+        vol.Optional("surplus_formula", default=DEFAULT_BILLING_SURPLUS_FORMULA): str,
+        vol.Optional("main_formula", default=DEFAULT_BILLING_MAIN_FORMULA): str,
+        vol.Optional("cycle_start_day", default=1): vol.Range(1, 30),
+    }
+)
+
+DEFAULT_PVPC_RULES = PricingRules(
+    p1_kw_year_eur=30.67266,
+    p2_kw_year_eur=1.4243591,
+    meter_month_eur=0.81,
+    market_kw_year_eur=3.113,
+    electricity_tax=1.0511300560,
+    iva_tax=1.05,
+)
+
+
+class ConsumptionAggData(TypedDict):
+    """A dict holding a Consumption item."""
+
+    datetime: dt.datetime
+    value_kWh: float
+    value_p1_kWh: float
+    value_p2_kWh: float
+    value_p3_kWh: float
+    surplus_kWh: float
+    surplus_p1_kWh: float
+    surplus_p2_kWh: float
+    surplus_p3_kWh: float
+    delta_h: float
+
+
+ConsumptionAggSchema = vol.Schema(
+    {
+        vol.Required("datetime"): dt.datetime,
+        vol.Required("value_kWh"): vol.Coerce(float),
+        vol.Required("value_p1_kWh"): vol.Coerce(float),
+        vol.Required("value_p2_kWh"): vol.Coerce(float),
+        vol.Required("value_p3_kWh"): vol.Coerce(float),
+        vol.Optional("surplus_kWh", default=0): vol.Coerce(float),
+        vol.Optional("surplus_p1_kWh", default=0): vol.Coerce(float),
+        vol.Optional("surplus_p2_kWh", default=0): vol.Coerce(float),
+        vol.Optional("surplus_p3_kWh", default=0): vol.Coerce(float),
+        vol.Required("delta_h"): vol.Coerce(float),
+    }
+)
+
+
+class PricingAggData(TypedDict):
+    """A dict holding a Billing item."""
+
+    datetime: dt.datetime
+    value_eur: float
+    energy_term: float
+    power_term: float
+    others_term: float
+    surplus_term: float
+    delta_h: float
+
+
+PricingAggSchema = vol.Schema(
+    {
+        vol.Required("datetime"): dt.datetime,
+        vol.Required("value_eur"): vol.Coerce(float),
+        vol.Required("energy_term"): vol.Coerce(float),
+        vol.Required("power_term"): vol.Coerce(float),
+        vol.Required("others_term"): vol.Coerce(float),
+        vol.Optional("surplus_term", default=0): vol.Coerce(float),
+        vol.Optional("delta_h", default=1): vol.Coerce(float),
+    },
+)
+
+
+class EdataData(TypedDict):
+    """A Typed Dict to handle Edata Aggregated Data."""
+
+    supplies: list[SupplyData]
+    contracts: list[ContractData]
+    consumptions: list[ConsumptionData]
+    maximeter: list[MaxPowerData]
+    pvpc: list[PricingData]
+    consumptions_daily_sum: list[ConsumptionAggData]
+    consumptions_monthly_sum: list[ConsumptionAggData]
+    cost_hourly_sum: list[PricingAggData]
+    cost_daily_sum: list[PricingAggData]
+    cost_monthly_sum: list[PricingAggData]
+
+
+EdataSchema = vol.Schema(
+    {
+        vol.Required("supplies"): [SupplySchema],
+        vol.Required("contracts"): [ContractSchema],
+        vol.Required("consumptions"): [ConsumptionSchema],
+        vol.Required("maximeter"): [MaxPowerSchema],
+        vol.Optional("pvpc", default=[]): [PricingSchema],
+        vol.Optional("consumptions_daily_sum", []): [ConsumptionAggSchema],
+        vol.Optional("consumptions_monthly_sum", []): [ConsumptionAggSchema],
+        vol.Optional("cost_hourly_sum", default=[]): [PricingAggSchema],
+        vol.Optional("cost_daily_sum", default=[]): [PricingAggSchema],
+        vol.Optional("cost_monthly_sum", default=[]): [PricingAggSchema],
+    }
+)

e_data-1.3.2/edata/helpers.py

@@ -0,0 +1,769 @@
+"""A module for edata helpers."""
+
+import asyncio
+import contextlib
+from datetime import datetime, timedelta
+import logging
+import os
+
+from dateutil.relativedelta import relativedelta
+import requests
+
+from . import const
+from .connectors.datadis import DatadisConnector
+from .connectors.redata import REDataConnector
+from .definitions import ATTRIBUTES, EdataData, PricingRules
+from .processors import utils
+from .processors.billing import BillingInput, BillingProcessor
+from .processors.consumption import ConsumptionProcessor
+from .processors.maximeter import MaximeterProcessor
+from .storage import check_storage_integrity, dump_storage, load_storage
+
+_LOGGER = logging.getLogger(__name__)
+
+
+def acups(cups):
+    """Print an abbreviated and anonymized CUPS."""
+
+    return cups[-5:]
+
+
+class EdataHelper:
+    """Main EdataHelper class."""
+
+    UPDATE_INTERVAL = timedelta(hours=1)
+
+    def __init__(
+        self,
+        datadis_username: str,
+        datadis_password: str,
+        cups: str,
+        datadis_authorized_nif: str | None = None,
+        pricing_rules: PricingRules | None = None,
+        storage_dir_path: str | None = None,
+        data: EdataData | None = None,
+    ) -> None:
+        self.data = EdataData(
+            supplies=[],
+            contracts=[],
+            consumptions=[],
+            maximeter=[],
+            pvpc=[],
+            consumptions_daily_sum=[],
+            consumptions_monthly_sum=[],
+            cost_hourly_sum=[],
+            cost_daily_sum=[],
+            cost_monthly_sum=[],
+        )
+
+        self.attributes = {}
+        self._storage_dir = storage_dir_path
+        self._cups = cups
+        self._scups = acups(cups)
+        self._authorized_nif = datadis_authorized_nif
+        self.last_update = {x: datetime(1970, 1, 1) for x in self.data}
+        self._date_from = datetime(1970, 1, 1)
+        self._date_to = datetime.today()
+        self._must_dump = True
+        self._incremental_update = True
+
+        if data is not None:
+            data = check_storage_integrity(data)
+            self.data = data
+        else:
+            with contextlib.suppress(Exception):
+                self.data = load_storage(self._cups, self._storage_dir)
+
+        for attr in ATTRIBUTES:
+            self.attributes[attr] = None
+
+        self.datadis_api = DatadisConnector(
+            datadis_username,
+            datadis_password,
+            storage_path=(
+                os.path.join(storage_dir_path, const.PROG_NAME)
+                if storage_dir_path is not None
+                else None
+            ),
+        )
+        self.redata_api = REDataConnector()
+
+        self.pricing_rules = pricing_rules
+
+        if self.pricing_rules is not None:
+            self.enable_billing = True
+            if not all(
+                x in self.pricing_rules and self.pricing_rules[x] is not None
+                for x in ("p1_kwh_eur", "p2_kwh_eur", "p3_kwh_eur")
+            ):
+                self.is_pvpc = True
+            else:
+                self.is_pvpc = False
+        else:
+            self.enable_billing = False
+            self.is_pvpc = False
+
+    async def async_update(
+        self,
+        date_from: datetime = datetime(1970, 1, 1),
+        date_to: datetime = datetime.today(),
+        incremental_update: bool = True,
+    ):
+        """Async call of update method."""
+        
+        _LOGGER.info(
+            "%s: update triggered",
+            self._scups,
+        )
+        self._date_from = date_from
+        self._date_to = date_to
+
+        # update datadis resources
+        await self.update_datadis(self._cups, date_from, date_to)
+
+        # update redata resources if pvpc is requested
+        if self.is_pvpc:
+            try:
+                await asyncio.to_thread(self.update_redata, date_from, date_to)
+            except requests.exceptions.Timeout:
+                _LOGGER.error("Timeout exception while updating from REData")
+
+        await asyncio.to_thread(self.process_data, incremental_update=incremental_update)
+
+        if self._must_dump:
+            await asyncio.to_thread(dump_storage, self._cups, self.data, self._storage_dir)
+
+    def update(
+        self,
+        date_from: datetime = datetime(1970, 1, 1),
+        date_to: datetime = datetime.today(),
+        incremental_update: bool = True,
+    ):
+        """Update synchronously."""
+
+        asyncio.run(self.async_update(date_from, date_to, incremental_update))
+
+    async def update_supplies(self):
+        """Update supplies."""
+
+        _LOGGER.debug("%s: supplies update triggered", self._scups)
+        if datetime.today().date() != self.last_update["supplies"].date():
+            # if supplies haven't been updated today
+            supplies = await self.datadis_api.async_get_supplies(
+                authorized_nif=self._authorized_nif
+            )  # fetch supplies
+            if len(supplies) > 0:
+                self.data["supplies"] = supplies
+                # if we got something, update last_update flag
+                self.last_update["supplies"] = datetime.now()
+                _LOGGER.info("%s: supplies update succeeded", self._scups)
+        else:
+            _LOGGER.info("%s: supplies are already updated (skipping)", self._scups)
+
+    async def update_contracts(self, cups: str, distributor_code: str):
+        """Update contracts."""
+
+        _LOGGER.debug("%s: contracts update triggered", self._scups)
+        if datetime.today().date() != self.last_update["contracts"].date():
+            # if contracts haven't been updated today
+            contracts = await self.datadis_api.async_get_contract_detail(
+                cups, distributor_code, authorized_nif=self._authorized_nif
+            )
+            if len(contracts) > 0:
+                self.data["contracts"] = utils.extend_by_key(
+                    self.data["contracts"], contracts, "date_start"
+                )  # extend contracts data with new ones
+                # if we got something, update last_update flag
+                self.last_update["contracts"] = datetime.now()
+                _LOGGER.info("%s: contracts update succeeded", self._scups)
+        else:
+            _LOGGER.info("%s: contracts are already updated (skipping)", self._scups)
+
+    async def update_consumptions(
+        self,
+        cups: str,
+        distributor_code: str,
+        start_date: datetime,
+        end_date: datetime,
+        measurement_type: str,
+        point_type: int,
+    ):
+        """Update consumptions."""
+
+        _LOGGER.debug("%s: consumptions update triggered", self._scups)
+        if (datetime.now() - self.last_update["consumptions"]) > self.UPDATE_INTERVAL:
+            consumptions = await self.datadis_api.async_get_consumption_data(
+                cups,
+                distributor_code,
+                start_date,
+                end_date,
+                measurement_type,
+                point_type,
+                authorized_nif=self._authorized_nif,
+            )
+            if len(consumptions) > 0:
+                _LOGGER.info(
+                    "%s: got consumptions from %s to %s",
+                    self._scups,
+                    consumptions[0]["datetime"].isoformat(),
+                    consumptions[-1]["datetime"].isoformat(),
+                )
+                self.data["consumptions"] = utils.extend_by_key(
+                    self.data["consumptions"], consumptions, "datetime"
+                )
+                self.last_update["consumptions"] = datetime.now()
+            else:
+                _LOGGER.info("%s: consumptions are up to date", self._scups)
+        else:
+            _LOGGER.info("%s: consumptions are already updated (skipping)", self._scups)
+
+    async def update_maximeter(self, cups, distributor_code, start_date, end_date):
+        """Update maximeter."""
+
+        _LOGGER.debug("%s: maximeter update triggered", self._scups)
+        if (datetime.now() - self.last_update["maximeter"]) > self.UPDATE_INTERVAL:
+            maximeter = await self.datadis_api.async_get_max_power(
+                cups,
+                distributor_code,
+                start_date,
+                end_date,
+                authorized_nif=self._authorized_nif,
+            )
+            if len(maximeter) > 0:
+                _LOGGER.info(
+                    "%s: maximeter update succeeded",
+                    self._scups,
+                )
+                self.data["maximeter"] = utils.extend_by_key(
+                    self.data["maximeter"], maximeter, "datetime"
+                )
+                self.last_update["maximeter"] = datetime.now()
+            else:
+                _LOGGER.info("%s: maximeter is up to date", self._scups)
+        else:
+            _LOGGER.info("%s: maximeter is already updated (skipping)", self._scups)
+
+    async def update_datadis(
+        self,
+        cups: str,
+        date_from: datetime = datetime(1970, 1, 1),
+        date_to: datetime = datetime.today(),
+    ):
+        """Update all data from Datadis."""
+
+        _LOGGER.info(
+            "%s: datadis update triggered (from %s to %s)",
+            self._scups,
+            date_from.isoformat(),
+            date_to.isoformat(),
+        )
+
+        # update supplies and get distributorCode
+        await self.update_supplies()
+
+        if len(self.data["supplies"]) == 0:
+            # return if no supplies were discovered
+            _LOGGER.warning(
+                "%s: supplies update failed or no supplies found in the provided account",
+                self._scups,
+            )
+            return False
+
+        # find requested cups in supplies
+        supply = utils.get_by_key(self.data["supplies"], "cups", cups)
+        if supply is None:
+            # return if specified cups seems not valid
+            _LOGGER.error(
+                "%s: CUPS not found. Got: %s",
+                self._scups,
+                [acups(x["cups"]) for x in self.data["supplies"]],
+            )
+            return False
+        _LOGGER.info("%s: CUPS found in account", self._scups)
+
+        # get some supply-related data
+        supply_date_start = supply["date_start"]
+        distributor_code = supply["distributorCode"]
+        point_type = supply["pointType"]
+
+        _LOGGER.info(
+            "%s: CUPS start date is %s", self._scups, supply_date_start.isoformat()
+        )
+        _LOGGER.info(
+            "%s: CUPS end date is %s", self._scups, supply["date_end"].isoformat()
+        )
+
+        # update contracts to get valid periods
+        await self.update_contracts(cups, distributor_code)
+        if len(self.data["contracts"]) == 0:
+            _LOGGER.warning(
+                "%s: contracts update failed or no contracts found in the provided account",
+                self._scups,
+            )
+            # return False
+
+        # filter consumptions and maximeter, and log gaps
+        def sort_and_filter(dt_from, dt_to):
+            self.data["consumptions"], miss_cons = utils.extract_dt_ranges(
+                self.data["consumptions"],
+                dt_from,
+                dt_to,
+                gap_interval=timedelta(hours=6),
+            )
+            self.data["maximeter"], miss_maxim = utils.extract_dt_ranges(
+                self.data["maximeter"],
+                dt_from,
+                dt_to,
+                gap_interval=timedelta(days=60),
+            )
+            return miss_cons, miss_maxim
+
+        miss_cons, miss_maxim = sort_and_filter(date_from, date_to)
+
+        # update consumptions
+        _LOGGER.info(
+            "%s: missing consumptions: %s",
+            self._scups,
+            ", ".join(
+                [
+                    "from "
+                    + (x["from"] + timedelta(hours=1)).isoformat()
+                    + " to "
+                    + x["to"].isoformat()
+                    for x in miss_cons
+                ]
+            ),
+        )
+        for gap in miss_cons:
+            if not (
+                gap["to"] < supply["date_start"] or gap["from"] > supply["date_end"]
+            ):
+                # fetch consumptions for each consumptions gap in valid periods
+                start = max([gap["from"] + timedelta(hours=1), supply["date_start"]])
+                end = min([gap["to"], supply["date_end"]])
+                _LOGGER.info(
+                    "%s: requesting consumptions from %s to %s",
+                    self._scups,
+                    start.isoformat(),
+                    end.isoformat(),
+                )
+                await self.update_consumptions(
+                    cups,
+                    distributor_code,
+                    start,
+                    end,
+                    "0",
+                    point_type,
+                )
+
+        # update maximeter
+        _LOGGER.info(
+            "%s: missing maximeter: %s",
+            self._scups,
+            ", ".join(
+                [
+                    "from " + x["from"].isoformat() + " to " + x["to"].isoformat()
+                    for x in miss_maxim
+                ]
+            ),
+        )
+        for gap in miss_maxim:
+            if not (date_to < supply["date_start"] or date_from > supply["date_end"]):
+                # fetch maximeter for each maximeter gap in valid periods
+                start = max(
+                    [gap["from"], supply["date_start"] + relativedelta(months=1)]
+                )
+                end = min([gap["to"], supply["date_end"]])
+                start = min([start, end])
+                _LOGGER.info(
+                    "%s: requesting maximeter from %s to %s",
+                    self._scups,
+                    start.isoformat(),
+                    end.isoformat(),
+                )
+                await self.update_maximeter(cups, distributor_code, start, end)
+
+        miss_cons, miss_maxim = sort_and_filter(date_from, date_to)
+
+        return True
+
+    def update_redata(
+        self,
+        date_from: datetime = (datetime.today() - timedelta(days=30)).replace(
+            hour=0, minute=0
+        ),
+        date_to: datetime = (datetime.today() + timedelta(days=2)).replace(
+            hour=0, minute=0
+        ),
+    ):
+        """Fetch PVPC prices using REData API."""
+
+        _LOGGER.info(
+            "%s: updating PVPC prices",
+            self._scups,
+        )
+
+        self.data["pvpc"], missing = utils.extract_dt_ranges(
+            self.data["pvpc"],
+            date_from,
+            date_to,
+            gap_interval=timedelta(hours=1),
+        )
+        for gap in missing:
+            prices = []
+            gap["from"] = max(
+                (datetime.today() - timedelta(days=30)).replace(hour=0, minute=0),
+                gap["from"],
+            )
+            while len(prices) == 0 and gap["from"] < gap["to"]:
+                prices = self.redata_api.get_realtime_prices(gap["from"], gap["to"])
+                gap["from"] = gap["from"] + timedelta(days=1)
+            self.data["pvpc"] = utils.extend_by_key(
+                self.data["pvpc"], prices, "datetime"
+            )
+
+        return True
+
+    def process_data(self, incremental_update: bool = True):
+        """Process all raw data."""
+
+        self._incremental_update = incremental_update
+
+        for process_method in [
+            self.process_supplies,
+            self.process_contracts,
+            self.process_consumptions,
+            self.process_maximeter,
+            self.process_cost,
+        ]:
+            try:
+                process_method()
+            except Exception as ex:  # pylint: disable=broad-except
+                _LOGGER.error("Unhandled exception while updating attributes")
+                _LOGGER.exception(ex)
+
+        for attribute in self.attributes:
+            if attribute in ATTRIBUTES and ATTRIBUTES[attribute] is not None:
+                self.attributes[attribute] = (
+                    round(self.attributes[attribute], 2)
+                    if self.attributes[attribute] is not None
+                    else None
+                )
+
+        if not incremental_update:
+            dump_storage(self._cups, self.data, self._storage_dir)
+
+    def process_supplies(self):
+        """Process supplies data."""
+        for i in self.data["supplies"]:
+            if i["cups"] == self._cups:
+                self.attributes["cups"] = self._cups
+                break
+
+    def process_contracts(self):
+        """Process contracts data."""
+        most_recent_date = datetime(1970, 1, 1)
+        for i in self.data["contracts"]:
+            if i["date_end"] > most_recent_date:
+                most_recent_date = i["date_end"]
+                self.attributes["contract_p1_kW"] = i.get("power_p1", None)
+                self.attributes["contract_p2_kW"] = i.get("power_p2", None)
+                break
+
+    def process_consumptions(self):
+        """Process consumptions data."""
+        if len(self.data["consumptions"]) > 0:
+            new_data_from = self._date_from
+            if self._incremental_update:
+                with contextlib.suppress(Exception):
+                    new_data_from = self.data["consumptions_monthly_sum"][-1][
+                        "datetime"
+                    ]
+
+            proc = ConsumptionProcessor(
+                {
+                    "consumptions": [
+                        x
+                        for x in self.data["consumptions"]
+                        if x["datetime"] >= new_data_from
+                    ],
+                    "cycle_start_day": 1,
+                }
+            )
+            today_starts = datetime(
+                datetime.today().year,
+                datetime.today().month,
+                datetime.today().day,
+                0,
+                0,
+                0,
+            )
+
+            month_starts = datetime(
+                datetime.today().year, datetime.today().month, 1, 0, 0, 0
+            )
+
+            # append new data
+            self.data["consumptions_daily_sum"] = utils.extend_and_filter(
+                self.data["consumptions_daily_sum"],
+                proc.output["daily"],
+                "datetime",
+                self._date_from,
+                self._date_to,
+            )
+            self.data["consumptions_monthly_sum"] = utils.extend_and_filter(
+                self.data["consumptions_monthly_sum"],
+                proc.output["monthly"],
+                "datetime",
+                self._date_from,
+                self._date_to,
+            )
+
+            yday = utils.get_by_key(
+                self.data["consumptions_daily_sum"],
+                "datetime",
+                today_starts - timedelta(days=1),
+            )
+            self.attributes["yesterday_kWh"] = (
+                yday.get("value_kWh", None) if yday is not None else None
+            )
+
+            for tariff in (1, 2, 3):
+                self.attributes[f"yesterday_p{tariff}_kWh"] = (
+                    yday.get(f"value_p{tariff}_kWh", None) if yday is not None else None
+                )
+
+            self.attributes["yesterday_surplus_kWh"] = (
+                yday.get("surplus_kWh", None) if yday is not None else None
+            )
+
+            for tariff in (1, 2, 3):
+                self.attributes[f"yesterday_surplus_p{tariff}_kWh"] = (
+                    yday.get(f"surplus_p{tariff}_kWh", None)
+                    if yday is not None
+                    else None
+                )
+
+            self.attributes["yesterday_hours"] = (
+                yday.get("delta_h", None) if yday is not None else None
+            )
+
+            month = utils.get_by_key(
+                self.data["consumptions_monthly_sum"], "datetime", month_starts
+            )
+            self.attributes["month_kWh"] = (
+                month.get("value_kWh", None) if month is not None else None
+            )
+            self.attributes["month_surplus_kWh"] = (
+                month.get("surplus_kWh", None) if month is not None else None
+            )
+            self.attributes["month_days"] = (
+                month.get("delta_h", 0) / 24 if month is not None else None
+            )
+            self.attributes["month_daily_kWh"] = (
+                (
+                    (self.attributes["month_kWh"] / self.attributes["month_days"])
+                    if self.attributes["month_days"] > 0
+                    else 0
+                )
+                if month is not None
+                else None
+            )
+
+            for tariff in (1, 2, 3):
+                self.attributes[f"month_p{tariff}_kWh"] = (
+                    month.get(f"value_p{tariff}_kWh", None)
+                    if month is not None
+                    else None
+                )
+                self.attributes[f"month_surplus_p{tariff}_kWh"] = (
+                    month.get(f"surplus_p{tariff}_kWh", None)
+                    if month is not None
+                    else None
+                )
+
+            last_month = utils.get_by_key(
+                self.data["consumptions_monthly_sum"],
+                "datetime",
+                (month_starts - relativedelta(months=1)),
+            )
+            self.attributes["last_month_kWh"] = (
+                last_month.get("value_kWh", None) if last_month is not None else None
+            )
+            self.attributes["last_month_surplus_kWh"] = (
+                last_month.get("surplus_kWh", None) if last_month is not None else None
+            )
+            self.attributes["last_month_days"] = (
+                last_month.get("delta_h", 0) / 24 if last_month is not None else None
+            )
+            self.attributes["last_month_daily_kWh"] = (
+                (
+                    (
+                        self.attributes["last_month_kWh"]
+                        / self.attributes["last_month_days"]
+                    )
+                    if self.attributes["last_month_days"] > 0
+                    else 0
+                )
+                if last_month is not None
+                else None
+            )
+            for tariff in (1, 2, 3):
+                self.attributes[f"last_month_p{tariff}_kWh"] = (
+                    last_month.get(f"value_p{tariff}_kWh", None)
+                    if last_month is not None
+                    else None
+                )
+                self.attributes[f"last_month_surplus_p{tariff}_kWh"] = (
+                    last_month.get(f"surplus_p{tariff}_kWh", None)
+                    if last_month is not None
+                    else None
+                )
+
+            if len(self.data["consumptions"]) > 0:
+                self.attributes["last_registered_date"] = self.data["consumptions"][-1][
+                    "datetime"
+                ]
+
+                if len(self.data["consumptions_daily_sum"]) > 0:
+                    last_day = self.data["consumptions_daily_sum"][-1]
+                    self.attributes["last_registered_day_kWh"] = (
+                        last_day.get("value_kWh", None)
+                        if last_day is not None
+                        else None
+                    )
+                    self.attributes["last_registered_day_surplus_kWh"] = (
+                        last_day.get("surplus_kWh", None)
+                        if last_day is not None
+                        else None
+                    )
+
+                    for tariff in (1, 2, 3):
+                        self.attributes[f"last_registered_day_p{tariff}_kWh"] = (
+                            last_day.get(f"value_p{tariff}_kWh", None)
+                            if last_day is not None
+                            else None
+                        )
+
+                        self.attributes[
+                            f"last_registered_day_surplus_p{tariff}_kWh"
+                        ] = (
+                            last_day.get(f"surplus_p{tariff}_kWh", None)
+                            if last_day is not None
+                            else None
+                        )
+
+                    self.attributes["last_registered_day_hours"] = (
+                        last_day.get("delta_h", None) if last_day is not None else None
+                    )
+
+    def process_maximeter(self):
+        """Process maximeter data."""
+        if len(self.data["maximeter"]) > 0:
+            processor = MaximeterProcessor(self.data["maximeter"])
+            last_relative_year = processor.output["stats"]
+            self.attributes["max_power_kW"] = last_relative_year.get(
+                "value_max_kW", None
+            )
+            self.attributes["max_power_date"] = last_relative_year.get("date_max", None)
+            self.attributes["max_power_mean_kW"] = last_relative_year.get(
+                "value_mean_kW", None
+            )
+            self.attributes["max_power_90perc_kW"] = last_relative_year.get(
+                "value_tile90_kW", None
+            )
+
+    def process_cost(self):
+        """Process costs."""
+        if self.enable_billing:
+            new_data_from = self._date_from
+            if self._incremental_update:
+                with contextlib.suppress(Exception):
+                    new_data_from = self.data["cost_monthly_sum"][-1]["datetime"]
+
+            proc = BillingProcessor(
+                BillingInput(
+                    contracts=self.data["contracts"],
+                    consumptions=[
+                        x
+                        for x in self.data["consumptions"]
+                        if x["datetime"] >= new_data_from
+                    ],
+                    prices=(
+                        [x for x in self.data["pvpc"] if x["datetime"] >= new_data_from]
+                        if self.is_pvpc
+                        else None
+                    ),
+                    rules=self.pricing_rules,
+                )
+            )
+            month_starts = datetime(
+                datetime.today().year, datetime.today().month, 1, 0, 0, 0
+            )
+
+            # append new data
+            hourly = proc.output["hourly"]
+            self.data["cost_hourly_sum"] = utils.extend_and_filter(
+                self.data["cost_hourly_sum"],
+                hourly,
+                "datetime",
+                self._date_from,
+                self._date_to,
+            )
+
+            daily = proc.output["daily"]
+            self.data["cost_daily_sum"] = utils.extend_and_filter(
+                self.data["cost_daily_sum"],
+                daily,
+                "datetime",
+                self._date_from,
+                self._date_to,
+            )
+
+            monthly = proc.output["monthly"]
+            self.data["cost_monthly_sum"] = utils.extend_and_filter(
+                self.data["cost_monthly_sum"],
+                monthly,
+                "datetime",
+                self._date_from,
+                self._date_to,
+            )
+
+            this_month = utils.get_by_key(
+                self.data["cost_monthly_sum"],
+                "datetime",
+                month_starts,
+            )
+
+            last_month = utils.get_by_key(
+                self.data["cost_monthly_sum"],
+                "datetime",
+                (month_starts - relativedelta(months=1)),
+            )
+
+            if this_month is not None:
+                self.attributes["month_€"] = this_month.get("value_eur", None)
+
+            if last_month is not None:
+                self.attributes["last_month_€"] = last_month.get("value_eur", None)
+
+    def reset(self):
+        """Reset in-mem objects."""
+
+        self.data = EdataData(
+            supplies=[],
+            contracts=[],
+            consumptions=[],
+            maximeter=[],
+            pvpc=[],
+            consumptions_daily_sum=[],
+            consumptions_monthly_sum=[],
+            cost_hourly_sum=[],
+            cost_daily_sum=[],
+            cost_monthly_sum=[],
+        )
+
+        for attr in ATTRIBUTES:
+            self.attributes[attr] = None
+
+        self.last_update = {x: datetime(1970, 1, 1) for x in self.data}

e_data-1.3.2/edata/storage.py

@@ -0,0 +1,46 @@
+import json
+import logging
+import os
+from .processors import utils
+from . import const as const
+from . import definitions as defs
+
+_LOGGER = logging.getLogger(__name__)
+
+DEFAULT_STORAGE_DIR = os.getenv("HOME")
+RECENT_CACHE_FILENAME = "edata_{id}.json"
+
+compile_storage_id = lambda cups: cups.lower()
+
+
+def check_storage_integrity(data: defs.EdataData):
+    """Check if an EdataData object follows a schema."""
+    return defs.EdataSchema(data)
+
+
+def load_storage(cups: str, storage_dir: str | None = None):
+    """Load EdataData storage from its config dir."""
+    if storage_dir is None:
+        storage_dir = DEFAULT_STORAGE_DIR
+    _subdir = os.path.join(storage_dir, const.PROG_NAME)
+    _recent_cache = os.path.join(
+        _subdir, RECENT_CACHE_FILENAME.format(id=compile_storage_id(cups))
+    )
+    os.makedirs(_subdir, exist_ok=True)
+
+    with open(_recent_cache, encoding="utf-8") as f:
+        return check_storage_integrity(utils.deserialize_dict(json.load(f)))
+
+
+def dump_storage(cups: str, storage: defs.EdataData, storage_dir: str | None = None):
+    """Update EdataData storage."""
+    if storage_dir is None:
+        storage_dir = DEFAULT_STORAGE_DIR
+    _subdir = os.path.join(storage_dir, const.PROG_NAME)
+    _recent_cache = os.path.join(
+        _subdir, RECENT_CACHE_FILENAME.format(id=compile_storage_id(cups))
+    )
+    os.makedirs(_subdir, exist_ok=True)
+
+    with open(_recent_cache, "w", encoding="utf-8") as f:
+        json.dump(utils.serialize_dict(check_storage_integrity(storage)), f)

{e_data-1.3.1 → e_data-1.3.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "e-data"
-version = "1.3.1"
+version = "1.3.2"
 description = "Python library for managing spanish energy data from various web providers"
 readme = "README.md"
 authors = [
@@ -37,7 +37,7 @@ dependencies = [
 Homepage = "https://github.com/uvejota/python-edata"
 
 [tool.setuptools.packages.find]
-where = ["edata"]
+where = ["."]
 
 [tool.setuptools]
 include-package-data = true

e_data-1.3.1/edata/e_data.egg-info/top_level.txt

@@ -1,3 +0,0 @@
-connectors
-processors
-tests