e-data 1.2.17__tar.gz → 1.2.19__tar.gz

{e-data-1.2.17/e_data.egg-info → e_data-1.2.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: e-data
-Version: 1.2.17
+Version: 1.2.19
 Summary: Python library for managing spanish energy data from various web providers
 Home-page: https://github.com/uvejota/python-edata
 Author: VMG
@@ -18,6 +18,14 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.6.0
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: dateparser>=1.1.2
+Requires-Dist: freezegun>=1.2.1
+Requires-Dist: holidays>=0.14.2
+Requires-Dist: pytest>=7.1.2
+Requires-Dist: python_dateutil>=2.8.2
+Requires-Dist: requests>=2.28.1
+Requires-Dist: voluptuous>=0.13.1
+Requires-Dist: Jinja2>=3.1.2
 
 
 [![Downloads](https://pepy.tech/badge/e-data)](https://pepy.tech/project/e-data)

{e-data-1.2.17 → e_data-1.2.19/e_data.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: e-data
-Version: 1.2.17
+Version: 1.2.19
 Summary: Python library for managing spanish energy data from various web providers
 Home-page: https://github.com/uvejota/python-edata
 Author: VMG
@@ -18,6 +18,14 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.6.0
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: dateparser>=1.1.2
+Requires-Dist: freezegun>=1.2.1
+Requires-Dist: holidays>=0.14.2
+Requires-Dist: pytest>=7.1.2
+Requires-Dist: python_dateutil>=2.8.2
+Requires-Dist: requests>=2.28.1
+Requires-Dist: voluptuous>=0.13.1
+Requires-Dist: Jinja2>=3.1.2
 
 
 [![Downloads](https://pepy.tech/badge/e-data)](https://pepy.tech/project/e-data)

{e-data-1.2.17 → e_data-1.2.19}/edata/connectors/datadis.py

@@ -4,17 +4,17 @@ To fetch data from datadis.es private API.
 There a few issues that are workarounded:
  - You have to wait 24h between two identical requests.
  - Datadis server does not like ranges greater than 1 month.
- - TODO: gzip header checksum errors under weird circumstances
 """
 
 import contextlib
+from datetime import datetime, timedelta
 import hashlib
 import json
 import logging
-from datetime import datetime, timedelta
 import os
-from dateutil.relativedelta import relativedelta
 
+from dateutil.relativedelta import relativedelta
+from filelock import FileLock
 import requests
 
 from ..definitions import ConsumptionData, ContractData, MaxPowerData, SupplyData
@@ -104,50 +104,60 @@ class DatadisConnector:
             self._recent_queries_file = DEFAULT_RECENT_QUERIES_FILE
             self._recent_queries_cache_file = DEFAULT_RECENT_QUERIES_CACHE
         self._warned_queries = []
+        self._load_recent_cache()
+
+    def _load_recent_cache(self) -> None:
+        """Load caches."""
 
-        # load caches (to avoid query spam if the program restarts)
         with contextlib.suppress(FileNotFoundError):
             with open(self._recent_queries_file, encoding="utf8") as dst_file:
-                self._recent_queries = json.load(dst_file)
-                for query in self._recent_queries:
-                    self._recent_queries[query] = datetime.fromisoformat(
-                        self._recent_queries[query]
+                recent_queries = json.load(dst_file)
+                for query in recent_queries:
+                    recent_queries[query] = datetime.fromisoformat(
+                        recent_queries[query]
                     )
+                self._recent_queries.update(recent_queries)
+
             with open(self._recent_queries_cache_file, encoding="utf8") as dst_file:
-                self._recent_cache = json.load(dst_file)
+                self._recent_cache.update(json.load(dst_file))
 
     def _update_recent_queries(self, query: str, data: dict | None = None) -> None:
         """Cache a successful query to avoid exceeding query limits."""
 
-        # identify the query by a md5 hash
-        hash_query = hashlib.md5(query.encode()).hexdigest()
-        # prepare key (hash) and values (timestamp and response)
-        self._recent_queries[hash_query] = datetime.now()
-        if data is not None:
-            self._recent_cache[hash_query] = data
-
-        # purge old cache
-        to_delete = []
-        for _query in self._recent_queries:
-            if (datetime.now() - self._recent_queries[_query]) > QUERY_LIMIT:
-                to_delete.append(_query)
-
-        for key in to_delete:
-            with contextlib.suppress(KeyError):
-                self._recent_queries.pop(key, None)
-                self._recent_cache.pop(key, None)
-
-        # dump current cache to disk
-        try:
-            with open(self._recent_queries_file, "w", encoding="utf8") as dst_file:
-                json.dump(utils.serialize_dict(self._recent_queries), dst_file)
+        with FileLock(self._recent_queries_file + ".lock"):
+            self._load_recent_cache()
+            # identify the query by a md5 hash
+            hash_query = hashlib.md5(query.encode()).hexdigest()
+            # prepare key (hash) and values (timestamp and response)
+            self._recent_queries[hash_query] = datetime.now()
             if data is not None:
-                with open(
-                    self._recent_queries_cache_file, "w", encoding="utf8"
-                ) as dst_file:
-                    json.dump(self._recent_cache, dst_file)
-        except Exception as e:
-            _LOGGER.warning("Unknown error while updating cache: %s", e)
+                self._recent_cache[hash_query] = data
+
+            # purge old cache
+            to_delete = [
+                x
+                for x in self._recent_queries
+                if (datetime.now() - self._recent_queries[x]) > QUERY_LIMIT
+            ]
+
+            for key in to_delete:
+                with contextlib.suppress(KeyError):
+                    self._recent_queries.pop(key, None)
+                    self._recent_cache.pop(key, None)
+
+            # dump current cache to disk
+            try:
+                with open(self._recent_queries_file, "w", encoding="utf8") as dst_file:
+                    json.dump(utils.serialize_dict(self._recent_queries), dst_file)
+                if data is not None:
+                    with open(
+                        self._recent_queries_cache_file, "w", encoding="utf8"
+                    ) as dst_file:
+                        json.dump(self._recent_cache, dst_file)
+                _LOGGER.debug("Local cache updated")
+
+            except Exception as e:
+                _LOGGER.warning("Unknown error while updating cache: %s", e)
 
     def _is_recent_query(self, query: str) -> bool:
         """Check if a query has been done recently to avoid exceeding query limits."""
@@ -160,7 +170,7 @@ class DatadisConnector:
     def _get_cache_for_query(self, query: str) -> dict:
         """Return cached response for a query."""
         hash_query = hashlib.md5(query.encode()).hexdigest()
-
+        _LOGGER.info("Serving '%s' from cache", query)
         return self._recent_cache.get(hash_query, None)
 
     def _get_token(self):
@@ -243,13 +253,15 @@ class DatadisConnector:
                 _LOGGER.info("Got 200 OK at %s", url + params)
                 if reply.json():
                     response = reply.json()
-                    self._update_recent_queries(url + params, response)
+                    if not ignore_recent_queries:
+                        self._update_recent_queries(url + params, response)
                 else:
                     # this mostly happens when datadis provides an empty response
                     _LOGGER.info(
                         "Datadis returned an empty response at %s", url + params
                     )
-                    self._update_recent_queries(url + params)
+                    if not ignore_recent_queries:
+                        self._update_recent_queries(url + params)
             elif reply.status_code == 401 and not refresh_token:
                 # we're here if we were unauthorized so we will refresh the token
                 response = self._get(
@@ -266,7 +278,8 @@ class DatadisConnector:
                     reply.text,
                     url + params,
                 )
-                self._update_recent_queries(url + params)
+                if not ignore_recent_queries:
+                    self._update_recent_queries(url + params)
             elif is_retry:
                 # otherwise, if this was a retried request... warn the user
                 if (url + params) not in self._warned_queries:
@@ -278,7 +291,8 @@ class DatadisConnector:
                         "Query temporary disabled",
                         "Future 500 code errors for this query will be silenced until restart",
                     )
-                self._update_recent_queries(url + params)
+                if not ignore_recent_queries:
+                    self._update_recent_queries(url + params)
                 self._warned_queries.append(url + params)
             else:
                 # finally, retry since an unexpected error took place (mostly 500 errors - server fault)
@@ -302,7 +316,7 @@ class DatadisConnector:
 
         # Request the resource
         response = self._get(
-            URL_GET_SUPPLIES, request_data=data, ignore_recent_queries=False
+            URL_GET_SUPPLIES, request_data=data, ignore_recent_queries=True
         )
 
         # Response is a list of serialized supplies.
@@ -318,23 +332,27 @@ class DatadisConnector:
                     SupplyData(
                         cups=i["cups"],  # the supply identifier
                         date_start=datetime.strptime(
-                            i["validDateFrom"]
-                            if i["validDateFrom"] != ""
-                            else "1970/01/01",
+                            (
+                                i["validDateFrom"]
+                                if i["validDateFrom"] != ""
+                                else "1970/01/01"
+                            ),
                             "%Y/%m/%d",
                         ),  # start date of the supply. 1970/01/01 if unset.
                         date_end=datetime.strptime(
-                            i["validDateTo"]
-                            if i["validDateTo"] != ""
-                            else tomorrow_str,
+                            (
+                                i["validDateTo"]
+                                if i["validDateTo"] != ""
+                                else tomorrow_str
+                            ),
                             "%Y/%m/%d",
                         ),  # end date of the supply, tomorrow if unset
                         # the following parameters are not crucial, so they can be none
-                        address=i["address"] if "address" in i else None,
-                        postal_code=i["postalCode"] if "postalCode" in i else None,
-                        province=i["province"] if "province" in i else None,
-                        municipality=i["municipality"] if "municipality" in i else None,
-                        distributor=i["distributor"] if "distributor" in i else None,
+                        address=i.get("address", None),
+                        postal_code=i.get("postalCode", None),
+                        province=i.get("province", None),
+                        municipality=i.get("municipality", None),
+                        distributor=i.get("distributor", None),
                         # these two are mandatory, we will use them to fetch contracts data
                         pointType=i["pointType"],
                         distributorCode=i["distributorCode"],
@@ -355,7 +373,7 @@ class DatadisConnector:
         if authorized_nif is not None:
             data["authorizedNif"] = authorized_nif
         response = self._get(
-            URL_GET_CONTRACT_DETAIL, request_data=data, ignore_recent_queries=False
+            URL_GET_CONTRACT_DETAIL, request_data=data, ignore_recent_queries=True
         )
         contracts = []
         tomorrow_str = (datetime.today() + timedelta(days=1)).strftime("%Y/%m/%d")
@@ -373,12 +391,16 @@ class DatadisConnector:
                         ),
                         marketer=i["marketer"],
                         distributorCode=distributor_code,
-                        power_p1=i["contractedPowerkW"][0]
-                        if isinstance(i["contractedPowerkW"], list)
-                        else None,
-                        power_p2=i["contractedPowerkW"][1]
-                        if (len(i["contractedPowerkW"]) > 1)
-                        else None,
+                        power_p1=(
+                            i["contractedPowerkW"][0]
+                            if isinstance(i["contractedPowerkW"], list)
+                            else None
+                        ),
+                        power_p2=(
+                            i["contractedPowerkW"][1]
+                            if (len(i["contractedPowerkW"]) > 1)
+                            else None
+                        ),
                     )
                 )
             else:

{e-data-1.2.17 → e_data-1.2.19}/edata/definitions.py

@@ -71,9 +71,7 @@ DEFAULT_BILLING_SURPLUS_FORMULA = (
 
 # Sum energy and power terms, and substract surplus until 0.
 # An alternative would be "[(energy_term + power_term - surplus_term), 0]|max + others_term"
-DEFAULT_BILLING_MAIN_FORMULA = (
-    "[(energy_term - surplus_term), 0]|max + power_term + others_term"
-)
+DEFAULT_BILLING_MAIN_FORMULA = "energy_term + power_term + others_term"
 
 
 class SupplyData(TypedDict):

{e-data-1.2.17 → e_data-1.2.19}/edata/helpers.py

@@ -22,6 +22,12 @@ from .storage import check_storage_integrity, dump_storage, load_storage
 _LOGGER = logging.getLogger(__name__)
 
 
+def acups(cups):
+    """Print an abbreviated and anonymized CUPS."""
+
+    return cups[-5:]
+
+
 class EdataHelper:
     """Main EdataHelper class."""
 
@@ -53,7 +59,7 @@ class EdataHelper:
         self.attributes = {}
         self._storage_dir = storage_dir_path
         self._cups = cups
-        self._scups = cups[-5:]
+        self._scups = acups(cups)
         self._authorized_nif = datadis_authorized_nif
         self.last_update = {x: datetime(1970, 1, 1) for x in self.data}
         self._date_from = datetime(1970, 1, 1)
@@ -74,9 +80,11 @@ class EdataHelper:
         self.datadis_api = DatadisConnector(
             datadis_username,
             datadis_password,
-            storage_path=os.path.join(storage_dir_path, const.PROG_NAME)
-            if storage_dir_path is not None
-            else None,
+            storage_path=(
+                os.path.join(storage_dir_path, const.PROG_NAME)
+                if storage_dir_path is not None
+                else None
+            ),
         )
         self.redata_api = REDataConnector()
 
@@ -111,7 +119,7 @@ class EdataHelper:
         date_to: datetime = datetime.today(),
         incremental_update: bool = True,
     ):
-        """Synchronous update."""
+        """Update synchronously."""
 
         _LOGGER.info(
             "%s: update triggered",
@@ -136,9 +144,9 @@ class EdataHelper:
             dump_storage(self._cups, self.data, self._storage_dir)
 
     def update_supplies(self):
-        """Synchronous data update of supplies."""
+        """Update supplies."""
 
-        _LOGGER.info("%s: supplies update triggered", self._scups)
+        _LOGGER.debug("%s: supplies update triggered", self._scups)
         if datetime.today().date() != self.last_update["supplies"].date():
             # if supplies haven't been updated today
             supplies = self.datadis_api.get_supplies(
@@ -153,9 +161,9 @@ class EdataHelper:
             _LOGGER.info("%s: supplies are already updated (skipping)", self._scups)
 
     def update_contracts(self, cups: str, distributor_code: str):
-        """Synchronous data update of contracts."""
+        """Update contracts."""
 
-        _LOGGER.info("%s: contracts update triggered", self._scups)
+        _LOGGER.debug("%s: contracts update triggered", self._scups)
         if datetime.today().date() != self.last_update["contracts"].date():
             # if contracts haven't been updated today
             contracts = self.datadis_api.get_contract_detail(
@@ -180,9 +188,9 @@ class EdataHelper:
         measurement_type: str,
         point_type: int,
     ):
-        """Synchronous data update of consumptions."""
+        """Update consumptions."""
 
-        _LOGGER.info("%s: consumptions update triggered", self._scups)
+        _LOGGER.debug("%s: consumptions update triggered", self._scups)
         if (datetime.now() - self.last_update["consumptions"]) > self.UPDATE_INTERVAL:
             consumptions = self.datadis_api.get_consumption_data(
                 cups,
@@ -194,22 +202,23 @@ class EdataHelper:
                 authorized_nif=self._authorized_nif,
             )
             if len(consumptions) > 0:
+                _LOGGER.info(
+                    "%s: got consumptions from %s to %s",
+                    self._scups,
+                    consumptions[0]["datetime"].isoformat(),
+                    consumptions[-1]["datetime"].isoformat(),
+                )
                 self.data["consumptions"] = utils.extend_by_key(
                     self.data["consumptions"], consumptions, "datetime"
                 )
                 self.last_update["consumptions"] = datetime.now()
-                _LOGGER.info(
-                    "%s: consumptions update succeeded (%s new records)",
-                    self._scups,
-                    len(consumptions),
-                )
         else:
             _LOGGER.info("%s: consumptions are already updated (skipping)", self._scups)
 
     def update_maximeter(self, cups, distributor_code, start_date, end_date):
-        """Synchronous data update of maximeter."""
+        """Update maximeter."""
 
-        _LOGGER.info("%s: maximeter update triggered", self._scups)
+        _LOGGER.debug("%s: maximeter update triggered", self._scups)
         if (datetime.now() - self.last_update["maximeter"]) > self.UPDATE_INTERVAL:
             maximeter = self.datadis_api.get_max_power(
                 cups,
@@ -219,15 +228,14 @@ class EdataHelper:
                 authorized_nif=self._authorized_nif,
             )
             if len(maximeter) > 0:
+                _LOGGER.info(
+                    "%s: maximeter update succeeded",
+                    self._scups,
+                )
                 self.data["maximeter"] = utils.extend_by_key(
                     self.data["maximeter"], maximeter, "datetime"
                 )
                 self.last_update["maximeter"] = datetime.now()
-                _LOGGER.info(
-                    "%s: maximeter update succeeded (%s new records)",
-                    self._scups,
-                    len(maximeter),
-                )
         else:
             _LOGGER.info("%s: maximeter is already updated (skipping)", self._scups)
 
@@ -237,7 +245,7 @@ class EdataHelper:
         date_from: datetime = datetime(1970, 1, 1),
         date_to: datetime = datetime.today(),
     ):
-        """Synchronous data update."""
+        """Update all data from Datadis."""
 
         _LOGGER.info(
             "%s: datadis update triggered (from %s to %s)",
@@ -262,15 +270,22 @@ class EdataHelper:
         if supply is None:
             # return if specified cups seems not valid
             _LOGGER.error(
-                "%s: CUPS not found, please check configured CUPS", self._scups
+                "%s: CUPS not found. Got: %s",
+                self._scups,
+                [acups(x["cups"]) for x in self.data["supplies"]],
             )
             return False
+        _LOGGER.info("%s: CUPS found in account", self._scups)
 
         # get some supply-related data
         supply_date_start = supply["date_start"]
         distributor_code = supply["distributorCode"]
         point_type = supply["pointType"]
 
+        _LOGGER.info(
+            "%s: CUPS start date is %s", self._scups, supply_date_start.isoformat()
+        )
+
         # update contracts to get valid periods
         self.update_contracts(cups, distributor_code)
         if len(self.data["contracts"]) == 0:
@@ -280,7 +295,7 @@ class EdataHelper:
             )
             return False
 
-        # filter consumptions and maximeter, and look for gaps
+        # filter consumptions and maximeter, and log gaps
         def sort_and_filter(dt_from, dt_to):
             self.data["consumptions"], miss_cons = utils.extract_dt_ranges(
                 self.data["consumptions"],
@@ -316,64 +331,47 @@ class EdataHelper:
             ),
         )
 
-        oldest_contract = datetime.today()
-        for contract in self.data["contracts"]:
-            # register oldest contract
-            if contract["date_start"] < oldest_contract:
-                oldest_contract = contract["date_start"]
-
-            # update consumptions
-            for gap in [
-                x
-                for x in miss_cons
-                if not (
-                    x["to"] < contract["date_start"] or x["from"] > contract["date_end"]
-                )
-            ]:
+        # update consumptions
+        for gap in miss_cons:
+            if not (
+                gap["to"] < supply["date_start"] or gap["from"] > supply["date_end"]
+            ):
                 # fetch consumptions for each consumptions gap in valid periods
+                start = max([gap["from"] + timedelta(hours=1), supply["date_start"]])
+                end = min([gap["to"], supply["date_end"]])
+                _LOGGER.info(
+                    "%s: request consumptions from %s to %s",
+                    self._scups,
+                    start.isoformat(),
+                    end.isoformat(),
+                )
                 self.update_consumptions(
                     cups,
                     distributor_code,
-                    max([gap["from"] + timedelta(hours=1), contract["date_start"]]),
-                    min([gap["to"], contract["date_end"]]),
+                    start,
+                    end,
                     "0",
                     point_type,
                 )
 
-            # update maximeter
-            for gap in [
-                x
-                for x in miss_maxim
-                if not (
-                    date_to < contract["date_start"] or date_from > contract["date_end"]
-                )
-            ]:
+        # update maximeter
+        for gap in miss_maxim:
+            if not (date_to < supply["date_start"] or date_from > supply["date_end"]):
                 # fetch maximeter for each maximeter gap in valid periods
                 start = max(
-                    [gap["from"], contract["date_start"] + relativedelta(months=1)]
+                    [gap["from"], supply["date_start"] + relativedelta(months=1)]
                 )
-                end = min([gap["to"], contract["date_end"]])
+                end = min([gap["to"], supply["date_end"]])
                 start = min([start, end])
+                _LOGGER.info(
+                    "%s: request maximeter from %s to %s",
+                    self._scups,
+                    start.isoformat(),
+                    end.isoformat(),
+                )
                 self.update_maximeter(cups, distributor_code, start, end)
 
-        # safe check periods in non-registered contracts
-        if oldest_contract != supply_date_start and oldest_contract > max(
-            [date_from, supply_date_start]
-        ):
-            _LOGGER.info(
-                "%s: supplies and contract start date do not match, exploring non-registered contracts",
-                self._scups,
-            )
-            start = max([supply_date_start, date_from])
-            self.update_consumptions(
-                cups, distributor_code, start, oldest_contract, "0", point_type
-            )
-            self.update_maximeter(cups, distributor_code, start, oldest_contract)
-            miss_cons, miss_maxim = sort_and_filter(start, date_to)
-        else:
-            miss_cons, miss_maxim = sort_and_filter(
-                max([date_from, oldest_contract]), date_to
-            )
+        miss_cons, miss_maxim = sort_and_filter(date_from, date_to)
 
         return True
 
@@ -679,11 +677,11 @@ class EdataHelper:
                         for x in self.data["consumptions"]
                         if x["datetime"] >= new_data_from
                     ],
-                    prices=[
-                        x for x in self.data["pvpc"] if x["datetime"] >= new_data_from
-                    ]
-                    if self.is_pvpc
-                    else None,
+                    prices=(
+                        [x for x in self.data["pvpc"] if x["datetime"] >= new_data_from]
+                        if self.is_pvpc
+                        else None
+                    ),
                     rules=self.pricing_rules,
                 )
             )

{e-data-1.2.17 → e_data-1.2.19}/setup.py

@@ -20,7 +20,7 @@ URL = "https://github.com/uvejota/python-edata"
 EMAIL = "vmayorg@outlook.es"
 AUTHOR = "VMG"
 REQUIRES_PYTHON = ">=3.6.0"
-VERSION = "1.2.17"
+VERSION = "1.2.19"
 
 # What packages are required for this module to be executed?
 REQUIRED = [