e-data 2.0.1.dev127__tar.gz → 2.0.1.dev129__tar.gz

{e_data-2.0.1.dev127/e_data.egg-info → e_data-2.0.1.dev129}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: e-data
-Version: 2.0.1.dev127
+Version: 2.0.1.dev129
 Summary: Python library for managing spanish energy data from various web providers
 Author-email: VMG <vmayorg@outlook.es>
 License:                     GNU GENERAL PUBLIC LICENSE

{e_data-2.0.1.dev127 → e_data-2.0.1.dev129/e_data.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: e-data
-Version: 2.0.1.dev127
+Version: 2.0.1.dev129
 Summary: Python library for managing spanish energy data from various web providers
 Author-email: VMG <vmayorg@outlook.es>
 License:                     GNU GENERAL PUBLIC LICENSE

{e_data-2.0.1.dev127 → e_data-2.0.1.dev129}/e_data.egg-info/SOURCES.txt

@@ -29,4 +29,5 @@ edata/services/data_service.py
 edata/tests/__init__.py
 edata/tests/test_datadis_connector.py
 edata/tests/test_redata_connector.py
-edata/tests/test_services.py
+edata/tests/test_services.py
+edata/tests/test_utils.py

{e_data-2.0.1.dev127 → e_data-2.0.1.dev129}/edata/core/utils.py

@@ -1,9 +1,12 @@
 """Collection of utilities."""
 
 import logging
-from datetime import datetime
+import typing
+from datetime import datetime, timedelta
+from functools import lru_cache
 from pathlib import Path
 
+from dateutil.relativedelta import relativedelta
 import holidays
 
 from edata.models.supply import Contract
@@ -19,10 +22,22 @@ TARIFF_BY_WEEKDAY = [[], [], [5, 6]]
 _LOGGER = logging.getLogger(__name__)
 
 
+@lru_cache(maxsize=None)
+def _es_holidays(year: int) -> holidays.HolidayBase:
+    """Return (and cache) Spanish holidays for a single year.
+
+    Building the holidays calendar is expensive; get_tariff is called once per
+    hourly record, so without this cache a full-history rebuild spends minutes
+    reconstructing the same handful of yearly calendars.
+    """
+
+    return holidays.country_holidays("ES", years=year)
+
+
 def get_tariff(dt: datetime) -> int:
     """Return the tariff for the selected datetime."""
 
-    hdays = holidays.country_holidays("ES")
+    hdays = _es_holidays(dt.year)
     hour = dt.hour
     weekday = dt.weekday()
 
@@ -64,6 +79,27 @@ def get_day(dt: datetime) -> datetime:
     return dt.replace(hour=0, minute=0, second=0, microsecond=0)
 
 
+def iter_month_windows(
+    start: datetime, end: datetime
+) -> typing.Iterator[tuple[datetime, datetime]]:
+    """Yield contiguous ``(window_start, window_end)`` pairs, one calendar month each.
+
+    Windows are non-overlapping and cover ``[start, end]`` inclusively; the first
+    starts at ``start`` and the last ends at ``end``. Iterating a month at a time
+    keeps the working set bounded to a single month regardless of history length.
+    """
+
+    if end < start:
+        return
+
+    win_start = start
+    while win_start <= end:
+        next_month = get_month(win_start) + relativedelta(months=1)
+        win_end = min(next_month - timedelta(microseconds=1), end)
+        yield win_start, win_end
+        win_start = next_month
+
+
 def redacted_cups(cups: str) -> str:
     """Return an anonymized version of the cups identifier."""
 

{e_data-2.0.1.dev127 → e_data-2.0.1.dev129}/edata/database/controller.py

@@ -3,7 +3,7 @@ import os
 import typing
 from datetime import datetime
 
-from sqlalchemy import Select, insert
+from sqlalchemy import Select, event, insert
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
 from sqlmodel import SQLModel
@@ -28,6 +28,25 @@ _LOGGER = logging.getLogger(__name__)
 T = typing.TypeVar("T", bound=SQLModel)
 
 
+def _set_sqlite_pragmas(dbapi_connection, connection_record) -> None:
+    """Tune SQLite for this write-heavy, single-writer workload.
+
+    WAL lets readers proceed while a write is in flight and avoids an fsync per
+    statement; synchronous=NORMAL is durable under WAL (only a crash mid-checkpoint
+    could lose the last transaction, and the data is re-fetchable from Datadis);
+    busy_timeout avoids spurious "database is locked" errors under concurrency.
+    journal_mode is persisted in the file; the others are per-connection.
+    """
+
+    cursor = dbapi_connection.cursor()
+    try:
+        cursor.execute("PRAGMA journal_mode=WAL")
+        cursor.execute("PRAGMA synchronous=NORMAL")
+        cursor.execute("PRAGMA busy_timeout=5000")
+    finally:
+        cursor.close()
+
+
 class EdataDB:
 
     _instance = None
@@ -39,10 +58,11 @@ class EdataDB:
         if cls._instance is None:
             cls._instance = super().__new__(cls)
             cls._db_url = db_url
-            cls._engine = create_async_engine(db_url, future=True)
-            # Ensure parent directory exists
+            # Ensure parent directory exists before the first connection is opened.
             dir_path = os.path.dirname(os.path.abspath(sqlite_path))
             os.makedirs(dir_path, exist_ok=True)
+            cls._engine = create_async_engine(db_url, future=True)
+            event.listen(cls._engine.sync_engine, "connect", _set_sqlite_pragmas)
             cls._instance._tables_initialized = False
         elif db_url != cls._db_url:
             raise ValueError("EdataDB already initialized with a different db_url")

{e_data-2.0.1.dev127 → e_data-2.0.1.dev129}/edata/services/bill_service.py

@@ -16,6 +16,7 @@ from edata.core.utils import (
     get_day,
     get_month,
     get_tariff,
+    iter_month_windows,
     redacted_cups,
 )
 from edata.database.controller import EdataDB
@@ -107,31 +108,39 @@ class BillService:
         # fetch contracts
         contracts = await self._get_contracts()
 
-        # fetch and filter energy items
-        energy = await self._get_energy()
-
-        _LOGGER.debug("%s compiling missing hourly bills", self._scups)
         if is_pvpc:
-            pvpc = await self._get_pvpc()
             billing_rules = PVPCBillingRules(**billing_rules.model_dump())
-            bills = await asyncio.to_thread(
-                self.simulate_pvpc, contracts, energy, pvpc, billing_rules
-            )
             confighash = f"pvpc-{hash(billing_rules.model_dump_json())}"
         else:
-            bills = await asyncio.to_thread(
-                self.simulate_custom, contracts, energy, billing_rules
-            )
             confighash = f"custom-{hash(billing_rules.model_dump_json())}"
 
-        _LOGGER.debug("%s pushing hourly bills", self._scups)
-        await self.db.add_bill_list(
-            cups=self._cups,
-            type_="hour",
-            confhash=confighash,
-            complete=True,
-            bill=bills,
-        )
+        # compile and persist hourly bills one month at a time so the energy
+        # (and simulated bills) held in memory stay bounded to a single month
+        for win_start, win_end in iter_month_windows(start, end):
+            energy = await self._get_energy(win_start, win_end)
+            if not energy:
+                continue
+
+            _LOGGER.debug(
+                "%s compiling hourly bills for %s..%s", self._scups, win_start, win_end
+            )
+            if is_pvpc:
+                pvpc = await self._get_pvpc(win_start, win_end)
+                bills = await asyncio.to_thread(
+                    self.simulate_pvpc, contracts, energy, pvpc, billing_rules
+                )
+            else:
+                bills = await asyncio.to_thread(
+                    self.simulate_custom, contracts, energy, billing_rules
+                )
+
+            await self.db.add_bill_list(
+                cups=self._cups,
+                type_="hour",
+                confhash=confighash,
+                complete=True,
+                bill=bills,
+            )
 
         _LOGGER.debug("%s updating daily and monthly bills", self._scups)
         await self.update_statistics(start, end)
@@ -171,10 +180,11 @@ class BillService:
         )
 
     async def update_statistics(self, start: datetime, end: datetime):
-        """Update the statistics during a period."""
+        """Update the statistics during a period, one month at a time."""
 
-        await self._update_daily_statistics(start, end)
-        await self._update_monthly_statistics(start, end)
+        for win_start, win_end in iter_month_windows(start, end):
+            await self._update_daily_statistics(win_start, win_end)
+            await self._update_monthly_statistics(win_start, win_end)
 
     async def _update_daily_statistics(self, start: datetime, end: datetime) -> None:
         """Update daily statistics within a date range."""
@@ -286,7 +296,7 @@ class BillService:
         missing = await self._find_missing_stats()
         for day in missing:
             _LOGGER.debug("%s updating daily statistics for date %s", self._scups, day)
-            end = day + relativedelta.relativedelta(day=1) - timedelta(minutes=1)
+            end = day + relativedelta.relativedelta(days=1) - timedelta(minutes=1)
             await self._update_daily_statistics(day, end)
 
         missing_months = list(set([get_month(x) for x in missing]))
@@ -306,21 +316,17 @@ class BillService:
     ) -> list[Bill]:
         """Compile bills assuming PVPC billing."""
 
-        energy_dt = [x.datetime for x in energy]
-        pvpc_dt = [x.datetime for x in pvpc]
-
-        # reduce computation range to valid periods
-        pvpc_valid_dt = []
-        for contract in contracts:
-            for dt in pvpc_dt:
-                if dt in pvpc_valid_dt:
-                    continue
-                if contract.date_start <= dt <= contract.date_end:
-                    pvpc_valid_dt.append(dt)
-        pvpc_valid_dt = [x for x in pvpc_valid_dt if x in energy_dt]
-
-        e = {x.datetime: x for x in energy if x.datetime in pvpc_valid_dt}
-        p = {x.datetime: x for x in pvpc if x.datetime in pvpc_valid_dt}
+        # reduce computation to timestamps present in both series and covered
+        # by a contract (set membership instead of nested list scans)
+        common_dt = {x.datetime for x in energy} & {x.datetime for x in pvpc}
+        valid_dt = {
+            dt
+            for dt in common_dt
+            if any(c.date_start <= dt <= c.date_end for c in contracts)
+        }
+
+        e = {x.datetime: x for x in energy if x.datetime in valid_dt}
+        p = {x.datetime: x for x in pvpc if x.datetime in valid_dt}
         b: dict[datetime, Bill] = {}
 
         for dt in e.keys():

{e_data-2.0.1.dev127 → e_data-2.0.1.dev129}/edata/services/data_service.py

@@ -11,7 +11,14 @@ from tempfile import gettempdir
 
 from dateutil import relativedelta
 
-from edata.core.utils import get_db_path, get_day, get_month, get_tariff, redacted_cups
+from edata.core.utils import (
+    get_db_path,
+    get_day,
+    get_month,
+    get_tariff,
+    iter_month_windows,
+    redacted_cups,
+)
 from edata.database.controller import EdataDB
 from edata.models import Contract, Energy, Power, Statistics, Supply
 from edata.models.bill import EnergyPrice
@@ -120,7 +127,7 @@ class DataService:
         missing = await self._find_missing_stats()
         for day in missing:
             _LOGGER.info("%s updating daily statistics for date %s", self._scups, day)
-            end = day + relativedelta.relativedelta(day=1) - timedelta(hours=1)
+            end = day + relativedelta.relativedelta(days=1) - timedelta(hours=1)
             await self._update_daily_statistics(day, end)
 
         missing_months = await self._find_missing_stats("month")
@@ -128,7 +135,7 @@ class DataService:
             _LOGGER.info(
                 "%s updating monthly statistics for date %s", self._scups, month
             )
-            end = month + relativedelta.relativedelta(month=1) - timedelta(hours=1)
+            end = month + relativedelta.relativedelta(months=1) - timedelta(hours=1)
             await self._update_monthly_statistics(month, end)
 
     async def update(
@@ -332,10 +339,15 @@ class DataService:
         return False
 
     async def update_statistics(self, start: datetime, end: datetime) -> None:
-        """Update the statistics during a period."""
+        """Update the statistics during a period, one month at a time.
 
-        await self._update_daily_statistics(start, end)
-        await self._update_monthly_statistics(start, end)
+        Iterating monthly windows keeps the energy loaded for compilation bounded
+        to a single month instead of the whole history.
+        """
+
+        for win_start, win_end in iter_month_windows(start, end):
+            await self._update_daily_statistics(win_start, win_end)
+            await self._update_monthly_statistics(win_start, win_end)
 
     async def _update_daily_statistics(self, start: datetime, end: datetime) -> None:
         """Update daily statistics within a date range."""

e_data-2.0.1.dev129/edata/tests/test_utils.py

@@ -0,0 +1,60 @@
+"""Tests for edata.core.utils helpers."""
+
+from datetime import datetime, timedelta
+
+import pytest
+
+from edata.core.utils import get_month, iter_month_windows
+
+
+def _assert_windows_valid(
+    windows: list[tuple[datetime, datetime]],
+    start: datetime,
+    end: datetime,
+) -> None:
+    """Assert windows tile [start, end] contiguously, one calendar month each."""
+    assert windows[0][0] == start
+    assert windows[-1][1] == end
+    for win_start, win_end in windows:
+        assert win_start <= win_end
+        # a window never straddles a calendar-month boundary
+        assert get_month(win_start) == get_month(win_end)
+    for (_, prev_end), (next_start, _) in zip(windows, windows[1:], strict=False):
+        # contiguous and non-overlapping
+        assert next_start == prev_end + timedelta(microseconds=1)
+
+
+@pytest.mark.parametrize(
+    ("start", "end", "expected_len"),
+    [
+        # sub-month range collapses to a single window
+        (datetime(2023, 1, 5), datetime(2023, 1, 20), 1),
+        # spans three calendar months
+        (datetime(2023, 1, 15), datetime(2023, 3, 10), 3),
+        # exact month boundaries
+        (datetime(2023, 1, 1), datetime(2023, 2, 28, 23, 59, 59), 2),
+        # crosses a year boundary
+        (datetime(2022, 12, 20), datetime(2023, 2, 1), 3),
+        # spring-forward DST month is handled like any other (naive datetimes)
+        (datetime(2023, 3, 1), datetime(2023, 4, 15), 2),
+    ],
+    ids=["sub-month", "three-months", "exact-boundaries", "year-boundary", "dst-month"],
+)
+def test_iter_month_windows_tiles_range(
+    start: datetime, end: datetime, expected_len: int
+) -> None:
+    """Windows cover the range contiguously, one calendar month at a time."""
+    windows = list(iter_month_windows(start, end))
+    assert len(windows) == expected_len
+    _assert_windows_valid(windows, start, end)
+
+
+def test_iter_month_windows_single_instant() -> None:
+    """A zero-length range yields exactly one window covering that instant."""
+    moment = datetime(2023, 6, 15, 10, 30)
+    assert list(iter_month_windows(moment, moment)) == [(moment, moment)]
+
+
+def test_iter_month_windows_empty_when_end_before_start() -> None:
+    """An inverted range yields no windows."""
+    assert list(iter_month_windows(datetime(2023, 2, 1), datetime(2023, 1, 1))) == []

{e_data-2.0.1.dev127 → e_data-2.0.1.dev129}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "e-data"
-version = "2.0.1.dev127"
+version = "2.0.1.dev129"
 description = "Python library for managing spanish energy data from various web providers"
 readme = "README.md"
 authors = [