e-data 2.0.1.dev129__tar.gz → 2.0.1.dev131__tar.gz

{e_data-2.0.1.dev129/e_data.egg-info → e_data-2.0.1.dev131}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: e-data
-Version: 2.0.1.dev129
+Version: 2.0.1.dev131
 Summary: Python library for managing spanish energy data from various web providers
 Author-email: VMG <vmayorg@outlook.es>
 License:                     GNU GENERAL PUBLIC LICENSE

{e_data-2.0.1.dev129 → e_data-2.0.1.dev131/e_data.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: e-data
-Version: 2.0.1.dev129
+Version: 2.0.1.dev131
 Summary: Python library for managing spanish energy data from various web providers
 Author-email: VMG <vmayorg@outlook.es>
 License:                     GNU GENERAL PUBLIC LICENSE

{e_data-2.0.1.dev129 → e_data-2.0.1.dev131}/e_data.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ e_data.egg-info/top_level.txt
 edata/__init__.py
 edata/cli.py
 edata/core/__init__.py
+edata/core/completion.py
 edata/core/const.py
 edata/core/utils.py
 edata/database/__init__.py
@@ -27,7 +28,9 @@ edata/providers/redata.py
 edata/services/bill_service.py
 edata/services/data_service.py
 edata/tests/__init__.py
+edata/tests/test_completion.py
 edata/tests/test_datadis_connector.py
+edata/tests/test_incremental.py
 edata/tests/test_redata_connector.py
 edata/tests/test_services.py
 edata/tests/test_utils.py

e_data-2.0.1.dev131/edata/core/completion.py

@@ -0,0 +1,114 @@
+"""Incremental completion tracking for statistics/bill compilation.
+
+The durable source of truth is the per-bucket ``complete`` flag stored on the
+statistics/bill rows. A ``PendingLedger`` is the cheap, in-memory work set of the
+day/month buckets a single sync still needs to (re)compile -- seeded from the
+incomplete rows plus the small recent range past the last complete bucket -- so a
+sync never has to scan the whole energy/bill history.
+
+A bucket is *final* once it reaches its nominal hour count, or once its period has
+fully elapsed and is older than ``SETTLE_DAYS`` (so DST days, partial months, and
+hours Datadis never delivered leave the ledger instead of being recompiled
+forever).
+"""
+
+import calendar
+import typing
+from datetime import datetime, timedelta
+
+from dateutil.relativedelta import relativedelta
+
+from edata.core.utils import get_day, get_month
+
+# Past periods older than this settle window are treated as final regardless of
+# their hour count. Matches the ~monthly window Datadis uses to revise data.
+SETTLE_DAYS = 30
+
+HOURS_PER_DAY = 24
+
+
+def day_buckets(start: datetime, end: datetime) -> typing.Iterator[datetime]:
+    """Yield each day-start in ``[start, end]``."""
+
+    day = get_day(start)
+    while day <= end:
+        yield day
+        day += timedelta(days=1)
+
+
+def _elapsed_beyond_settle(period_end: datetime, now: datetime, settle_days: int) -> bool:
+    """Return whether a period ended more than ``settle_days`` before ``now``."""
+
+    return period_end < now - timedelta(days=settle_days)
+
+
+def is_day_final(
+    day: datetime, delta_h: float, now: datetime, settle_days: int = SETTLE_DAYS
+) -> bool:
+    """Return whether a daily bucket needs no further recompilation."""
+
+    if delta_h >= HOURS_PER_DAY:
+        return True
+    return _elapsed_beyond_settle(day + timedelta(days=1), now, settle_days)
+
+
+def is_month_final(
+    month: datetime, delta_h: float, now: datetime, settle_days: int = SETTLE_DAYS
+) -> bool:
+    """Return whether a monthly bucket needs no further recompilation."""
+
+    nominal = calendar.monthrange(month.year, month.month)[1] * HOURS_PER_DAY
+    if delta_h >= nominal:
+        return True
+    return _elapsed_beyond_settle(month + relativedelta(months=1), now, settle_days)
+
+
+class PendingLedger:
+    """In-memory set of day/month buckets still needing (re)compilation."""
+
+    def __init__(self) -> None:
+        self.days: set[datetime] = set()
+        self.months: set[datetime] = set()
+
+    def mark_range(self, start: datetime, end: datetime) -> None:
+        """Mark every day (and its month) spanning ``[start, end]`` as pending."""
+
+        for day in day_buckets(start, end):
+            self.days.add(day)
+            self.months.add(get_month(day))
+
+    def add_days(self, days: typing.Iterable[datetime]) -> None:
+        """Mark specific days (and their months) as pending."""
+
+        for value in days:
+            day = get_day(value)
+            self.days.add(day)
+            self.months.add(get_month(day))
+
+    def add_months(self, months: typing.Iterable[datetime]) -> None:
+        """Mark specific months as pending."""
+
+        for value in months:
+            self.months.add(get_month(value))
+
+    def days_in(self, month: datetime) -> list[datetime]:
+        """Return the sorted pending days that belong to ``month``."""
+
+        return sorted(day for day in self.days if get_month(day) == month)
+
+    def sorted_months(self) -> list[datetime]:
+        """Return the pending months in chronological order."""
+
+        return sorted(self.months)
+
+    def resolve_day(self, day: datetime, *, final: bool) -> None:
+        """Drop a day from the ledger once it is final."""
+
+        if final:
+            self.days.discard(day)
+
+    def resolve_month(self, month: datetime, *, final: bool) -> None:
+        """Drop a month from the ledger once it is final."""
+
+        if final:
+            self.months.discard(month)

{e_data-2.0.1.dev129 → e_data-2.0.1.dev131}/edata/database/controller.py

@@ -228,6 +228,26 @@ class EdataDB:
             result = await session.exec(q.get_last_bill(cups))
             return result.first()
 
+    async def get_last_complete_statistic(
+        self, cups: str, type_: typing.Literal["day", "month"]
+    ) -> StatisticsModel | None:
+        """Get the most recent complete statistics record by cups and type."""
+
+        await self._ensure_tables()
+        async with AsyncSession(self.engine) as session:
+            result = await session.exec(q.get_last_complete_statistic(cups, type_))
+            return result.first()
+
+    async def get_last_complete_bill(
+        self, cups: str, type_: typing.Literal["hour", "day", "month"]
+    ) -> BillModel | None:
+        """Get the most recent complete bill record by cups and type."""
+
+        await self._ensure_tables()
+        async with AsyncSession(self.engine) as session:
+            result = await session.exec(q.get_last_complete_bill(cups, type_))
+            return result.first()
+
     async def add_contract(self, cups: str, contract: Contract) -> ContractModel | None:
         """Add or update a contract record."""
 

{e_data-2.0.1.dev129 → e_data-2.0.1.dev131}/edata/database/queries.py

@@ -152,6 +152,19 @@ def list_statistics(
     return query
 
 
+def get_last_complete_statistic(
+    cups: str, type_: typing.Literal["day", "month"]
+) -> SelectOfScalar[StatisticsModel]:
+    """Query that selects the most recent complete statistics record."""
+
+    query = select(StatisticsModel).where(StatisticsModel.cups == cups)
+    query = query.where(StatisticsModel.type == type_)
+    query = query.where(StatisticsModel.complete == True)  # noqa: E712
+    query = query.order_by(desc(StatisticsModel.datetime))
+    query = query.limit(1)
+    return query
+
+
 def list_bill(
     cups: str,
     type_: typing.Literal["hour", "day", "month"],
@@ -185,6 +198,19 @@ def get_last_bill(
     return query
 
 
+def get_last_complete_bill(
+    cups: str, type_: typing.Literal["hour", "day", "month"]
+) -> SelectOfScalar[BillModel]:
+    """Query that selects the most recent complete bill record."""
+
+    query = select(BillModel).where(BillModel.cups == cups)
+    query = query.where(BillModel.type == type_)
+    query = query.where(BillModel.complete == True)  # noqa: E712
+    query = query.order_by(desc(BillModel.datetime))
+    query = query.limit(1)
+    return query
+
+
 def delete_bill(cups: str, date_from: datetime | None = None) -> Delete:
     """Query that deletes bill records, optionally from a datetime onwards."""
 

{e_data-2.0.1.dev129 → e_data-2.0.1.dev131}/edata/services/bill_service.py

@@ -1,5 +1,4 @@
 import asyncio
-import calendar
 import logging
 import os
 import typing
@@ -10,6 +9,7 @@ from tempfile import gettempdir
 from dateutil import relativedelta
 from jinja2 import Environment
 
+from edata.core.completion import PendingLedger, is_day_final, is_month_final
 from edata.core.utils import (
     get_db_path,
     get_contract_for_dt,
@@ -143,8 +143,7 @@ class BillService:
             )
 
         _LOGGER.debug("%s updating daily and monthly bills", self._scups)
-        await self.update_statistics(start, end)
-        await self.fix_missing_statistics()
+        await self.update_statistics_incremental()
 
     async def clear_bills(self, since: datetime | None = None) -> None:
         """Delete stored bills for this cups, optionally only from a datetime onwards."""
@@ -180,76 +179,81 @@ class BillService:
         )
 
     async def update_statistics(self, start: datetime, end: datetime):
-        """Update the statistics during a period, one month at a time."""
+        """Compile the daily/monthly bills for a range plus any incomplete backlog."""
 
-        for win_start, win_end in iter_month_windows(start, end):
-            await self._update_daily_statistics(win_start, win_end)
-            await self._update_monthly_statistics(win_start, win_end)
+        ledger = PendingLedger()
+        ledger.mark_range(start, end)
+        await self._seed_pending_backlog(ledger)
+        await self._compile_pending(ledger)
 
-    async def _update_daily_statistics(self, start: datetime, end: datetime) -> None:
-        """Update daily statistics within a date range."""
+    async def update_statistics_incremental(self) -> None:
+        """Compile only the daily/monthly bills that are new or still incomplete.
 
-        day_start = get_day(start)
-        day_end = end
-        daily = await self.db.list_bill(self._cups, "day", day_start, day_end)
+        The pending buckets come from cheap indexed queries (incomplete rows plus
+        the recent range past the last complete bucket), so a sync never rolls up
+        the whole hourly-bill history.
+        """
 
-        complete = []
-        for stat in daily:
-            if stat.complete:
-                complete.append(stat.datetime)
-                continue
+        ledger = PendingLedger()
+        await self._seed_pending_backlog(ledger)
+        await self._seed_pending_recent(ledger)
+        await self._compile_pending(ledger)
 
-        data = await self.get_bills(day_start, day_end)
-        stats = await asyncio.to_thread(
-            self._compile_statistics, data, get_day, skip=complete
-        )
+    async def _seed_pending_backlog(self, ledger: PendingLedger) -> None:
+        """Seed the ledger with the buckets whose stored bills are incomplete."""
 
-        for stat in stats:
-            is_complete = stat.delta_h == 24
-            await self.db.add_bill(self._cups, "day", stat, "mix", is_complete)
-            if not is_complete:
-                _LOGGER.info(
-                    "%s daily statistics for %s are incomplete",
-                    self._scups,
-                    stat.datetime.date(),
-                )
+        day_incomplete = await self.db.list_bill(self._cups, "day", complete=False)
+        month_incomplete = await self.db.list_bill(self._cups, "month", complete=False)
+        ledger.add_days(x.datetime for x in day_incomplete)
+        ledger.add_months(x.datetime for x in month_incomplete)
 
-    async def _update_monthly_statistics(self, start: datetime, end: datetime) -> None:
-        """Update monthly statistics within a date range."""
+    async def _seed_pending_recent(self, ledger: PendingLedger) -> None:
+        """Seed the ledger with the range past the last complete daily bill."""
 
-        month_start = get_month(start)
-        month_end = end
-        monthly = await self.db.list_bill(self._cups, "month", month_start, month_end)
+        last_bill = await self.db.get_last_bill(self._cups)
+        if not last_bill:
+            return
 
-        complete = []
-        for stat in monthly:
-            if stat.complete:
-                complete.append(stat.datetime)
-                continue
+        last_complete = await self.db.get_last_complete_bill(self._cups, "day")
+        if last_complete:
+            start = get_day(last_complete.datetime) + timedelta(days=1)
+        else:
+            supply = await self.db.get_supply(self._cups)
+            start = get_day(
+                supply.data.date_start if supply else last_bill.datetime
+            )
+        ledger.mark_range(start, last_bill.datetime)
 
-        data = await self.get_bills(month_start, month_end)
-        stats = await asyncio.to_thread(
-            self._compile_statistics, data, get_month, skip=complete
-        )
+    async def _compile_pending(self, ledger: PendingLedger) -> None:
+        """Roll up the pending buckets, one month of hourly bills loaded at a time."""
 
-        for stat in stats:
-            target_hours = (
-                calendar.monthrange(stat.datetime.year, stat.datetime.month)[1] * 24
+        now = datetime.now()
+        for month in ledger.sorted_months():
+            month_end = (
+                month + relativedelta.relativedelta(months=1) - timedelta(microseconds=1)
             )
-            is_complete = stat.delta_h == target_hours
-            await self.db.add_bill(self._cups, "month", stat, "mix", is_complete)
-            if not is_complete:
-                _LOGGER.info(
-                    "%s monthly statistics for %s are incomplete",
-                    self._scups,
-                    stat.datetime.date(),
-                )
-
-    async def _find_missing_stats(self) -> list[datetime]:
-        """Return the list of days that are missing billing data."""
+            data = await self.get_bills(month, month_end, "hour")
 
-        stats = await self.db.list_bill(self._cups, "day", complete=False)
-        return [x.datetime for x in stats]
+            for day in ledger.days_in(month):
+                day_end = day + timedelta(days=1) - timedelta(microseconds=1)
+                day_data = [x for x in data if day <= x.datetime <= day_end]
+                stat = await asyncio.to_thread(
+                    self._compile_statistics, day_data, get_day
+                )
+                delta_h = stat[0].delta_h if stat else 0.0
+                final = is_day_final(day, delta_h, now)
+                if stat:
+                    await self.db.add_bill(self._cups, "day", stat[0], "mix", final)
+                ledger.resolve_day(day, final=final)
+
+            month_stat = await asyncio.to_thread(
+                self._compile_statistics, data, get_month
+            )
+            delta_h = month_stat[0].delta_h if month_stat else 0.0
+            final = is_month_final(month, delta_h, now)
+            if month_stat:
+                await self.db.add_bill(self._cups, "month", month_stat[0], "mix", final)
+            ledger.resolve_month(month, final=final)
 
     def _compile_statistics(
         self,
@@ -290,23 +294,6 @@ class BillService:
 
         return [agg_data[x] for x in agg_data]
 
-    async def fix_missing_statistics(self) -> None:
-        """Recompile statistics to fix missing data."""
-
-        missing = await self._find_missing_stats()
-        for day in missing:
-            _LOGGER.debug("%s updating daily statistics for date %s", self._scups, day)
-            end = day + relativedelta.relativedelta(days=1) - timedelta(minutes=1)
-            await self._update_daily_statistics(day, end)
-
-        missing_months = list(set([get_month(x) for x in missing]))
-        for month in missing_months:
-            _LOGGER.debug(
-                "%s updating monthly statistics for date %s", self._scups, month
-            )
-            end = month + relativedelta.relativedelta(months=1) - timedelta(minutes=1)
-            await self._update_monthly_statistics(month, end)
-
     def simulate_pvpc(
         self,
         contracts: list[Contract],

{e_data-2.0.1.dev129 → e_data-2.0.1.dev131}/edata/services/data_service.py

@@ -1,7 +1,6 @@
 """Definition of a service for telemetry data handling."""
 
 import asyncio
-import calendar
 import logging
 import os
 import typing
@@ -11,12 +10,12 @@ from tempfile import gettempdir
 
 from dateutil import relativedelta
 
+from edata.core.completion import PendingLedger, is_day_final, is_month_final
 from edata.core.utils import (
     get_db_path,
     get_day,
     get_month,
     get_tariff,
-    iter_month_windows,
     redacted_cups,
 )
 from edata.database.controller import EdataDB
@@ -121,23 +120,6 @@ class DataService:
 
         return await self._get_last_energy_dt()
 
-    async def fix_missing_statistics(self) -> None:
-        """Recompile statistics to fix missing data."""
-
-        missing = await self._find_missing_stats()
-        for day in missing:
-            _LOGGER.info("%s updating daily statistics for date %s", self._scups, day)
-            end = day + relativedelta.relativedelta(days=1) - timedelta(hours=1)
-            await self._update_daily_statistics(day, end)
-
-        missing_months = await self._find_missing_stats("month")
-        for month in missing_months:
-            _LOGGER.info(
-                "%s updating monthly statistics for date %s", self._scups, month
-            )
-            end = month + relativedelta.relativedelta(months=1) - timedelta(hours=1)
-            await self._update_monthly_statistics(month, end)
-
     async def update(
         self,
         start_date: datetime | None = None,
@@ -228,9 +210,6 @@ class DataService:
                 _LOGGER.info("%s trying to update energy for %s", scups, day)
                 await self.update_energy(day, day + timedelta(days=1))
 
-            # and recreate statistics
-            await self.fix_missing_statistics()
-
             # fetch upstream records
             await self.update_energy(last_energy_dt + timedelta(hours=1), end_date)
         else:
@@ -243,8 +222,8 @@ class DataService:
         # fetch pvpc data
         await self.update_pvpc(start_date, end_date)
 
-        # update new statistics
-        await self.update_statistics(start_date, end_date)
+        # (re)compile only the statistics buckets that are new or still incomplete
+        await self.update_statistics_incremental()
 
         return True
 
@@ -339,76 +318,79 @@ class DataService:
         return False
 
     async def update_statistics(self, start: datetime, end: datetime) -> None:
-        """Update the statistics during a period, one month at a time.
+        """Compile the statistics for a range plus any incomplete backlog."""
 
-        Iterating monthly windows keeps the energy loaded for compilation bounded
-        to a single month instead of the whole history.
-        """
+        ledger = PendingLedger()
+        ledger.mark_range(start, end)
+        await self._seed_pending_backlog(ledger)
+        await self._compile_pending(ledger)
 
-        for win_start, win_end in iter_month_windows(start, end):
-            await self._update_daily_statistics(win_start, win_end)
-            await self._update_monthly_statistics(win_start, win_end)
+    async def update_statistics_incremental(self) -> None:
+        """Compile only the statistics buckets that are new or still incomplete.
 
-    async def _update_daily_statistics(self, start: datetime, end: datetime) -> None:
-        """Update daily statistics within a date range."""
+        The pending buckets are derived from cheap indexed queries (incomplete
+        rows plus the recent range past the last complete bucket), so a sync never
+        scans the whole energy history.
+        """
 
-        day_start = get_day(start)
-        day_end = end
-        daily = await self.db.list_statistics(self._cups, "day", day_start, day_end)
+        ledger = PendingLedger()
+        await self._seed_pending_backlog(ledger)
+        await self._seed_pending_recent(ledger)
+        await self._compile_pending(ledger)
 
-        complete = []
-        for stat in daily:
-            if stat.complete:
-                complete.append(stat.datetime)
-                continue
+    async def _seed_pending_backlog(self, ledger: PendingLedger) -> None:
+        """Seed the ledger with the buckets whose stored stats are incomplete."""
 
-        data = await self.get_energy(day_start, day_end)
-        stats = await asyncio.to_thread(
-            self._compile_statistics, data, get_day, skip=complete
-        )
+        ledger.add_days(await self._find_missing_stats("day"))
+        ledger.add_months(await self._find_missing_stats("month"))
 
-        for stat in stats:
-            is_complete = stat.delta_h == 24
-            await self.db.add_statistics(self._cups, "day", stat, is_complete)
-            if not is_complete:
-                _LOGGER.info(
-                    "%s daily statistics for %s are incomplete",
-                    self._scups,
-                    stat.datetime.date(),
-                )
-
-    async def _update_monthly_statistics(self, start: datetime, end: datetime) -> None:
-        """Update monthly statistics within a date range."""
+    async def _seed_pending_recent(self, ledger: PendingLedger) -> None:
+        """Seed the ledger with the range past the last complete daily stat."""
 
-        month_start = get_month(start)
-        month_end = end
-        monthly = await self.db.list_statistics(
-            self._cups, "month", month_start, month_end
-        )
+        last_energy = await self.db.get_last_energy(self._cups)
+        if not last_energy:
+            return
 
-        complete = []
-        for stat in monthly:
-            if stat.complete:
-                complete.append(stat.datetime)
-                continue
+        last_complete = await self.db.get_last_complete_statistic(self._cups, "day")
+        if last_complete:
+            start = get_day(last_complete.datetime) + timedelta(days=1)
+        else:
+            supply = await self.db.get_supply(self._cups)
+            start = get_day(
+                supply.data.date_start if supply else last_energy.datetime
+            )
+        ledger.mark_range(start, last_energy.datetime)
 
-        data = await self.get_energy(month_start, month_end)
-        stats = await asyncio.to_thread(
-            self._compile_statistics, data, get_month, skip=complete
-        )
+    async def _compile_pending(self, ledger: PendingLedger) -> None:
+        """Compile the pending buckets, one month of energy loaded at a time."""
 
-        for stat in stats:
-            target_hours = (
-                calendar.monthrange(stat.datetime.year, stat.datetime.month)[1] * 24
+        now = datetime.now()
+        for month in ledger.sorted_months():
+            month_end = (
+                month + relativedelta.relativedelta(months=1) - timedelta(microseconds=1)
             )
-            is_complete = stat.delta_h == target_hours
-            await self.db.add_statistics(self._cups, "month", stat, is_complete)
-            if not is_complete:
-                _LOGGER.info(
-                    "%s monthly statistics for %s are incomplete",
-                    self._scups,
-                    stat.datetime.date(),
+            data = await self.get_energy(month, month_end)
+
+            for day in ledger.days_in(month):
+                day_end = day + timedelta(days=1) - timedelta(microseconds=1)
+                day_data = [x for x in data if day <= x.datetime <= day_end]
+                stat = await asyncio.to_thread(
+                    self._compile_statistics, day_data, get_day
                 )
+                delta_h = stat[0].delta_h if stat else 0.0
+                final = is_day_final(day, delta_h, now)
+                if stat:
+                    await self.db.add_statistics(self._cups, "day", stat[0], final)
+                ledger.resolve_day(day, final=final)
+
+            month_stat = await asyncio.to_thread(
+                self._compile_statistics, data, get_month
+            )
+            delta_h = month_stat[0].delta_h if month_stat else 0.0
+            final = is_month_final(month, delta_h, now)
+            if month_stat:
+                await self.db.add_statistics(self._cups, "month", month_stat[0], final)
+            ledger.resolve_month(month, final=final)
 
     def _compile_statistics(
         self,

e_data-2.0.1.dev131/edata/tests/test_completion.py

@@ -0,0 +1,104 @@
+"""Tests for the incremental completion ledger."""
+
+from datetime import datetime
+
+import pytest
+
+from edata.core.completion import (
+    PendingLedger,
+    day_buckets,
+    is_day_final,
+    is_month_final,
+)
+
+NOW = datetime(2026, 7, 26, 12, 0)
+
+
+def test_day_buckets_covers_range_inclusively() -> None:
+    """Every day-start in the range is yielded once."""
+    buckets = list(day_buckets(datetime(2023, 1, 30, 5), datetime(2023, 2, 2)))
+    assert buckets == [
+        datetime(2023, 1, 30),
+        datetime(2023, 1, 31),
+        datetime(2023, 2, 1),
+        datetime(2023, 2, 2),
+    ]
+
+
+@pytest.mark.parametrize(
+    ("day", "delta_h", "expected"),
+    [
+        (datetime(2026, 7, 25), 24, True),  # nominal reached
+        (datetime(2026, 7, 25), 25, True),  # fall-back DST day (25h)
+        (datetime(2026, 7, 25), 23, False),  # recent + short -> keep
+        (datetime(2026, 5, 1), 23, True),  # short but settled (>30d old)
+    ],
+    ids=["nominal", "dst-25h", "recent-short", "settled-short"],
+)
+def test_is_day_final(day: datetime, delta_h: float, expected: bool) -> None:
+    """A day is final at nominal hours or once settled past the window."""
+    assert is_day_final(day, delta_h, NOW) is expected
+
+
+@pytest.mark.parametrize(
+    ("month", "delta_h", "expected"),
+    [
+        (datetime(2026, 7, 1), 31 * 24, True),  # nominal (July has 31 days)
+        (datetime(2026, 7, 1), 700, False),  # current month, short -> keep
+        (datetime(2026, 5, 1), 700, True),  # short but settled
+    ],
+    ids=["nominal", "current-short", "settled-short"],
+)
+def test_is_month_final(month: datetime, delta_h: float, expected: bool) -> None:
+    """A month is final at nominal hours or once settled past the window."""
+    assert is_month_final(month, delta_h, NOW) is expected
+
+
+def test_mark_range_adds_days_and_their_months() -> None:
+    """mark_range records each day and the month it belongs to."""
+    ledger = PendingLedger()
+    ledger.mark_range(datetime(2023, 1, 30), datetime(2023, 2, 1))
+
+    assert ledger.days == {
+        datetime(2023, 1, 30),
+        datetime(2023, 1, 31),
+        datetime(2023, 2, 1),
+    }
+    assert ledger.months == {datetime(2023, 1, 1), datetime(2023, 2, 1)}
+
+
+def test_add_days_and_months_normalize_and_pair() -> None:
+    """add_days aligns to day-start and also marks the month; add_months aligns."""
+    ledger = PendingLedger()
+    ledger.add_days([datetime(2023, 3, 15, 8, 30)])
+    ledger.add_months([datetime(2023, 4, 20, 0, 0)])
+
+    assert ledger.days == {datetime(2023, 3, 15)}
+    assert ledger.months == {datetime(2023, 3, 1), datetime(2023, 4, 1)}
+
+
+def test_days_in_filters_and_sorts_by_month() -> None:
+    """days_in returns only the sorted days belonging to the given month."""
+    ledger = PendingLedger()
+    ledger.add_days(
+        [datetime(2023, 1, 31), datetime(2023, 1, 5), datetime(2023, 2, 3)]
+    )
+    assert ledger.days_in(datetime(2023, 1, 1)) == [
+        datetime(2023, 1, 5),
+        datetime(2023, 1, 31),
+    ]
+
+
+def test_resolve_only_drops_when_final() -> None:
+    """resolve_* removes a bucket when final and keeps it otherwise."""
+    ledger = PendingLedger()
+    ledger.mark_range(datetime(2023, 1, 1), datetime(2023, 1, 1))
+
+    ledger.resolve_day(datetime(2023, 1, 1), final=False)
+    assert datetime(2023, 1, 1) in ledger.days
+
+    ledger.resolve_day(datetime(2023, 1, 1), final=True)
+    assert datetime(2023, 1, 1) not in ledger.days
+
+    ledger.resolve_month(datetime(2023, 1, 1), final=True)
+    assert datetime(2023, 1, 1) not in ledger.months

e_data-2.0.1.dev131/edata/tests/test_incremental.py

@@ -0,0 +1,108 @@
+"""Incrementality tests: a steady-state sync only touches recent buckets."""
+
+from collections.abc import AsyncIterator
+from datetime import datetime, timedelta
+from unittest.mock import patch
+
+import pytest
+import pytest_asyncio
+
+from edata.core.utils import get_month
+from edata.database.controller import EdataDB
+from edata.models import Energy, Supply
+from edata.services.data_service import DataService
+
+CUPS = "ESXXXXXXXXXXXXXXXXTEST"
+
+
+def _reset_singleton() -> None:
+    EdataDB._instance = None
+    EdataDB._engine = None
+    EdataDB._db_url = None
+
+
+@pytest_asyncio.fixture
+async def data_service(tmp_path) -> AsyncIterator[DataService]:
+    """A DataService on an isolated on-disk database, singleton reset around it."""
+    _reset_singleton()
+    with (
+        patch("edata.services.data_service.DatadisConnector"),
+        patch("edata.services.data_service.REDataConnector"),
+    ):
+        service = DataService(CUPS, "user", "pwd", storage_path=str(tmp_path))
+    yield service
+    if EdataDB._engine is not None:
+        await EdataDB._engine.dispose()
+    _reset_singleton()
+
+
+def _hourly_energy(start: datetime, end: datetime) -> list[Energy]:
+    """Build deterministic hourly energy records over [start, end)."""
+    records = []
+    cursor = start
+    while cursor < end:
+        records.append(
+            Energy(
+                datetime=cursor,
+                delta_h=1.0,
+                consumption_kwh=0.1,
+                surplus_kwh=0.0,
+                generation_kwh=0.0,
+                selfconsumption_kwh=0.0,
+                real=True,
+            )
+        )
+        cursor += timedelta(hours=1)
+    return records
+
+
+async def _seed(service: DataService, start: datetime, end: datetime) -> None:
+    await service.db.add_supply(
+        Supply(
+            cups=CUPS,
+            date_start=start,
+            date_end=end,
+            address=None,
+            postal_code=None,
+            province=None,
+            municipality=None,
+            distributor=None,
+            point_type=5,
+            distributor_code="2",
+        )
+    )
+    await service.db.add_energy_list(CUPS, _hourly_energy(start, end))
+
+
+@pytest.mark.asyncio
+async def test_incremental_sync_only_loads_recent_month(
+    data_service: DataService,
+) -> None:
+    """After a full build, an incremental sync must not reload complete months."""
+    now = datetime.now().replace(minute=0, second=0, microsecond=0)
+    start = now - timedelta(days=70)
+    await _seed(data_service, start, now)
+
+    # Full build establishes completeness for every past month.
+    await data_service.update_statistics(start, now)
+    daily_before = await data_service.get_statistics("day")
+
+    # Spy on the energy loads issued by a subsequent incremental sync.
+    loaded_ranges: list[tuple[datetime, datetime]] = []
+    original = data_service.get_energy
+
+    async def _spy(start=None, end=None):
+        loaded_ranges.append((start, end))
+        return await original(start=start, end=end)
+
+    with patch.object(data_service, "get_energy", side_effect=_spy):
+        await data_service.update_statistics_incremental()
+
+    # Something was loaded, but never anything before the current month.
+    current_month = get_month(now)
+    assert loaded_ranges
+    assert all(rng[0] >= current_month for rng in loaded_ranges)
+
+    # Stats are unchanged by the idempotent re-run.
+    daily_after = await data_service.get_statistics("day")
+    assert len(daily_after) == len(daily_before)

{e_data-2.0.1.dev129 → e_data-2.0.1.dev131}/edata/tests/test_services.py

@@ -83,8 +83,7 @@ async def populated_data_service(mock_connector, energy, power, storage_dir):
         await ds.update_energy(start_date, end_date)
         await ds.update_power(start_date, end_date)
 
-        await ds._update_daily_statistics(start_date, end_date)
-        await ds._update_monthly_statistics(start_date, end_date)
+        await ds.update_statistics(start_date, end_date)
 
         return ds
 

{e_data-2.0.1.dev129 → e_data-2.0.1.dev131}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "e-data"
-version = "2.0.1.dev129"
+version = "2.0.1.dev131"
 description = "Python library for managing spanish energy data from various web providers"
 readme = "README.md"
 authors = [