e-data 2.0.0.dev125__tar.gz → 2.0.1__tar.gz

{e_data-2.0.0.dev125/e_data.egg-info → e_data-2.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: e-data
-Version: 2.0.0.dev125
+Version: 2.0.1
 Summary: Python library for managing spanish energy data from various web providers
 Author-email: VMG <vmayorg@outlook.es>
 License:                     GNU GENERAL PUBLIC LICENSE
@@ -696,7 +696,7 @@ Requires-Dist: Jinja2<4,>=3.1
 Requires-Dist: pydantic<3,>=2.10
 Requires-Dist: python_dateutil<3,>=2.8
 Requires-Dist: Requests<3,>=2.31
-Requires-Dist: SQLAlchemy<3,>=2.0
+Requires-Dist: SQLAlchemy[asyncio]<3,>=2.0
 Requires-Dist: sqlmodel<0.1,>=0.0.22
 Requires-Dist: typer<1,>=0.12
 Requires-Dist: aiosqlite<1,>=0.21

{e_data-2.0.0.dev125 → e_data-2.0.1/e_data.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: e-data
-Version: 2.0.0.dev125
+Version: 2.0.1
 Summary: Python library for managing spanish energy data from various web providers
 Author-email: VMG <vmayorg@outlook.es>
 License:                     GNU GENERAL PUBLIC LICENSE
@@ -696,7 +696,7 @@ Requires-Dist: Jinja2<4,>=3.1
 Requires-Dist: pydantic<3,>=2.10
 Requires-Dist: python_dateutil<3,>=2.8
 Requires-Dist: Requests<3,>=2.31
-Requires-Dist: SQLAlchemy<3,>=2.0
+Requires-Dist: SQLAlchemy[asyncio]<3,>=2.0
 Requires-Dist: sqlmodel<0.1,>=0.0.22
 Requires-Dist: typer<1,>=0.12
 Requires-Dist: aiosqlite<1,>=0.21

{e_data-2.0.0.dev125 → e_data-2.0.1}/e_data.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ e_data.egg-info/top_level.txt
 edata/__init__.py
 edata/cli.py
 edata/core/__init__.py
+edata/core/completion.py
 edata/core/const.py
 edata/core/utils.py
 edata/database/__init__.py
@@ -27,6 +28,9 @@ edata/providers/redata.py
 edata/services/bill_service.py
 edata/services/data_service.py
 edata/tests/__init__.py
+edata/tests/test_completion.py
 edata/tests/test_datadis_connector.py
+edata/tests/test_incremental.py
 edata/tests/test_redata_connector.py
-edata/tests/test_services.py
+edata/tests/test_services.py
+edata/tests/test_utils.py

{e_data-2.0.0.dev125 → e_data-2.0.1}/e_data.egg-info/requires.txt

@@ -7,7 +7,7 @@ Jinja2<4,>=3.1
 pydantic<3,>=2.10
 python_dateutil<3,>=2.8
 Requests<3,>=2.31
-SQLAlchemy<3,>=2.0
+SQLAlchemy[asyncio]<3,>=2.0
 sqlmodel<0.1,>=0.0.22
 typer<1,>=0.12
 aiosqlite<1,>=0.21

e_data-2.0.1/edata/core/completion.py

@@ -0,0 +1,114 @@
+"""Incremental completion tracking for statistics/bill compilation.
+
+The durable source of truth is the per-bucket ``complete`` flag stored on the
+statistics/bill rows. A ``PendingLedger`` is the cheap, in-memory work set of the
+day/month buckets a single sync still needs to (re)compile -- seeded from the
+incomplete rows plus the small recent range past the last complete bucket -- so a
+sync never has to scan the whole energy/bill history.
+
+A bucket is *final* once it reaches its nominal hour count, or once its period has
+fully elapsed and is older than ``SETTLE_DAYS`` (so DST days, partial months, and
+hours Datadis never delivered leave the ledger instead of being recompiled
+forever).
+"""
+
+import calendar
+import typing
+from datetime import datetime, timedelta
+
+from dateutil.relativedelta import relativedelta
+
+from edata.core.utils import get_day, get_month
+
+# Past periods older than this settle window are treated as final regardless of
+# their hour count. Matches the ~monthly window Datadis uses to revise data.
+SETTLE_DAYS = 30
+
+HOURS_PER_DAY = 24
+
+
+def day_buckets(start: datetime, end: datetime) -> typing.Iterator[datetime]:
+    """Yield each day-start in ``[start, end]``."""
+
+    day = get_day(start)
+    while day <= end:
+        yield day
+        day += timedelta(days=1)
+
+
+def _elapsed_beyond_settle(period_end: datetime, now: datetime, settle_days: int) -> bool:
+    """Return whether a period ended more than ``settle_days`` before ``now``."""
+
+    return period_end < now - timedelta(days=settle_days)
+
+
+def is_day_final(
+    day: datetime, delta_h: float, now: datetime, settle_days: int = SETTLE_DAYS
+) -> bool:
+    """Return whether a daily bucket needs no further recompilation."""
+
+    if delta_h >= HOURS_PER_DAY:
+        return True
+    return _elapsed_beyond_settle(day + timedelta(days=1), now, settle_days)
+
+
+def is_month_final(
+    month: datetime, delta_h: float, now: datetime, settle_days: int = SETTLE_DAYS
+) -> bool:
+    """Return whether a monthly bucket needs no further recompilation."""
+
+    nominal = calendar.monthrange(month.year, month.month)[1] * HOURS_PER_DAY
+    if delta_h >= nominal:
+        return True
+    return _elapsed_beyond_settle(month + relativedelta(months=1), now, settle_days)
+
+
+class PendingLedger:
+    """In-memory set of day/month buckets still needing (re)compilation."""
+
+    def __init__(self) -> None:
+        self.days: set[datetime] = set()
+        self.months: set[datetime] = set()
+
+    def mark_range(self, start: datetime, end: datetime) -> None:
+        """Mark every day (and its month) spanning ``[start, end]`` as pending."""
+
+        for day in day_buckets(start, end):
+            self.days.add(day)
+            self.months.add(get_month(day))
+
+    def add_days(self, days: typing.Iterable[datetime]) -> None:
+        """Mark specific days (and their months) as pending."""
+
+        for value in days:
+            day = get_day(value)
+            self.days.add(day)
+            self.months.add(get_month(day))
+
+    def add_months(self, months: typing.Iterable[datetime]) -> None:
+        """Mark specific months as pending."""
+
+        for value in months:
+            self.months.add(get_month(value))
+
+    def days_in(self, month: datetime) -> list[datetime]:
+        """Return the sorted pending days that belong to ``month``."""
+
+        return sorted(day for day in self.days if get_month(day) == month)
+
+    def sorted_months(self) -> list[datetime]:
+        """Return the pending months in chronological order."""
+
+        return sorted(self.months)
+
+    def resolve_day(self, day: datetime, *, final: bool) -> None:
+        """Drop a day from the ledger once it is final."""
+
+        if final:
+            self.days.discard(day)
+
+    def resolve_month(self, month: datetime, *, final: bool) -> None:
+        """Drop a month from the ledger once it is final."""
+
+        if final:
+            self.months.discard(month)

{e_data-2.0.0.dev125 → e_data-2.0.1}/edata/core/utils.py

@@ -1,9 +1,12 @@
 """Collection of utilities."""
 
 import logging
-from datetime import datetime
+import typing
+from datetime import datetime, timedelta
+from functools import lru_cache
 from pathlib import Path
 
+from dateutil.relativedelta import relativedelta
 import holidays
 
 from edata.models.supply import Contract
@@ -19,10 +22,22 @@ TARIFF_BY_WEEKDAY = [[], [], [5, 6]]
 _LOGGER = logging.getLogger(__name__)
 
 
+@lru_cache(maxsize=None)
+def _es_holidays(year: int) -> holidays.HolidayBase:
+    """Return (and cache) Spanish holidays for a single year.
+
+    Building the holidays calendar is expensive; get_tariff is called once per
+    hourly record, so without this cache a full-history rebuild spends minutes
+    reconstructing the same handful of yearly calendars.
+    """
+
+    return holidays.country_holidays("ES", years=year)
+
+
 def get_tariff(dt: datetime) -> int:
     """Return the tariff for the selected datetime."""
 
-    hdays = holidays.country_holidays("ES")
+    hdays = _es_holidays(dt.year)
     hour = dt.hour
     weekday = dt.weekday()
 
@@ -64,6 +79,27 @@ def get_day(dt: datetime) -> datetime:
     return dt.replace(hour=0, minute=0, second=0, microsecond=0)
 
 
+def iter_month_windows(
+    start: datetime, end: datetime
+) -> typing.Iterator[tuple[datetime, datetime]]:
+    """Yield contiguous ``(window_start, window_end)`` pairs, one calendar month each.
+
+    Windows are non-overlapping and cover ``[start, end]`` inclusively; the first
+    starts at ``start`` and the last ends at ``end``. Iterating a month at a time
+    keeps the working set bounded to a single month regardless of history length.
+    """
+
+    if end < start:
+        return
+
+    win_start = start
+    while win_start <= end:
+        next_month = get_month(win_start) + relativedelta(months=1)
+        win_end = min(next_month - timedelta(microseconds=1), end)
+        yield win_start, win_end
+        win_start = next_month
+
+
 def redacted_cups(cups: str) -> str:
     """Return an anonymized version of the cups identifier."""
 

{e_data-2.0.0.dev125 → e_data-2.0.1}/edata/database/controller.py

@@ -3,7 +3,7 @@ import os
 import typing
 from datetime import datetime
 
-from sqlalchemy import Select, insert
+from sqlalchemy import Select, event, insert
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
 from sqlmodel import SQLModel
@@ -28,6 +28,25 @@ _LOGGER = logging.getLogger(__name__)
 T = typing.TypeVar("T", bound=SQLModel)
 
 
+def _set_sqlite_pragmas(dbapi_connection, connection_record) -> None:
+    """Tune SQLite for this write-heavy, single-writer workload.
+
+    WAL lets readers proceed while a write is in flight and avoids an fsync per
+    statement; synchronous=NORMAL is durable under WAL (only a crash mid-checkpoint
+    could lose the last transaction, and the data is re-fetchable from Datadis);
+    busy_timeout avoids spurious "database is locked" errors under concurrency.
+    journal_mode is persisted in the file; the others are per-connection.
+    """
+
+    cursor = dbapi_connection.cursor()
+    try:
+        cursor.execute("PRAGMA journal_mode=WAL")
+        cursor.execute("PRAGMA synchronous=NORMAL")
+        cursor.execute("PRAGMA busy_timeout=5000")
+    finally:
+        cursor.close()
+
+
 class EdataDB:
 
     _instance = None
@@ -39,10 +58,11 @@ class EdataDB:
         if cls._instance is None:
             cls._instance = super().__new__(cls)
             cls._db_url = db_url
-            cls._engine = create_async_engine(db_url, future=True)
-            # Ensure parent directory exists
+            # Ensure parent directory exists before the first connection is opened.
             dir_path = os.path.dirname(os.path.abspath(sqlite_path))
             os.makedirs(dir_path, exist_ok=True)
+            cls._engine = create_async_engine(db_url, future=True)
+            event.listen(cls._engine.sync_engine, "connect", _set_sqlite_pragmas)
             cls._instance._tables_initialized = False
         elif db_url != cls._db_url:
             raise ValueError("EdataDB already initialized with a different db_url")
@@ -208,6 +228,26 @@ class EdataDB:
             result = await session.exec(q.get_last_bill(cups))
             return result.first()
 
+    async def get_last_complete_statistic(
+        self, cups: str, type_: typing.Literal["day", "month"]
+    ) -> StatisticsModel | None:
+        """Get the most recent complete statistics record by cups and type."""
+
+        await self._ensure_tables()
+        async with AsyncSession(self.engine) as session:
+            result = await session.exec(q.get_last_complete_statistic(cups, type_))
+            return result.first()
+
+    async def get_last_complete_bill(
+        self, cups: str, type_: typing.Literal["hour", "day", "month"]
+    ) -> BillModel | None:
+        """Get the most recent complete bill record by cups and type."""
+
+        await self._ensure_tables()
+        async with AsyncSession(self.engine) as session:
+            result = await session.exec(q.get_last_complete_bill(cups, type_))
+            return result.first()
+
     async def add_contract(self, cups: str, contract: Contract) -> ContractModel | None:
         """Add or update a contract record."""
 

{e_data-2.0.0.dev125 → e_data-2.0.1}/edata/database/queries.py

@@ -152,6 +152,19 @@ def list_statistics(
     return query
 
 
+def get_last_complete_statistic(
+    cups: str, type_: typing.Literal["day", "month"]
+) -> SelectOfScalar[StatisticsModel]:
+    """Query that selects the most recent complete statistics record."""
+
+    query = select(StatisticsModel).where(StatisticsModel.cups == cups)
+    query = query.where(StatisticsModel.type == type_)
+    query = query.where(StatisticsModel.complete == True)  # noqa: E712
+    query = query.order_by(desc(StatisticsModel.datetime))
+    query = query.limit(1)
+    return query
+
+
 def list_bill(
     cups: str,
     type_: typing.Literal["hour", "day", "month"],
@@ -185,6 +198,19 @@ def get_last_bill(
     return query
 
 
+def get_last_complete_bill(
+    cups: str, type_: typing.Literal["hour", "day", "month"]
+) -> SelectOfScalar[BillModel]:
+    """Query that selects the most recent complete bill record."""
+
+    query = select(BillModel).where(BillModel.cups == cups)
+    query = query.where(BillModel.type == type_)
+    query = query.where(BillModel.complete == True)  # noqa: E712
+    query = query.order_by(desc(BillModel.datetime))
+    query = query.limit(1)
+    return query
+
+
 def delete_bill(cups: str, date_from: datetime | None = None) -> Delete:
     """Query that deletes bill records, optionally from a datetime onwards."""
 

{e_data-2.0.0.dev125 → e_data-2.0.1}/edata/providers/datadis.py

@@ -403,10 +403,17 @@ class DatadisConnector:
         for i in response.get("timeCurve", []):
             if "consumptionKWh" in i:
                 if all(k in i for k in GET_CONSUMPTION_DATA_MANDATORY_FIELDS):
-                    hour = str(int(i["time"].split(":")[0]) - 1)
+                    raw_hour = int(i["time"].split(":")[0])
+                    # Datadis delivers hours 1..24 (end-of-interval). A sporadic
+                    # i-DE glitch emits an extra "00:00" row on a day already
+                    # carrying its full 24 hours, so drop it -- 01:00..24:00
+                    # already covers the day -- instead of remapping onto 23:00
+                    # and overlapping that day's 24:00 slot.
+                    if raw_hour == 0:
+                        continue
                     date_as_dt = datetime.strptime(
-                        f"{i['date']} {hour.zfill(2)}:00", "%Y/%m/%d %H:%M"
-                    )
+                        i["date"], "%Y/%m/%d"
+                    ) + timedelta(hours=raw_hour - 1)
                     if not (start_date <= date_as_dt <= end_date):
                         continue  # skip element if dt is out of range
 

{e_data-2.0.0.dev125 → e_data-2.0.1}/edata/services/bill_service.py

@@ -1,5 +1,4 @@
 import asyncio
-import calendar
 import logging
 import os
 import typing
@@ -10,12 +9,14 @@ from tempfile import gettempdir
 from dateutil import relativedelta
 from jinja2 import Environment
 
+from edata.core.completion import PendingLedger, is_day_final, is_month_final
 from edata.core.utils import (
     get_db_path,
     get_contract_for_dt,
     get_day,
     get_month,
     get_tariff,
+    iter_month_windows,
     redacted_cups,
 )
 from edata.database.controller import EdataDB
@@ -107,35 +108,42 @@ class BillService:
         # fetch contracts
         contracts = await self._get_contracts()
 
-        # fetch and filter energy items
-        energy = await self._get_energy()
-
-        _LOGGER.debug("%s compiling missing hourly bills", self._scups)
         if is_pvpc:
-            pvpc = await self._get_pvpc()
             billing_rules = PVPCBillingRules(**billing_rules.model_dump())
-            bills = await asyncio.to_thread(
-                self.simulate_pvpc, contracts, energy, pvpc, billing_rules
-            )
             confighash = f"pvpc-{hash(billing_rules.model_dump_json())}"
         else:
-            bills = await asyncio.to_thread(
-                self.simulate_custom, contracts, energy, billing_rules
-            )
             confighash = f"custom-{hash(billing_rules.model_dump_json())}"
 
-        _LOGGER.debug("%s pushing hourly bills", self._scups)
-        await self.db.add_bill_list(
-            cups=self._cups,
-            type_="hour",
-            confhash=confighash,
-            complete=True,
-            bill=bills,
-        )
+        # compile and persist hourly bills one month at a time so the energy
+        # (and simulated bills) held in memory stay bounded to a single month
+        for win_start, win_end in iter_month_windows(start, end):
+            energy = await self._get_energy(win_start, win_end)
+            if not energy:
+                continue
+
+            _LOGGER.debug(
+                "%s compiling hourly bills for %s..%s", self._scups, win_start, win_end
+            )
+            if is_pvpc:
+                pvpc = await self._get_pvpc(win_start, win_end)
+                bills = await asyncio.to_thread(
+                    self.simulate_pvpc, contracts, energy, pvpc, billing_rules
+                )
+            else:
+                bills = await asyncio.to_thread(
+                    self.simulate_custom, contracts, energy, billing_rules
+                )
+
+            await self.db.add_bill_list(
+                cups=self._cups,
+                type_="hour",
+                confhash=confighash,
+                complete=True,
+                bill=bills,
+            )
 
         _LOGGER.debug("%s updating daily and monthly bills", self._scups)
-        await self.update_statistics(start, end)
-        await self.fix_missing_statistics()
+        await self.update_statistics_incremental()
 
     async def clear_bills(self, since: datetime | None = None) -> None:
         """Delete stored bills for this cups, optionally only from a datetime onwards."""
@@ -171,75 +179,81 @@ class BillService:
         )
 
     async def update_statistics(self, start: datetime, end: datetime):
-        """Update the statistics during a period."""
+        """Compile the daily/monthly bills for a range plus any incomplete backlog."""
 
-        await self._update_daily_statistics(start, end)
-        await self._update_monthly_statistics(start, end)
+        ledger = PendingLedger()
+        ledger.mark_range(start, end)
+        await self._seed_pending_backlog(ledger)
+        await self._compile_pending(ledger)
 
-    async def _update_daily_statistics(self, start: datetime, end: datetime) -> None:
-        """Update daily statistics within a date range."""
+    async def update_statistics_incremental(self) -> None:
+        """Compile only the daily/monthly bills that are new or still incomplete.
 
-        day_start = get_day(start)
-        day_end = end
-        daily = await self.db.list_bill(self._cups, "day", day_start, day_end)
+        The pending buckets come from cheap indexed queries (incomplete rows plus
+        the recent range past the last complete bucket), so a sync never rolls up
+        the whole hourly-bill history.
+        """
 
-        complete = []
-        for stat in daily:
-            if stat.complete:
-                complete.append(stat.datetime)
-                continue
+        ledger = PendingLedger()
+        await self._seed_pending_backlog(ledger)
+        await self._seed_pending_recent(ledger)
+        await self._compile_pending(ledger)
 
-        data = await self.get_bills(day_start, day_end)
-        stats = await asyncio.to_thread(
-            self._compile_statistics, data, get_day, skip=complete
-        )
+    async def _seed_pending_backlog(self, ledger: PendingLedger) -> None:
+        """Seed the ledger with the buckets whose stored bills are incomplete."""
 
-        for stat in stats:
-            is_complete = stat.delta_h == 24
-            await self.db.add_bill(self._cups, "day", stat, "mix", is_complete)
-            if not is_complete:
-                _LOGGER.info(
-                    "%s daily statistics for %s are incomplete",
-                    self._scups,
-                    stat.datetime.date(),
-                )
+        day_incomplete = await self.db.list_bill(self._cups, "day", complete=False)
+        month_incomplete = await self.db.list_bill(self._cups, "month", complete=False)
+        ledger.add_days(x.datetime for x in day_incomplete)
+        ledger.add_months(x.datetime for x in month_incomplete)
 
-    async def _update_monthly_statistics(self, start: datetime, end: datetime) -> None:
-        """Update monthly statistics within a date range."""
+    async def _seed_pending_recent(self, ledger: PendingLedger) -> None:
+        """Seed the ledger with the range past the last complete daily bill."""
 
-        month_start = get_month(start)
-        month_end = end
-        monthly = await self.db.list_bill(self._cups, "month", month_start, month_end)
+        last_bill = await self.db.get_last_bill(self._cups)
+        if not last_bill:
+            return
 
-        complete = []
-        for stat in monthly:
-            if stat.complete:
-                complete.append(stat.datetime)
-                continue
+        last_complete = await self.db.get_last_complete_bill(self._cups, "day")
+        if last_complete:
+            start = get_day(last_complete.datetime) + timedelta(days=1)
+        else:
+            supply = await self.db.get_supply(self._cups)
+            start = get_day(
+                supply.data.date_start if supply else last_bill.datetime
+            )
+        ledger.mark_range(start, last_bill.datetime)
 
-        data = await self.get_bills(month_start, month_end)
-        stats = await asyncio.to_thread(
-            self._compile_statistics, data, get_month, skip=complete
-        )
+    async def _compile_pending(self, ledger: PendingLedger) -> None:
+        """Roll up the pending buckets, one month of hourly bills loaded at a time."""
 
-        for stat in stats:
-            target_hours = (
-                calendar.monthrange(stat.datetime.year, stat.datetime.month)[1] * 24
+        now = datetime.now()
+        for month in ledger.sorted_months():
+            month_end = (
+                month + relativedelta.relativedelta(months=1) - timedelta(microseconds=1)
             )
-            is_complete = stat.delta_h == target_hours
-            await self.db.add_bill(self._cups, "month", stat, "mix", is_complete)
-            if not is_complete:
-                _LOGGER.info(
-                    "%s monthly statistics for %s are incomplete",
-                    self._scups,
-                    stat.datetime.date(),
-                )
-
-    async def _find_missing_stats(self) -> list[datetime]:
-        """Return the list of days that are missing billing data."""
+            data = await self.get_bills(month, month_end, "hour")
 
-        stats = await self.db.list_bill(self._cups, "day", complete=False)
-        return [x.datetime for x in stats]
+            for day in ledger.days_in(month):
+                day_end = day + timedelta(days=1) - timedelta(microseconds=1)
+                day_data = [x for x in data if day <= x.datetime <= day_end]
+                stat = await asyncio.to_thread(
+                    self._compile_statistics, day_data, get_day
+                )
+                delta_h = stat[0].delta_h if stat else 0.0
+                final = is_day_final(day, delta_h, now)
+                if stat:
+                    await self.db.add_bill(self._cups, "day", stat[0], "mix", final)
+                ledger.resolve_day(day, final=final)
+
+            month_stat = await asyncio.to_thread(
+                self._compile_statistics, data, get_month
+            )
+            delta_h = month_stat[0].delta_h if month_stat else 0.0
+            final = is_month_final(month, delta_h, now)
+            if month_stat:
+                await self.db.add_bill(self._cups, "month", month_stat[0], "mix", final)
+            ledger.resolve_month(month, final=final)
 
     def _compile_statistics(
         self,
@@ -280,23 +294,6 @@ class BillService:
 
         return [agg_data[x] for x in agg_data]
 
-    async def fix_missing_statistics(self) -> None:
-        """Recompile statistics to fix missing data."""
-
-        missing = await self._find_missing_stats()
-        for day in missing:
-            _LOGGER.debug("%s updating daily statistics for date %s", self._scups, day)
-            end = day + relativedelta.relativedelta(day=1) - timedelta(minutes=1)
-            await self._update_daily_statistics(day, end)
-
-        missing_months = list(set([get_month(x) for x in missing]))
-        for month in missing_months:
-            _LOGGER.debug(
-                "%s updating monthly statistics for date %s", self._scups, month
-            )
-            end = month + relativedelta.relativedelta(months=1) - timedelta(minutes=1)
-            await self._update_monthly_statistics(month, end)
-
     def simulate_pvpc(
         self,
         contracts: list[Contract],
@@ -306,21 +303,17 @@ class BillService:
     ) -> list[Bill]:
         """Compile bills assuming PVPC billing."""
 
-        energy_dt = [x.datetime for x in energy]
-        pvpc_dt = [x.datetime for x in pvpc]
-
-        # reduce computation range to valid periods
-        pvpc_valid_dt = []
-        for contract in contracts:
-            for dt in pvpc_dt:
-                if dt in pvpc_valid_dt:
-                    continue
-                if contract.date_start <= dt <= contract.date_end:
-                    pvpc_valid_dt.append(dt)
-        pvpc_valid_dt = [x for x in pvpc_valid_dt if x in energy_dt]
-
-        e = {x.datetime: x for x in energy if x.datetime in pvpc_valid_dt}
-        p = {x.datetime: x for x in pvpc if x.datetime in pvpc_valid_dt}
+        # reduce computation to timestamps present in both series and covered
+        # by a contract (set membership instead of nested list scans)
+        common_dt = {x.datetime for x in energy} & {x.datetime for x in pvpc}
+        valid_dt = {
+            dt
+            for dt in common_dt
+            if any(c.date_start <= dt <= c.date_end for c in contracts)
+        }
+
+        e = {x.datetime: x for x in energy if x.datetime in valid_dt}
+        p = {x.datetime: x for x in pvpc if x.datetime in valid_dt}
         b: dict[datetime, Bill] = {}
 
         for dt in e.keys():