e-data 2.0.2.dev134__py3-none-any.whl → 2.0.2.dev136__py3-none-any.whl

{e_data-2.0.2.dev134.dist-info → e_data-2.0.2.dev136.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: e-data
-Version: 2.0.2.dev134
+Version: 2.0.2.dev136
 Summary: Python library for managing spanish energy data from various web providers
 Author-email: VMG <vmayorg@outlook.es>
 License:                     GNU GENERAL PUBLIC LICENSE
@@ -697,7 +697,7 @@ Requires-Dist: pydantic<3,>=2.10
 Requires-Dist: python_dateutil<3,>=2.8
 Requires-Dist: Requests<3,>=2.31
 Requires-Dist: SQLAlchemy[asyncio]<3,>=2.0
-Requires-Dist: sqlmodel<0.1,>=0.0.22
+Requires-Dist: sqlmodel<0.0.45,>=0.0.22
 Requires-Dist: typer<1,>=0.12
 Requires-Dist: aiosqlite<1,>=0.21
 Dynamic: license-file

{e_data-2.0.2.dev134.dist-info → e_data-2.0.2.dev136.dist-info}/RECORD

@@ -1,4 +1,4 @@
-e_data-2.0.2.dev134.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+e_data-2.0.2.dev136.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
 edata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edata/cli.py,sha256=CsBYDggt0grkLb4cebrckOvJyHr9PaGTnlWqT4yjuFo,5215
 edata/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -6,9 +6,9 @@ edata/core/completion.py,sha256=A0ekfu7eCjaLblUYO5DrvaWuDPAO9k1buzy2qdfjdNc,3914
 edata/core/const.py,sha256=X9J2f7lt9jXJnghqz0JFi-J7USa3xlYvbuiki70eHNE,43
 edata/core/utils.py,sha256=uQX8GmNjuhof56jiXXoV-Fk-d590OuIoBJbM9ymZyV8,3194
 edata/database/__init__.py,sha256=HP_pDbGLzeMY9Ufm_Dps5cxzW_3YTonT8w8dstG558c,46
-edata/database/controller.py,sha256=EW7YKnq1Ti_rslVAuIq5TDjf8THyWBAt-rwMxnsSvdQ,18507
-edata/database/models.py,sha256=4rlRpPErULqYr6QJxZar1NBpJPWTcHftDN1MB3bwGyA,5096
-edata/database/queries.py,sha256=yfUAwHtEv6JkcGf8aGiQ1Z5ihbgdMtKLGNiwvBikb7Y,8013
+edata/database/controller.py,sha256=bI3cE1Q4SK9U1Eep_61EhlmyVjgYH87F5lfBLhwSpmg,23211
+edata/database/models.py,sha256=tSkxf2OKgT1m2F1td3j3GtYcboUtSuYbhnHWQtpvJuY,5525
+edata/database/queries.py,sha256=XziDSG2V-PVFmvRkj8UrEvgrc9Qc-ShhnxgsPHIsyik,8304
 edata/database/utils.py,sha256=V-FJfyAh8YJBGHB85BbWtG3Ilq09pM9Gl-L86TffB4g,1062
 edata/database/migrations/__init__.py,sha256=P8VXZzfyceOV3GknYXif2HEMB8Bo4B7f7vKdIC02ZO0,1203
 edata/database/migrations/base.py,sha256=V7jfMaTl6XPCwM6Wei4bNxfOKSKanIHF30UgH-_CJNM,444
@@ -18,19 +18,20 @@ edata/models/bill.py,sha256=-qZ4l9f8BnPFlykFRoNYnmRe6-ELWmuNh6usks_k4X0,2674
 edata/models/data.py,sha256=O1aj0H_G92mnJjm6mYhC8fnl8ZOoatGiokRf8b7ziD8,2388
 edata/models/supply.py,sha256=qM0Q22-7PfBN93-Tu3ljAE58p5gfob0YWw-o5FccjrI,878
 edata/providers/__init__.py,sha256=5xaRKPttYM6EpVE1I_vhUAF25L9pEM_qK_5Ix6xnQe0,104
-edata/providers/datadis.py,sha256=t8xrGYpTORHUH_FNZjwrvGnx6L_CBaaD0pKtvfFTITU,20588
-edata/providers/redata.py,sha256=uvxO2TTtSNSfPRWHZifLjbaP7zAr6O7CZ1oRQFNbIhQ,3122
-edata/services/bill_service.py,sha256=vukgeQAN0fMjAkyI9cqPqQ4wSRvPdOaswfSR1KheIYc,14947
-edata/services/data_service.py,sha256=henmYVQUynCTl1g83mZuU4Fwi2d5Qk6yhEjXOOmvhk4,18278
+edata/providers/datadis.py,sha256=VZ_MhO3x1crSmOH2dn3SWJ5Eq1gWUIDCEvB9oRRMZfQ,21468
+edata/providers/redata.py,sha256=i6aBjunyZMb34SGgAgNQvaNu_GFoe5P2rUIY-_vOElw,3727
+edata/services/bill_service.py,sha256=KAM78u7TpoVF-KTlB7bqj2Kp-rJERiPPoGTJCc-7rM0,15264
+edata/services/data_service.py,sha256=JJCdivjrwNt3aLhz3uMBWv06y-EL7UiK2G5EBr97l_8,19374
 edata/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 edata/tests/test_completion.py,sha256=-iyVt-juU3U1HqMuLUJXJthil2-n_4ddWiJq2GwnXTo,3551
-edata/tests/test_datadis_connector.py,sha256=pT161wQdA_GbdutqnWDVkM7itNQlK2NIvEETCi33oDM,11711
+edata/tests/test_controller.py,sha256=E1Blvel2dtS8CiyxErP4V_wpzWlEcxTYA1gW1f8-Gg4,3449
+edata/tests/test_datadis_connector.py,sha256=8Xet2M2eG-kMhiRDTCSH0R5nYO2oIpQTPOuTZKMKoLo,12538
 edata/tests/test_incremental.py,sha256=dn1LRgjBi2HuNYELPKDVdZQlS01hk4lsP9L-vyvtEHo,3479
 edata/tests/test_migrations.py,sha256=3Io2nokZVdv2xU9fCZCMIvwAwPVtGc0UivClSouP3pw,6418
 edata/tests/test_redata_connector.py,sha256=SBvxsma5t-qvz2drxNisNM2P1STTurqEs1TSDgwaPJ8,508
-edata/tests/test_services.py,sha256=BMw-MYXs2L_c1Sg_v4TwAZGOJnx2BoWH8U50M05-ZC8,6816
+edata/tests/test_services.py,sha256=Ya5cbADHj1Qsffb_IuMeDD0BViz8nqrgwazUWK0-hRc,8297
 edata/tests/test_utils.py,sha256=h6yBzT7j3eA42zbXJ-AK9lXcZkjCMf16wnob1oqGvUE,2307
-e_data-2.0.2.dev134.dist-info/METADATA,sha256=piYxSiQvkdZOXOfPXqGyIkoMRqrDCpChwfbqJC7J0R8,48084
-e_data-2.0.2.dev134.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
-e_data-2.0.2.dev134.dist-info/top_level.txt,sha256=Ez-fReWtUVTMcFuH0dzCnoKIgMBHgjLaBuba8QiKC7g,6
-e_data-2.0.2.dev134.dist-info/RECORD,,
+e_data-2.0.2.dev136.dist-info/METADATA,sha256=YZbWL0ZGXBHvfFBODBeXBwD8CcQHH5VXqYqY_NV42GE,48087
+e_data-2.0.2.dev136.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+e_data-2.0.2.dev136.dist-info/top_level.txt,sha256=Ez-fReWtUVTMcFuH0dzCnoKIgMBHgjLaBuba8QiKC7g,6
+e_data-2.0.2.dev136.dist-info/RECORD,,

{e_data-2.0.2.dev134.dist-info → e_data-2.0.2.dev136.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (83.0.0)
+Generator: setuptools (84.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

edata/database/controller.py

@@ -1,12 +1,14 @@
+import asyncio
 import logging
 import os
 import typing
 from datetime import datetime
 
-from sqlalchemy import Select, event, insert
+from sqlalchemy import Select, Table, event, insert, or_
+from sqlalchemy.dialects.sqlite import insert as sqlite_insert
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
-from sqlmodel import SQLModel
+from sqlmodel import SQLModel, UniqueConstraint
 from sqlmodel.ext.asyncio.session import AsyncSession
 from sqlmodel.sql.expression import SelectOfScalar
 
@@ -47,6 +49,34 @@ def _set_sqlite_pragmas(dbapi_connection, connection_record) -> None:
         cursor.close()
 
 
+def _create_missing_indexes(connection) -> None:
+    """Create indexes added after a table was first created.
+
+    ``create_all`` skips tables that already exist, indexes included, so
+    databases created by an older version would never get new indexes.
+    """
+
+    for table in SQLModel.metadata.sorted_tables:
+        for index in table.indexes:
+            index.create(connection, checkfirst=True)
+
+
+def _conflict_columns(table: Table) -> list[str]:
+    """Return the columns that identify a row for upserts on ``table``.
+
+    That is the table's unique constraint, else its unique index, else its
+    primary key.
+    """
+
+    for constraint in table.constraints:
+        if isinstance(constraint, UniqueConstraint):
+            return [c.name for c in constraint.columns]
+    for index in table.indexes:
+        if index.unique:
+            return [c.name for c in index.columns]
+    return [c.name for c in table.primary_key.columns]
+
+
 class EdataDB:
 
     _instance = None
@@ -64,6 +94,7 @@ class EdataDB:
             cls._engine = create_async_engine(db_url, future=True)
             event.listen(cls._engine.sync_engine, "connect", _set_sqlite_pragmas)
             cls._instance._tables_initialized = False
+            cls._instance._tables_lock = asyncio.Lock()
         elif db_url != cls._db_url:
             raise ValueError("EdataDB already initialized with a different db_url")
         return cls._instance
@@ -79,9 +110,15 @@ class EdataDB:
 
         if self._tables_initialized:
             return
-        if self.engine:
+        # concurrent first calls (e.g. several websocket requests right after
+        # startup) would otherwise all see the index missing and race to create
+        # it, failing with "index ... already exists"
+        async with self._tables_lock:
+            if self._tables_initialized or not self.engine:
+                return
             async with self.engine.begin() as conn:
                 await conn.run_sync(SQLModel.metadata.create_all)
+                await conn.run_sync(_create_missing_indexes)
             self._tables_initialized = True
 
     async def _add_one(
@@ -158,33 +195,46 @@ class EdataDB:
     async def _add_or_update_many(
         self,
         session: AsyncSession,
-        queries: list[SelectOfScalar],
-        records: list[T],
-        batch_size: int = 100,
+        model: type[SQLModel],
+        rows: list[dict[str, typing.Any]],
         override: list[str] | None = None,
-    ) -> list[T]:
-        """Updates many records in the database"""
-        if not records:
-            return []
-
-        for i in range(0, len(records), batch_size):
-            chunk_records = records[i : i + batch_size]
-            chunk_queries = queries[i : i + batch_size]
-            try:
-                async with session.begin_nested():
-                    session.add_all(chunk_records)
-                    await session.flush()
-            except IntegrityError:
-                for j, record in enumerate(chunk_records):
-                    await self._add_or_update_one(
-                        session,
-                        chunk_queries[j],
-                        record,
-                        commit=False,
-                        override=override,
-                    )
+    ) -> None:
+        """Insert many rows, updating the existing ones, in a single statement.
+
+        Uses SQLite's ``INSERT ... ON CONFLICT DO UPDATE`` so re-syncing rows that
+        are already stored costs one bulk statement instead of a savepoint, a
+        failed insert and a lookup per row. Existing rows are only rewritten (and
+        their ``updated_at`` bumped) when ``data`` or an ``override`` column
+        actually changed.
+
+        Rows are plain column dicts: building an ORM instance per row only to
+        read it back cost about half of a full-history import. Columns missing
+        from a row take the model defaults, resolved once per batch.
+        """
+        if not rows:
+            return
+
+        table = model.__table__  # type: ignore[attr-defined]
+        defaults = {
+            c.name: model.model_fields[c.name].get_default(call_default_factory=True)
+            for c in table.columns
+            if c.name != "id" and c.name not in rows[0]
+        }
+        rows = [{**defaults, **row} for row in rows]
+
+        updated = ["data", *(override or [])]
+        stmt = sqlite_insert(table)
+        stmt = stmt.on_conflict_do_update(
+            index_elements=_conflict_columns(table),
+            set_={
+                **{c: stmt.excluded[c] for c in updated},
+                "updated_at": stmt.excluded.updated_at,
+            },
+            where=or_(*(table.c[c].is_distinct_from(stmt.excluded[c]) for c in updated)),
+        )
+        connection = await session.connection()
+        await connection.execute(stmt, rows)
         await session.commit()
-        return records
 
     async def get_supply(self, cups: str) -> SupplyModel | None:
         """Get a supply record by cups."""
@@ -212,6 +262,14 @@ class EdataDB:
             result = await session.exec(q.get_last_energy(cups))
             return result.first()
 
+    async def get_last_power(self, cups: str) -> PowerModel | None:
+        """Get the most recent power record by cups."""
+
+        await self._ensure_tables()
+        async with AsyncSession(self.engine) as session:
+            result = await session.exec(q.get_last_power(cups))
+            return result.first()
+
     async def get_last_pvpc(self) -> PVPCModel | None:
         """Get the most recent pvpc."""
 
@@ -287,12 +345,11 @@ class EdataDB:
         async with AsyncSession(self.engine) as session:
             unique_map = {item.datetime: item for item in energy}
             unique = list(unique_map.values())
-            queries = [q.get_energy(cups, x.datetime) for x in unique]
-            items = [
-                EnergyModel(cups=cups, delta_h=x.delta_h, datetime=x.datetime, data=x)
+            rows = [
+                {"cups": cups, "delta_h": x.delta_h, "datetime": x.datetime, "data": x}
                 for x in unique
             ]
-            await self._add_or_update_many(session, queries, items)
+            await self._add_or_update_many(session, EnergyModel, rows)
 
     async def add_power(self, cups: str, power: Power) -> PowerModel | None:
         """Add or update a power record for a given CUPS and Power instance."""
@@ -311,9 +368,8 @@ class EdataDB:
         async with AsyncSession(self.engine) as session:
             unique_map = {item.datetime: item for item in power}
             unique = list(unique_map.values())
-            queries = [q.get_power(cups, x.datetime) for x in unique]
-            items = [PowerModel(cups=cups, datetime=x.datetime, data=x) for x in unique]
-            await self._add_or_update_many(session, queries, items)
+            rows = [{"cups": cups, "datetime": x.datetime, "data": x} for x in unique]
+            await self._add_or_update_many(session, PowerModel, rows)
 
     async def add_pvpc(self, pvpc: EnergyPrice) -> PVPCModel | None:
         """Add or update a pvpc record."""
@@ -332,9 +388,8 @@ class EdataDB:
         async with AsyncSession(self.engine) as session:
             unique_map = {item.datetime: item for item in pvpc}
             unique = list(unique_map.values())
-            queries = [q.get_pvpc(x.datetime) for x in unique]
-            items = [PVPCModel(datetime=x.datetime, data=x) for x in unique]
-            await self._add_or_update_many(session, queries, items)
+            rows = [{"datetime": x.datetime, "data": x} for x in unique]
+            await self._add_or_update_many(session, PVPCModel, rows)
 
     async def add_statistics(
         self,
@@ -357,6 +412,32 @@ class EdataDB:
                 override=["complete"],
             )
 
+    async def add_statistics_list(
+        self,
+        cups: str,
+        type_: typing.Literal["day", "month"],
+        complete: bool,
+        statistics: list[Statistics],
+    ) -> None:
+        """Add or update a list of statistics records."""
+
+        await self._ensure_tables()
+        async with AsyncSession(self.engine) as session:
+            unique_map = {item.datetime: item for item in statistics}
+            rows = [
+                {
+                    "cups": cups,
+                    "datetime": x.datetime,
+                    "type": type_,
+                    "complete": complete,
+                    "data": x,
+                }
+                for x in unique_map.values()
+            ]
+            await self._add_or_update_many(
+                session, StatisticsModel, rows, override=["complete"]
+            )
+
     async def add_bill(
         self,
         cups: str,
@@ -398,20 +479,19 @@ class EdataDB:
         async with AsyncSession(self.engine) as session:
             unique_map = {item.datetime: item for item in bill}
             unique = list(unique_map.values())
-            queries = [q.get_bill(cups, type_, x.datetime) for x in unique]
-            items = [
-                BillModel(
-                    cups=cups,
-                    datetime=x.datetime,
-                    type=type_,
-                    confhash=confhash,
-                    complete=complete,
-                    data=x,
-                )
+            rows = [
+                {
+                    "cups": cups,
+                    "datetime": x.datetime,
+                    "type": type_,
+                    "confhash": confhash,
+                    "complete": complete,
+                    "data": x,
+                }
                 for x in unique
             ]
             await self._add_or_update_many(
-                session, queries, items, override=["complete", "confhash"]
+                session, BillModel, rows, override=["complete", "confhash"]
             )
 
     async def clear_bills(self, cups: str, since: datetime | None = None) -> None:
@@ -440,6 +520,54 @@ class EdataDB:
             result = await session.exec(q.list_contract(cups))
             return result.all()
 
+    async def _list_data(self, query: SelectOfScalar, model: type[SQLModel]) -> list:
+        """Return only the ``data`` payload of the rows selected by ``query``.
+
+        Skips building an ORM instance per row, which nearly halves the cost of
+        reading a month of hourly records.
+        """
+
+        await self._ensure_tables()
+        async with AsyncSession(self.engine) as session:
+            result = await session.exec(
+                query.with_only_columns(model.data)  # type: ignore[attr-defined]
+            )
+            return list(result.all())
+
+    async def list_energy_data(
+        self,
+        cups: str,
+        date_from: datetime | None = None,
+        date_to: datetime | None = None,
+    ) -> list[Energy]:
+        """List the energy data (without row metadata)."""
+
+        return await self._list_data(
+            q.list_energy(cups, date_from, date_to), EnergyModel
+        )
+
+    async def list_pvpc_data(
+        self,
+        date_from: datetime | None = None,
+        date_to: datetime | None = None,
+    ) -> list[EnergyPrice]:
+        """List the pvpc data (without row metadata)."""
+
+        return await self._list_data(q.list_pvpc(date_from, date_to), PVPCModel)
+
+    async def list_bill_data(
+        self,
+        cups: str,
+        type_: typing.Literal["hour", "day", "month"],
+        date_from: datetime | None = None,
+        date_to: datetime | None = None,
+    ) -> list[Bill]:
+        """List the bill data (without row metadata)."""
+
+        return await self._list_data(
+            q.list_bill(cups, type_, date_from, date_to), BillModel
+        )
+
     async def list_energy(
         self,
         cups: str,

edata/database/models.py

@@ -1,12 +1,23 @@
 import typing
 from datetime import datetime as dt
 
-from sqlmodel import AutoString, Column, Field, SQLModel, UniqueConstraint
+from sqlmodel import AutoString, Column, Field, Index, SQLModel, UniqueConstraint
 
 from edata.database.utils import PydanticJSON
 from edata.models import Bill, Contract, Energy, EnergyPrice, Power, Statistics, Supply
 
 
+def _now() -> dt:
+    """Return the current time.
+
+    A plain Python function on purpose: with the ``dt.now`` builtin as
+    ``default_factory`` pydantic re-inspects its signature on every instance,
+    which made building each row ~6x slower.
+    """
+
+    return dt.now()
+
+
 class SupplyModel(SQLModel, table=True):
 
     __tablename__ = "supply"  # type: ignore
@@ -16,9 +27,9 @@ class SupplyModel(SQLModel, table=True):
     cups: str = Field(default=None, primary_key=True)
     data: Supply = Field(sa_column=Column(PydanticJSON(Supply)))
     version: int = Field(default=1)
-    created_at: dt = Field(default_factory=dt.now, nullable=False)
+    created_at: dt = Field(default_factory=_now, nullable=False)
     updated_at: dt = Field(
-        default_factory=dt.now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
+        default_factory=_now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
     )
 
 
@@ -37,9 +48,9 @@ class ContractModel(SQLModel, table=True):
     data: Contract = Field(sa_column=Column(PydanticJSON(Contract)))
 
     version: int = Field(default=1)
-    created_at: dt = Field(default_factory=dt.now, nullable=False)
+    created_at: dt = Field(default_factory=_now, nullable=False)
     updated_at: dt = Field(
-        default_factory=dt.now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
+        default_factory=_now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
     )
 
 
@@ -51,6 +62,9 @@ class EnergyModel(SQLModel, table=True):
         UniqueConstraint(
             "cups", "delta_h", "datetime", name="uq_energy_cups_delta_datetime"
         ),
+        # serves the per-cups range scans and "latest record" lookups without
+        # sorting the whole table
+        Index("ix_energy_cups_datetime", "cups", "datetime"),
         {"extend_existing": True},
     )
 
@@ -62,9 +76,9 @@ class EnergyModel(SQLModel, table=True):
     data: Energy = Field(sa_column=Column(PydanticJSON(Energy)))
 
     version: int = Field(default=1)
-    created_at: dt = Field(default_factory=dt.now, nullable=False)
+    created_at: dt = Field(default_factory=_now, nullable=False)
     updated_at: dt = Field(
-        default_factory=dt.now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
+        default_factory=_now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
     )
 
 
@@ -84,9 +98,9 @@ class PowerModel(SQLModel, table=True):
     data: Power = Field(sa_column=Column(PydanticJSON(Power)))
 
     version: int = Field(default=1)
-    created_at: dt = Field(default_factory=dt.now, nullable=False)
+    created_at: dt = Field(default_factory=_now, nullable=False)
     updated_at: dt = Field(
-        default_factory=dt.now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
+        default_factory=_now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
     )
 
 
@@ -112,9 +126,9 @@ class StatisticsModel(SQLModel, table=True):
     data: Statistics = Field(sa_column=Column(PydanticJSON(Statistics)))
 
     version: int = Field(default=1)
-    created_at: dt = Field(default_factory=dt.now, nullable=False)
+    created_at: dt = Field(default_factory=_now, nullable=False)
     updated_at: dt = Field(
-        default_factory=dt.now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
+        default_factory=_now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
     )
 
 
@@ -127,9 +141,9 @@ class PVPCModel(SQLModel, table=True):
     data: EnergyPrice = Field(sa_column=Column(PydanticJSON(EnergyPrice)))
 
     version: int = Field(default=1)
-    created_at: dt = Field(default_factory=dt.now, nullable=False)
+    created_at: dt = Field(default_factory=_now, nullable=False)
     updated_at: dt = Field(
-        default_factory=dt.now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
+        default_factory=_now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
     )
 
 
@@ -156,7 +170,7 @@ class BillModel(SQLModel, table=True):
     data: Bill = Field(sa_column=Column(PydanticJSON(Bill)))
 
     version: int = Field(default=1)
-    created_at: dt = Field(default_factory=dt.now, nullable=False)
+    created_at: dt = Field(default_factory=_now, nullable=False)
     updated_at: dt = Field(
-        default_factory=dt.now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
+        default_factory=_now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
     )

edata/database/queries.py

@@ -118,6 +118,18 @@ def list_power(
     return query
 
 
+def get_last_power(
+    cups: str,
+) -> SelectOfScalar[PowerModel]:
+    """Query that selects the most recent power record."""
+
+    query = select(PowerModel).where(PowerModel.cups == cups)
+    query = query.order_by(desc(PowerModel.datetime))
+    query = query.limit(1)
+
+    return query
+
+
 # Queries for "statistics" table
 def get_statistics(
     cups: str, type_: typing.Literal["day", "month"], datetime_: datetime | None = None

edata/providers/datadis.py

@@ -78,7 +78,15 @@ class DatadisConnector:
         password: str,
         enable_smart_fetch: bool = True,
         storage_path: str | None = None,
+        session: aiohttp.ClientSession | None = None,
     ) -> None:
+        """Init the connector.
+
+        ``session`` lets the caller share a long-lived aiohttp session (e.g. Home
+        Assistant's) so requests reuse pooled TLS connections; without it a
+        short-lived session is opened per request.
+        """
+        self._session = session
         self._usr = username
         self._pwd = password
         self._token = {}
@@ -96,6 +104,15 @@ class DatadisConnector:
         os.makedirs(self._recent_cache_dir, exist_ok=True)
         self._cache = diskcache.Cache(self._recent_cache_dir)
 
+    @contextlib.asynccontextmanager
+    async def _client(self) -> typing.AsyncIterator[aiohttp.ClientSession]:
+        """Yield the shared session, or a short-lived one when none was given."""
+        if self._session is not None:
+            yield self._session
+        else:
+            async with aiohttp.ClientSession() as session:
+                yield session
+
     def _get_hash(self, item: str) -> str:
         """Return a hash."""
         return hashlib.md5(item.encode()).hexdigest()
@@ -127,7 +144,7 @@ class DatadisConnector:
         _LOGGER.debug("No token found, fetching a new one")
         is_valid_token = False
         timeout = aiohttp.ClientTimeout(total=TIMEOUT)
-        async with aiohttp.ClientSession(timeout=timeout) as session:
+        async with self._client() as session:
             try:
                 async with session.post(
                     URL_TOKEN,
@@ -135,6 +152,7 @@ class DatadisConnector:
                         TOKEN_USERNAME: self._usr,
                         TOKEN_PASSWD: self._pwd,
                     },
+                    timeout=timeout,
                 ) as response:
                     text = await response.text()
                     if response.status == 200:
@@ -209,12 +227,14 @@ class DatadisConnector:
                 if self._token.get("headers"):
                     headers.update(self._token["headers"])
                 timeout = aiohttp.ClientTimeout(total=TIMEOUT)
-                async with aiohttp.ClientSession(timeout=timeout) as session:
+                async with self._client() as session:
                     async with session.get(
                         url + params,
                         headers=headers,
+                        timeout=timeout,
                     ) as reply:
-                        text = await reply.text()
+                        # the body is only needed as text to log errors; decoding
+                        # it here as well would decode every (large) payload twice
                         if reply.status == 200:
                             try:
                                 json_data = await reply.json(content_type=None)
@@ -247,7 +267,7 @@ class DatadisConnector:
                             _LOGGER.warning(
                                 "%s with message '%s'",
                                 reply.status,
-                                text,
+                                await reply.text(),
                             )
                             if not ignore_cache:
                                 await asyncio.to_thread(self._set_cache, url + params)
@@ -256,7 +276,7 @@ class DatadisConnector:
                                 _LOGGER.warning(
                                     "%s with message '%s'. %s. %s",
                                     reply.status,
-                                    text,
+                                    await reply.text(),
                                     "Query temporary disabled",
                                     "Future 500 code errors for this query will be silenced until restart",
                                 )

edata/providers/redata.py

@@ -1,8 +1,10 @@
 """A REData API connector"""
 
 import asyncio
+import contextlib
 import datetime as dt
 import logging
+import typing
 
 import aiohttp
 from dateutil import parser
@@ -26,8 +28,23 @@ class REDataConnector:
 
     def __init__(
         self,
+        session: aiohttp.ClientSession | None = None,
     ) -> None:
-        """Init method for REDataConnector"""
+        """Init method for REDataConnector
+
+        ``session`` lets the caller share a long-lived aiohttp session; without
+        it a short-lived session is opened per request.
+        """
+        self._session = session
+
+    @contextlib.asynccontextmanager
+    async def _client(self) -> typing.AsyncIterator[aiohttp.ClientSession]:
+        """Yield the shared session, or a short-lived one when none was given."""
+        if self._session is not None:
+            yield self._session
+        else:
+            async with aiohttp.ClientSession() as session:
+                yield session
 
     async def async_get_realtime_prices(
         self, dt_from: dt.datetime, dt_to: dt.datetime, is_ceuta_melilla: bool = False
@@ -41,10 +58,9 @@ class REDataConnector:
         data = []
         _LOGGER.info("GET %s", url)
         timeout = aiohttp.ClientTimeout(total=REQUESTS_TIMEOUT)
-        async with aiohttp.ClientSession(timeout=timeout) as session:
+        async with self._client() as session:
             try:
-                async with session.get(url) as res:
-                    text = await res.text()
+                async with session.get(url, timeout=timeout) as res:
                     if res.status == 200:
                         try:
                             res_json = await res.json()
@@ -53,7 +69,7 @@ class REDataConnector:
                             _LOGGER.error(
                                 "%s returned a malformed response: %s ",
                                 url,
-                                text,
+                                await res.text(),
                             )
                             return data
                         for element in res_list:
@@ -70,7 +86,7 @@ class REDataConnector:
                         _LOGGER.error(
                             "%s returned %s with code %s",
                             url,
-                            text,
+                            await res.text(),
                             res.status,
                         )
             except Exception as e:

edata/services/bill_service.py

@@ -44,8 +44,7 @@ class BillService:
     ) -> list[Bill]:
         """Return the list of bills."""
 
-        res = await self.db.list_bill(self._cups, type_, start, end)
-        return [x.data for x in res]
+        return await self.db.list_bill_data(self._cups, type_, start, end)
 
     async def update(
         self,
@@ -234,27 +233,39 @@ class BillService:
             )
             data = await self.get_bills(month, month_end, "hour")
 
+            # aggregate the whole month in a single pass (and a single thread hop)
+            day_bills, month_bills = await asyncio.to_thread(
+                self._compile_day_and_month, data
+            )
+            by_day = {x.datetime: x for x in day_bills}
+
+            done: dict[bool, list[Bill]] = {True: [], False: []}
             for day in ledger.days_in(month):
-                day_end = day + timedelta(days=1) - timedelta(microseconds=1)
-                day_data = [x for x in data if day <= x.datetime <= day_end]
-                stat = await asyncio.to_thread(
-                    self._compile_statistics, day_data, get_day
-                )
-                delta_h = stat[0].delta_h if stat else 0.0
-                final = is_day_final(day, delta_h, now)
-                if stat:
-                    await self.db.add_bill(self._cups, "day", stat[0], "mix", final)
+                bill = by_day.get(day)
+                final = is_day_final(day, bill.delta_h if bill else 0.0, now)
+                if bill:
+                    done[final].append(bill)
                 ledger.resolve_day(day, final=final)
+            for complete, bills in done.items():
+                if bills:
+                    await self.db.add_bill_list(
+                        self._cups, "day", "mix", complete, bills
+                    )
 
-            month_stat = await asyncio.to_thread(
-                self._compile_statistics, data, get_month
-            )
-            delta_h = month_stat[0].delta_h if month_stat else 0.0
+            delta_h = month_bills[0].delta_h if month_bills else 0.0
             final = is_month_final(month, delta_h, now)
-            if month_stat:
-                await self.db.add_bill(self._cups, "month", month_stat[0], "mix", final)
+            if month_bills:
+                await self.db.add_bill(self._cups, "month", month_bills[0], "mix", final)
             ledger.resolve_month(month, final=final)
 
+    def _compile_day_and_month(self, data: list[Bill]) -> tuple[list[Bill], list[Bill]]:
+        """Return the daily and monthly aggregates of a month of hourly bills."""
+
+        return (
+            self._compile_statistics(data, get_day),
+            self._compile_statistics(data, get_month),
+        )
+
     def _compile_statistics(
         self,
         data: list[Bill],
@@ -422,15 +433,13 @@ class BillService:
         self, start: datetime | None = None, end: datetime | None = None
     ) -> list[Energy]:
         """Get energy."""
-        res = await self.db.list_energy(self._cups, start, end)
-        return [x.data for x in res]
+        return await self.db.list_energy_data(self._cups, start, end)
 
     async def _get_pvpc(
         self, start: datetime | None = None, end: datetime | None = None
     ) -> list[EnergyPrice]:
         """Get PVPC."""
-        res = await self.db.list_pvpc(start, end)
-        return [x.data for x in res]
+        return await self.db.list_pvpc_data(start, end)
 
     async def _get_last_bill_dt(self) -> datetime | None:
         """Return the timestamp of the latest bill record."""

edata/services/data_service.py

@@ -8,6 +8,7 @@ from datetime import datetime, timedelta
 from pathlib import Path
 from tempfile import gettempdir
 
+import aiohttp
 from dateutil import relativedelta
 
 from edata.core.completion import PendingLedger, is_day_final, is_month_final
@@ -37,12 +38,13 @@ class DataService:
         datadis_pwd: str,
         storage_path: str,
         datadis_authorized_nif: str | None = None,
+        session: aiohttp.ClientSession | None = None,
     ) -> None:
 
         self.datadis = DatadisConnector(
-            datadis_user, datadis_pwd, storage_path=storage_path
+            datadis_user, datadis_pwd, storage_path=storage_path, session=session
         )
-        self.redata = REDataConnector()
+        self.redata = REDataConnector(session=session)
 
         # params
         self._cups = cups
@@ -86,8 +88,7 @@ class DataService:
     ) -> list[Energy]:
         """Return a list of energy records for the selected cups."""
 
-        res = await self.db.list_energy(self._cups, start, end)
-        return [x.data for x in res]
+        return await self.db.list_energy_data(self._cups, start, end)
 
     async def get_power(
         self, start: datetime | None = None, end: datetime | None = None
@@ -102,8 +103,7 @@ class DataService:
     ) -> list[EnergyPrice]:
         """Return a list of pvpc records (energy prices) for the selected cups."""
 
-        res = await self.db.list_pvpc(start, end)
-        return [x.data for x in res]
+        return await self.db.list_pvpc_data(start, end)
 
     async def get_statistics(
         self,
@@ -164,6 +164,7 @@ class DataService:
             supply.date_end,
         )
 
+        explicit_start = start_date is not None
         if not start_date:
             start_date = supply.date_start
             _LOGGER.debug(
@@ -218,8 +219,12 @@ class DataService:
             # we have no data yet, fetch from start
             await self.update_energy(start_date, end_date)
 
-        # update power records
-        await self.update_power(start_date, end_date)
+        # update power records; unless a start is forced, only refetch from the
+        # month of the latest stored peak instead of the whole supply history
+        power_start = start_date
+        if not explicit_start and (last_power_dt := await self._get_last_power_dt()):
+            power_start = max(start_date, get_month(last_power_dt))
+        await self.update_power(power_start, end_date)
 
         # fetch pvpc data
         await self.update_pvpc(start_date, end_date)
@@ -394,27 +399,41 @@ class DataService:
             )
             data = await self.get_energy(month, month_end)
 
+            # aggregate the whole month in a single pass (and a single thread hop)
+            day_stats, month_stats = await asyncio.to_thread(
+                self._compile_day_and_month, data
+            )
+            by_day = {x.datetime: x for x in day_stats}
+
+            done: dict[bool, list[Statistics]] = {True: [], False: []}
             for day in ledger.days_in(month):
-                day_end = day + timedelta(days=1) - timedelta(microseconds=1)
-                day_data = [x for x in data if day <= x.datetime <= day_end]
-                stat = await asyncio.to_thread(
-                    self._compile_statistics, day_data, get_day
-                )
-                delta_h = stat[0].delta_h if stat else 0.0
-                final = is_day_final(day, delta_h, now)
+                stat = by_day.get(day)
+                final = is_day_final(day, stat.delta_h if stat else 0.0, now)
                 if stat:
-                    await self.db.add_statistics(self._cups, "day", stat[0], final)
+                    done[final].append(stat)
                 ledger.resolve_day(day, final=final)
+            for complete, stats in done.items():
+                if stats:
+                    await self.db.add_statistics_list(
+                        self._cups, "day", complete, stats
+                    )
 
-            month_stat = await asyncio.to_thread(
-                self._compile_statistics, data, get_month
-            )
-            delta_h = month_stat[0].delta_h if month_stat else 0.0
+            delta_h = month_stats[0].delta_h if month_stats else 0.0
             final = is_month_final(month, delta_h, now)
-            if month_stat:
-                await self.db.add_statistics(self._cups, "month", month_stat[0], final)
+            if month_stats:
+                await self.db.add_statistics(self._cups, "month", month_stats[0], final)
             ledger.resolve_month(month, final=final)
 
+    def _compile_day_and_month(
+        self, data: list[Energy]
+    ) -> tuple[list[Statistics], list[Statistics]]:
+        """Return the daily and monthly aggregates of a month of energy data."""
+
+        return (
+            self._compile_statistics(data, get_day),
+            self._compile_statistics(data, get_month),
+        )
+
     def _compile_statistics(
         self,
         data: list[Energy],
@@ -491,6 +510,13 @@ class DataService:
         if last_record:
             return last_record.datetime
 
+    async def _get_last_power_dt(self) -> datetime | None:
+        """Return the timestamp of the latest power record."""
+
+        last_record = await self.db.get_last_power(self._cups)
+        if last_record:
+            return last_record.datetime
+
     async def _get_last_pvpc_dt(self) -> datetime | None:
         """Return the timestamp of the latest pvpc record."""
 

edata/tests/test_controller.py

@@ -0,0 +1,108 @@
+"""Tests for the database controller."""
+
+import asyncio
+from collections.abc import AsyncIterator
+from datetime import datetime, timedelta
+
+import pytest
+import pytest_asyncio
+
+from edata.database.controller import EdataDB
+from edata.models import Bill, Energy, Supply
+
+CUPS = "ESXXXXXXXXXXXXXXXXTEST"
+START = datetime(2024, 1, 1)
+
+
+def _reset_singleton() -> None:
+    EdataDB._instance = None
+    EdataDB._engine = None
+    EdataDB._db_url = None
+
+
+@pytest_asyncio.fixture
+async def db(tmp_path) -> AsyncIterator[EdataDB]:
+    """An EdataDB on an isolated on-disk database with one supply."""
+    _reset_singleton()
+    database = EdataDB(str(tmp_path / "edata.db"))
+    await database.add_supply(
+        Supply(
+            cups=CUPS,
+            date_start=START,
+            date_end=START + timedelta(days=30),
+            address=None,
+            postal_code=None,
+            province=None,
+            municipality=None,
+            distributor=None,
+            point_type=5,
+            distributor_code="2",
+        )
+    )
+    yield database
+    if EdataDB._engine is not None:
+        await EdataDB._engine.dispose()
+    _reset_singleton()
+
+
+def _energy(hours: int, kwh: float) -> list[Energy]:
+    return [
+        Energy(
+            datetime=START + timedelta(hours=h),
+            delta_h=1.0,
+            consumption_kwh=kwh,
+            real=True,
+        )
+        for h in range(hours)
+    ]
+
+
+@pytest.mark.asyncio
+async def test_add_energy_list_upserts(db: EdataDB) -> None:
+    await db.add_energy_list(CUPS, _energy(24, 0.1))
+    before = {x.datetime: x for x in await db.list_energy(CUPS)}
+
+    # overlapping batch: first 12 hours unchanged, last 12 changed, 12 new
+    await db.add_energy_list(CUPS, _energy(12, 0.1) + _energy(36, 0.5)[12:])
+    after = {x.datetime: x for x in await db.list_energy(CUPS)}
+
+    assert len(after) == 36
+    for h in range(36):
+        dt = START + timedelta(hours=h)
+        assert after[dt].data.consumption_kwh == (0.1 if h < 12 else 0.5)
+        if h < 12:
+            # unchanged rows are not rewritten
+            assert after[dt].id == before[dt].id
+            assert after[dt].updated_at == before[dt].updated_at
+        elif h < 24:
+            assert after[dt].id == before[dt].id
+            assert after[dt].updated_at > before[dt].updated_at
+
+
+@pytest.mark.asyncio
+async def test_add_bill_list_applies_overrides(db: EdataDB) -> None:
+    bills = [Bill(datetime=START + timedelta(hours=h), delta_h=1) for h in range(3)]
+    await db.add_bill_list(CUPS, "hour", "hash-a", False, bills)
+
+    # same data, only the override columns change
+    await db.add_bill_list(CUPS, "hour", "hash-b", True, bills)
+
+    stored = await db.list_bill(CUPS, "hour")
+    assert len(stored) == 3
+    assert all(x.complete and x.confhash == "hash-b" for x in stored)
+
+
+@pytest.mark.asyncio
+async def test_concurrent_first_calls_do_not_race_index_creation(db: EdataDB) -> None:
+    # simulate a database created before the index existed, opened fresh
+    async with db.engine.begin() as conn:
+        await conn.exec_driver_sql("DROP INDEX ix_energy_cups_datetime")
+    db._tables_initialized = False
+
+    await asyncio.gather(*(db.get_last_energy(CUPS) for _ in range(10)))
+
+    async with db.engine.connect() as conn:
+        result = await conn.exec_driver_sql(
+            "SELECT name FROM sqlite_master WHERE name='ix_energy_cups_datetime'"
+        )
+        assert result.first() is not None

edata/tests/test_datadis_connector.py

@@ -4,6 +4,8 @@ import datetime
 import os
 from unittest.mock import AsyncMock, MagicMock, patch
 
+import pytest
+
 from edata.providers.datadis import DatadisConnector
 
 MOCK_USERNAME = "USERNAME"
@@ -347,3 +349,26 @@ def test_get_supplies_optional_fields_none(mock_token, mock_get, snapshot):
     mock_get.return_value.__aenter__.return_value = mock_response
     connector = DatadisConnector(MOCK_USERNAME, MOCK_PASSWORD)
     assert connector.get_supplies() == snapshot
+
+
+@pytest.mark.asyncio
+@patch.object(
+    DatadisConnector, "_async_get_token", new_callable=AsyncMock, return_value=True
+)
+async def test_shared_session_is_reused(mock_token, tmp_path, snapshot):
+    """A caller-provided session serves the requests and is left open."""
+    mock_response = MagicMock()
+    mock_response.status = 200
+    mock_response.json = AsyncMock(return_value=SUPPLIES_RESPONSE)
+    session = MagicMock()
+    session.get.return_value.__aenter__.return_value = mock_response
+    session.close = AsyncMock()
+
+    connector = DatadisConnector(
+        MOCK_USERNAME, MOCK_PASSWORD, storage_path=str(tmp_path), session=session
+    )
+    supplies = await connector.async_get_supplies()
+
+    assert supplies == snapshot
+    session.get.assert_called_once()
+    session.close.assert_not_awaited()

edata/tests/test_services.py

@@ -8,7 +8,7 @@ import pytest
 import pytest_asyncio
 from syrupy.assertion import SnapshotAssertion
 
-from edata.core.utils import get_day
+from edata.core.utils import get_day, get_month
 from edata.models.bill import BillingRules
 from edata.models.data import Energy, Power
 from edata.models.supply import Contract, Supply
@@ -231,3 +231,40 @@ async def test_update_pvpc_clamps_range_to_min_date(populated_data_service):
     mock_fetch.assert_awaited_once()
     called_start = mock_fetch.await_args.args[0]
     assert called_start >= get_day(now) - timedelta(days=28)
+
+
+@pytest.mark.asyncio
+async def test_update_power_is_incremental(populated_data_service, power):
+    ds = populated_data_service
+    last_power_dt = max(x.datetime for x in power)
+
+    with (
+        patch.object(ds, "update_energy", AsyncMock(return_value=True)),
+        patch.object(ds, "update_pvpc", AsyncMock(return_value=True)),
+        patch.object(ds, "update_statistics_incremental", AsyncMock()),
+        patch.object(ds, "update_power", AsyncMock(return_value=True)) as mock_power,
+    ):
+        await ds.update()
+        # only refetch from the month of the latest stored peak
+        assert mock_power.await_args.args[0] == get_month(last_power_dt)
+
+        # an explicit start date still forces the full range
+        forced_start = datetime(2020, 1, 1)
+        await ds.update(start_date=forced_start)
+        assert mock_power.await_args.args[0] == forced_start
+
+
+@pytest.mark.asyncio
+async def test_missing_indexes_are_created_on_existing_db(populated_data_service):
+    db = populated_data_service.db
+
+    async with db.engine.begin() as conn:
+        await conn.exec_driver_sql("DROP INDEX IF EXISTS ix_energy_cups_datetime")
+    db._tables_initialized = False
+    await db._ensure_tables()
+
+    async with db.engine.connect() as conn:
+        result = await conn.exec_driver_sql(
+            "SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='energy'"
+        )
+        assert "ix_energy_cups_datetime" in {row[0] for row in result}