e-data 2.0.2.dev136__py3-none-any.whl → 2.0.2.dev138__py3-none-any.whl

edata/database/models.py

@@ -1,6 +1,7 @@
 import typing
 from datetime import datetime as dt
 
+from pydantic import NaiveDatetime
 from sqlmodel import AutoString, Column, Field, Index, SQLModel, UniqueConstraint
 
 from edata.database.utils import PydanticJSON
@@ -27,8 +28,8 @@ class SupplyModel(SQLModel, table=True):
     cups: str = Field(default=None, primary_key=True)
     data: Supply = Field(sa_column=Column(PydanticJSON(Supply)))
     version: int = Field(default=1)
-    created_at: dt = Field(default_factory=_now, nullable=False)
-    updated_at: dt = Field(
+    created_at: NaiveDatetime = Field(default_factory=_now, nullable=False)
+    updated_at: NaiveDatetime = Field(
         default_factory=_now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
     )
 
@@ -44,12 +45,12 @@ class ContractModel(SQLModel, table=True):
 
     id: int | None = Field(default=None, primary_key=True)
     cups: str = Field(foreign_key="supply.cups", index=True)
-    date_start: dt = Field(index=True)
+    date_start: NaiveDatetime = Field(index=True)
     data: Contract = Field(sa_column=Column(PydanticJSON(Contract)))
 
     version: int = Field(default=1)
-    created_at: dt = Field(default_factory=_now, nullable=False)
-    updated_at: dt = Field(
+    created_at: NaiveDatetime = Field(default_factory=_now, nullable=False)
+    updated_at: NaiveDatetime = Field(
         default_factory=_now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
     )
 
@@ -71,13 +72,13 @@ class EnergyModel(SQLModel, table=True):
     id: int | None = Field(default=None, primary_key=True)
     cups: str = Field(foreign_key="supply.cups", index=True)
     delta_h: float
-    datetime: dt = Field(index=True)
+    datetime: NaiveDatetime = Field(index=True)
 
     data: Energy = Field(sa_column=Column(PydanticJSON(Energy)))
 
     version: int = Field(default=1)
-    created_at: dt = Field(default_factory=_now, nullable=False)
-    updated_at: dt = Field(
+    created_at: NaiveDatetime = Field(default_factory=_now, nullable=False)
+    updated_at: NaiveDatetime = Field(
         default_factory=_now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
     )
 
@@ -93,13 +94,13 @@ class PowerModel(SQLModel, table=True):
 
     id: int | None = Field(default=None, primary_key=True)
     cups: str = Field(foreign_key="supply.cups", index=True)
-    datetime: dt = Field(index=True)
+    datetime: NaiveDatetime = Field(index=True)
 
     data: Power = Field(sa_column=Column(PydanticJSON(Power)))
 
     version: int = Field(default=1)
-    created_at: dt = Field(default_factory=_now, nullable=False)
-    updated_at: dt = Field(
+    created_at: NaiveDatetime = Field(default_factory=_now, nullable=False)
+    updated_at: NaiveDatetime = Field(
         default_factory=_now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
     )
 
@@ -120,14 +121,14 @@ class StatisticsModel(SQLModel, table=True):
 
     id: int | None = Field(default=None, primary_key=True)
     cups: str = Field(foreign_key="supply.cups", index=True)
-    datetime: dt = Field(index=True)
+    datetime: NaiveDatetime = Field(index=True)
     type: typing.Literal["day", "month"] = Field(index=True, sa_type=AutoString)
     complete: bool = Field(False)
     data: Statistics = Field(sa_column=Column(PydanticJSON(Statistics)))
 
     version: int = Field(default=1)
-    created_at: dt = Field(default_factory=_now, nullable=False)
-    updated_at: dt = Field(
+    created_at: NaiveDatetime = Field(default_factory=_now, nullable=False)
+    updated_at: NaiveDatetime = Field(
         default_factory=_now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
     )
 
@@ -137,12 +138,12 @@ class PVPCModel(SQLModel, table=True):
     __tablename__ = "pvpc"  # type: ignore
 
     id: int | None = Field(default=None, primary_key=True)
-    datetime: dt = Field(index=True, unique=True)
+    datetime: NaiveDatetime = Field(index=True, unique=True)
     data: EnergyPrice = Field(sa_column=Column(PydanticJSON(EnergyPrice)))
 
     version: int = Field(default=1)
-    created_at: dt = Field(default_factory=_now, nullable=False)
-    updated_at: dt = Field(
+    created_at: NaiveDatetime = Field(default_factory=_now, nullable=False)
+    updated_at: NaiveDatetime = Field(
         default_factory=_now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
     )
 
@@ -163,14 +164,14 @@ class BillModel(SQLModel, table=True):
 
     id: int | None = Field(default=None, primary_key=True)
     cups: str = Field(foreign_key="supply.cups", index=True)
-    datetime: dt = Field(index=True)
+    datetime: NaiveDatetime = Field(index=True)
     type: typing.Literal["hour", "day", "month"] = Field(index=True, sa_type=AutoString)
     complete: bool = Field(False)
     confhash: str
     data: Bill = Field(sa_column=Column(PydanticJSON(Bill)))
 
     version: int = Field(default=1)
-    created_at: dt = Field(default_factory=_now, nullable=False)
-    updated_at: dt = Field(
+    created_at: NaiveDatetime = Field(default_factory=_now, nullable=False)
+    updated_at: NaiveDatetime = Field(
         default_factory=_now, nullable=False, sa_column_kwargs={"onupdate": dt.now}
     )

edata/providers/datadis.py

@@ -3,6 +3,7 @@
 import asyncio
 import contextlib
 import hashlib
+import json
 import logging
 import os
 import tempfile
@@ -69,6 +70,59 @@ def migrate_storage(storage_dir: str) -> None:
         os.remove(os.path.join(storage_dir, "edata_recent_queries_cache.json"))
 
 
+def _parse_consumptions(
+    response: dict[str, typing.Any], start_date: datetime, end_date: datetime
+) -> list[Energy]:
+    """Build the energy records of a 'get_consumption_data' response."""
+
+    consumptions = []
+    for i in response.get("timeCurve", []):
+        if "consumptionKWh" in i:
+            if all(k in i for k in GET_CONSUMPTION_DATA_MANDATORY_FIELDS):
+                raw_hour = int(i["time"].split(":")[0])
+                # Datadis delivers hours 1..24 (end-of-interval). A sporadic
+                # i-DE glitch emits an extra "00:00" row on a day already
+                # carrying its full 24 hours, so drop it -- 01:00..24:00
+                # already covers the day -- instead of remapping onto 23:00
+                # and overlapping that day's 24:00 slot.
+                if raw_hour == 0:
+                    continue
+                date_as_dt = datetime.strptime(
+                    i["date"], "%Y/%m/%d"
+                ) + timedelta(hours=raw_hour - 1)
+                if not (start_date <= date_as_dt <= end_date):
+                    continue  # skip element if dt is out of range
+
+                # sanitize these values
+                _surplus_kwh = i.get("surplusEnergyKWh", 0)
+                if _surplus_kwh is None:
+                    _surplus_kwh = 0
+                _generation_kwh = i.get("generationEnergyKWh", 0)
+                if _generation_kwh is None:
+                    _generation_kwh = 0
+                _selfconsumption_kwh = i.get("selfConsumptionEnergyKWh", 0)
+                if _selfconsumption_kwh is None:
+                    _selfconsumption_kwh = 0
+
+                consumptions.append(
+                    Energy(
+                        datetime=date_as_dt,
+                        delta_h=1,
+                        consumption_kwh=i["consumptionKWh"],
+                        surplus_kwh=_surplus_kwh,
+                        generation_kwh=_generation_kwh,
+                        selfconsumption_kwh=_selfconsumption_kwh,
+                        real=i["obtainMethod"] == "Real",
+                    )
+                )
+            else:
+                _LOGGER.warning(
+                    "Weird data structure while fetching consumption data, got %s",
+                    response,
+                )
+    return consumptions
+
+
 class DatadisConnector:
     """A Datadis private API connector."""
 
@@ -237,7 +291,10 @@ class DatadisConnector:
                         # it here as well would decode every (large) payload twice
                         if reply.status == 200:
                             try:
-                                json_data = await reply.json(content_type=None)
+                                # decode off the event loop: a full-history
+                                # response is several MB of JSON
+                                body = await reply.read()
+                                json_data = await asyncio.to_thread(json.loads, body)
                                 if json_data:
                                     response = json_data
                                     if not ignore_cache:
@@ -419,52 +476,11 @@ class DatadisConnector:
 
         response = await self._async_get(URL_GET_CONSUMPTION_DATA, request_data=data)
 
-        consumptions = []
-        for i in response.get("timeCurve", []):
-            if "consumptionKWh" in i:
-                if all(k in i for k in GET_CONSUMPTION_DATA_MANDATORY_FIELDS):
-                    raw_hour = int(i["time"].split(":")[0])
-                    # Datadis delivers hours 1..24 (end-of-interval). A sporadic
-                    # i-DE glitch emits an extra "00:00" row on a day already
-                    # carrying its full 24 hours, so drop it -- 01:00..24:00
-                    # already covers the day -- instead of remapping onto 23:00
-                    # and overlapping that day's 24:00 slot.
-                    if raw_hour == 0:
-                        continue
-                    date_as_dt = datetime.strptime(
-                        i["date"], "%Y/%m/%d"
-                    ) + timedelta(hours=raw_hour - 1)
-                    if not (start_date <= date_as_dt <= end_date):
-                        continue  # skip element if dt is out of range
-
-                    # sanitize these values
-                    _surplus_kwh = i.get("surplusEnergyKWh", 0)
-                    if _surplus_kwh is None:
-                        _surplus_kwh = 0
-                    _generation_kwh = i.get("generationEnergyKWh", 0)
-                    if _generation_kwh is None:
-                        _generation_kwh = 0
-                    _selfconsumption_kwh = i.get("selfConsumptionEnergyKWh", 0)
-                    if _selfconsumption_kwh is None:
-                        _selfconsumption_kwh = 0
-
-                    consumptions.append(
-                        Energy(
-                            datetime=date_as_dt,
-                            delta_h=1,
-                            consumption_kwh=i["consumptionKWh"],
-                            surplus_kwh=_surplus_kwh,
-                            generation_kwh=_generation_kwh,
-                            selfconsumption_kwh=_selfconsumption_kwh,
-                            real=i["obtainMethod"] == "Real",
-                        )
-                    )
-                else:
-                    _LOGGER.warning(
-                        "Weird data structure while fetching consumption data, got %s",
-                        response,
-                    )
-        return consumptions
+        # building tens of thousands of records on a first import takes long
+        # enough to stall the event loop, so parse in a worker thread
+        return await asyncio.to_thread(
+            _parse_consumptions, response, start_date, end_date
+        )
 
     def get_consumption_data(
         self,

edata/tests/test_controller.py

@@ -1,11 +1,14 @@
 """Tests for the database controller."""
 
 import asyncio
+import threading
 from collections.abc import AsyncIterator
 from datetime import datetime, timedelta
 
 import pytest
 import pytest_asyncio
+from sqlalchemy import DateTime
+from sqlmodel import SQLModel
 
 from edata.database.controller import EdataDB
 from edata.models import Bill, Energy, Supply
@@ -14,16 +17,10 @@ CUPS = "ESXXXXXXXXXXXXXXXXTEST"
 START = datetime(2024, 1, 1)
 
 
-def _reset_singleton() -> None:
-    EdataDB._instance = None
-    EdataDB._engine = None
-    EdataDB._db_url = None
-
-
 @pytest_asyncio.fixture
 async def db(tmp_path) -> AsyncIterator[EdataDB]:
     """An EdataDB on an isolated on-disk database with one supply."""
-    _reset_singleton()
+    EdataDB.reset()
     database = EdataDB(str(tmp_path / "edata.db"))
     await database.add_supply(
         Supply(
@@ -40,9 +37,7 @@ async def db(tmp_path) -> AsyncIterator[EdataDB]:
         )
     )
     yield database
-    if EdataDB._engine is not None:
-        await EdataDB._engine.dispose()
-    _reset_singleton()
+    EdataDB.reset()
 
 
 def _energy(hours: int, kwh: float) -> list[Energy]:
@@ -95,14 +90,38 @@ async def test_add_bill_list_applies_overrides(db: EdataDB) -> None:
 @pytest.mark.asyncio
 async def test_concurrent_first_calls_do_not_race_index_creation(db: EdataDB) -> None:
     # simulate a database created before the index existed, opened fresh
-    async with db.engine.begin() as conn:
-        await conn.exec_driver_sql("DROP INDEX ix_energy_cups_datetime")
+    with db.engine.begin() as conn:
+        conn.exec_driver_sql("DROP INDEX ix_energy_cups_datetime")
     db._tables_initialized = False
 
     await asyncio.gather(*(db.get_last_energy(CUPS) for _ in range(10)))
 
-    async with db.engine.connect() as conn:
-        result = await conn.exec_driver_sql(
+    with db.engine.connect() as conn:
+        result = conn.exec_driver_sql(
             "SELECT name FROM sqlite_master WHERE name='ix_energy_cups_datetime'"
         )
         assert result.first() is not None
+
+
+def test_datetime_columns_are_naive() -> None:
+    # the supply timezone is unknown, so datetimes are stored as-is (naive);
+    # plain ``datetime`` fields map to UTC-aware columns on sqlmodel>=0.0.45
+    columns = [
+        column
+        for table in SQLModel.metadata.sorted_tables
+        for column in table.columns
+        if isinstance(column.type, DateTime)
+    ]
+    assert len(columns) == 20
+    assert all(column.type.timezone is False for column in columns)
+
+
+@pytest.mark.asyncio
+async def test_reset_stops_db_thread_and_allows_new_path(db: EdataDB, tmp_path) -> None:
+    assert await db.list_supplies()
+    EdataDB.reset()
+
+    assert not any(t.name.startswith("edata-db") for t in threading.enumerate())
+    other = EdataDB(str(tmp_path / "other.db"))
+    assert other is not db
+    assert await other.list_supplies() == []

edata/tests/test_datadis_connector.py

@@ -1,6 +1,7 @@
 """Tests for DatadisConnector (offline)."""
 
 import datetime
+import json
 import os
 from unittest.mock import AsyncMock, MagicMock, patch
 
@@ -8,6 +9,12 @@ import pytest
 
 from edata.providers.datadis import DatadisConnector
 
+
+def _json_body(payload) -> AsyncMock:
+    """Mock ``reply.read()`` returning ``payload`` as a JSON body."""
+    return AsyncMock(return_value=json.dumps(payload).encode())
+
+
 MOCK_USERNAME = "USERNAME"
 MOCK_PASSWORD = "PASSWORD"
 
@@ -126,7 +133,7 @@ def test_get_supplies(mock_token, mock_get, snapshot):
     mock_response = MagicMock()
     mock_response.status = 200
     mock_response.text = AsyncMock(return_value="text")
-    mock_response.json = AsyncMock(return_value=SUPPLIES_RESPONSE)
+    mock_response.read = _json_body(SUPPLIES_RESPONSE)
     mock_get.return_value.__aenter__.return_value = mock_response
     connector = DatadisConnector(MOCK_USERNAME, MOCK_PASSWORD)
     assert connector.get_supplies() == snapshot
@@ -141,7 +148,7 @@ def test_get_contract_detail(mock_token, mock_get, snapshot):
     mock_response = MagicMock()
     mock_response.status = 200
     mock_response.text = AsyncMock(return_value="text")
-    mock_response.json = AsyncMock(return_value=CONTRACTS_RESPONSE)
+    mock_response.read = _json_body(CONTRACTS_RESPONSE)
     mock_get.return_value.__aenter__.return_value = mock_response
     connector = DatadisConnector(MOCK_USERNAME, MOCK_PASSWORD)
     assert connector.get_contract_detail("ESXXXXXXXXXXXXXXXXTEST", "2") == snapshot
@@ -156,7 +163,7 @@ def test_get_consumption_data(mock_token, mock_get, snapshot):
     mock_response = MagicMock()
     mock_response.status = 200
     mock_response.text = AsyncMock(return_value="text")
-    mock_response.json = AsyncMock(return_value=CONSUMPTIONS_RESPONSE)
+    mock_response.read = _json_body(CONSUMPTIONS_RESPONSE)
     mock_get.return_value.__aenter__.return_value = mock_response
     connector = DatadisConnector(MOCK_USERNAME, MOCK_PASSWORD)
     assert (
@@ -181,9 +188,7 @@ def test_get_consumption_data_skips_zero_hour(mock_token, mock_get):
     mock_response = MagicMock()
     mock_response.status = 200
     mock_response.text = AsyncMock(return_value="text")
-    mock_response.json = AsyncMock(
-        return_value=CONSUMPTIONS_RESPONSE_WITH_ZERO_HOUR
-    )
+    mock_response.read = _json_body(CONSUMPTIONS_RESPONSE_WITH_ZERO_HOUR)
     mock_get.return_value.__aenter__.return_value = mock_response
     connector = DatadisConnector(MOCK_USERNAME, MOCK_PASSWORD)
 
@@ -215,7 +220,7 @@ def test_get_max_power(mock_token, mock_get, snapshot):
     mock_response = MagicMock()
     mock_response.status = 200
     mock_response.text = AsyncMock(return_value="text")
-    mock_response.json = AsyncMock(return_value=MAXIMETER_RESPONSE)
+    mock_response.read = _json_body(MAXIMETER_RESPONSE)
     mock_get.return_value.__aenter__.return_value = mock_response
     connector = DatadisConnector(MOCK_USERNAME, MOCK_PASSWORD)
     assert (
@@ -239,7 +244,7 @@ def test_get_supplies_empty_response(mock_token, mock_get, snapshot):
     mock_response = MagicMock()
     mock_response.status = 200
     mock_response.text = AsyncMock(return_value="text")
-    mock_response.json = AsyncMock(return_value={"supplies": []})
+    mock_response.read = _json_body({"supplies": []})
     mock_get.return_value.__aenter__.return_value = mock_response
     connector = DatadisConnector(MOCK_USERNAME, MOCK_PASSWORD)
     assert connector.get_supplies() == snapshot
@@ -255,7 +260,7 @@ def test_get_supplies_malformed_response(mock_token, mock_get, snapshot):
     mock_response = MagicMock()
     mock_response.status = 200
     mock_response.text = AsyncMock(return_value="text")
-    mock_response.json = AsyncMock(return_value=malformed)
+    mock_response.read = _json_body(malformed)
     mock_get.return_value.__aenter__.return_value = mock_response
     connector = DatadisConnector(MOCK_USERNAME, MOCK_PASSWORD)
     assert connector.get_supplies() == snapshot
@@ -274,7 +279,7 @@ def test_get_supplies_partial_response(mock_token, mock_get, snapshot):
     mock_response = MagicMock()
     mock_response.status = 200
     mock_response.text = AsyncMock(return_value="text")
-    mock_response.json = AsyncMock(return_value=partial)
+    mock_response.read = _json_body(partial)
     mock_get.return_value.__aenter__.return_value = mock_response
     connector = DatadisConnector(MOCK_USERNAME, MOCK_PASSWORD)
     assert connector.get_supplies() == snapshot
@@ -289,7 +294,7 @@ def test_get_consumption_data_cache(mock_token, mock_get, snapshot):
     mock_response = MagicMock()
     mock_response.status = 200
     mock_response.text = AsyncMock(return_value="text")
-    mock_response.json = AsyncMock(return_value=CONSUMPTIONS_RESPONSE)
+    mock_response.read = _json_body(CONSUMPTIONS_RESPONSE)
     mock_get.return_value.__aenter__.return_value = mock_response
     connector = DatadisConnector(MOCK_USERNAME, MOCK_PASSWORD)
     # First call populates cache
@@ -345,7 +350,7 @@ def test_get_supplies_optional_fields_none(mock_token, mock_get, snapshot):
     mock_response = MagicMock()
     mock_response.status = 200
     mock_response.text = AsyncMock(return_value="text")
-    mock_response.json = AsyncMock(return_value=response)
+    mock_response.read = _json_body(response)
     mock_get.return_value.__aenter__.return_value = mock_response
     connector = DatadisConnector(MOCK_USERNAME, MOCK_PASSWORD)
     assert connector.get_supplies() == snapshot
@@ -359,7 +364,7 @@ async def test_shared_session_is_reused(mock_token, tmp_path, snapshot):
     """A caller-provided session serves the requests and is left open."""
     mock_response = MagicMock()
     mock_response.status = 200
-    mock_response.json = AsyncMock(return_value=SUPPLIES_RESPONSE)
+    mock_response.read = _json_body(SUPPLIES_RESPONSE)
     session = MagicMock()
     session.get.return_value.__aenter__.return_value = mock_response
     session.close = AsyncMock()

edata/tests/test_incremental.py

@@ -15,25 +15,17 @@ from edata.services.data_service import DataService
 CUPS = "ESXXXXXXXXXXXXXXXXTEST"
 
 
-def _reset_singleton() -> None:
-    EdataDB._instance = None
-    EdataDB._engine = None
-    EdataDB._db_url = None
-
-
 @pytest_asyncio.fixture
 async def data_service(tmp_path) -> AsyncIterator[DataService]:
     """A DataService on an isolated on-disk database, singleton reset around it."""
-    _reset_singleton()
+    EdataDB.reset()
     with (
         patch("edata.services.data_service.DatadisConnector"),
         patch("edata.services.data_service.REDataConnector"),
     ):
         service = DataService(CUPS, "user", "pwd", storage_path=str(tmp_path))
     yield service
-    if EdataDB._engine is not None:
-        await EdataDB._engine.dispose()
-    _reset_singleton()
+    EdataDB.reset()
 
 
 def _hourly_energy(start: datetime, end: datetime) -> list[Energy]:

edata/tests/test_migrations.py

@@ -18,12 +18,6 @@ ASSETS = os.path.join(os.path.dirname(__file__), "assets")
 CUPS = "ESXXXXXXXXXXXXXXXXTEST"
 
 
-def _reset_singleton() -> None:
-    EdataDB._instance = None
-    EdataDB._engine = None
-    EdataDB._db_url = None
-
-
 def _raw() -> dict:
     with open(os.path.join(ASSETS, "legacy_1.3.3.json"), encoding="utf-8") as f:
         return json.load(f)
@@ -40,16 +34,14 @@ def _install_legacy_file(storage_dir: str, cups: str = CUPS) -> str:
 @pytest_asyncio.fixture
 async def data_service(tmp_path) -> AsyncIterator[DataService]:
     """A DataService on an isolated on-disk database, singleton reset around it."""
-    _reset_singleton()
+    EdataDB.reset()
     with (
         patch("edata.services.data_service.DatadisConnector"),
         patch("edata.services.data_service.REDataConnector"),
     ):
         service = DataService(CUPS, "user", "pwd", storage_path=str(tmp_path))
     yield service
-    if EdataDB._engine is not None:
-        await EdataDB._engine.dispose()
-    _reset_singleton()
+    EdataDB.reset()
 
 
 # --- pure mappers ---

edata/tests/test_services.py

@@ -258,13 +258,13 @@ async def test_update_power_is_incremental(populated_data_service, power):
 async def test_missing_indexes_are_created_on_existing_db(populated_data_service):
     db = populated_data_service.db
 
-    async with db.engine.begin() as conn:
-        await conn.exec_driver_sql("DROP INDEX IF EXISTS ix_energy_cups_datetime")
+    with db.engine.begin() as conn:
+        conn.exec_driver_sql("DROP INDEX IF EXISTS ix_energy_cups_datetime")
     db._tables_initialized = False
-    await db._ensure_tables()
+    await db.list_supplies()  # any call lazily re-runs the table/index setup
 
-    async with db.engine.connect() as conn:
-        result = await conn.exec_driver_sql(
+    with db.engine.connect() as conn:
+        result = conn.exec_driver_sql(
             "SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='energy'"
         )
         assert "ix_energy_cups_datetime" in {row[0] for row in result}