e-data 2.0.2.dev138__tar.gz → 2.0.2.dev140__tar.gz

{e_data-2.0.2.dev138/e_data.egg-info → e_data-2.0.2.dev140}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: e-data
-Version: 2.0.2.dev138
+Version: 2.0.2.dev140
 Summary: Python library for managing spanish energy data from various web providers
 Author-email: VMG <vmayorg@outlook.es>
 License:                     GNU GENERAL PUBLIC LICENSE

{e_data-2.0.2.dev138 → e_data-2.0.2.dev140/e_data.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: e-data
-Version: 2.0.2.dev138
+Version: 2.0.2.dev140
 Summary: Python library for managing spanish energy data from various web providers
 Author-email: VMG <vmayorg@outlook.es>
 License:                     GNU GENERAL PUBLIC LICENSE

{e_data-2.0.2.dev138 → e_data-2.0.2.dev140}/edata/database/migrations/legacy_json_1_3_3.py

@@ -10,6 +10,8 @@ Self-contained on purpose: everything it needs lives here, so a future version
 can drop this file (and its entry in the registry) once the upgrade window closes.
 """
 
+import asyncio
+from dataclasses import dataclass
 from datetime import datetime
 import json
 import logging
@@ -83,44 +85,68 @@ def _to_pvpc(d: dict) -> EnergyPrice:
     )
 
 
+@dataclass
+class _LegacyRecords:
+    """Records parsed from a legacy JSON export."""
+
+    supplies: list[Supply]
+    contracts: list[Contract]
+    energy: list[Energy]
+    power: list[Power]
+    pvpc: list[EnergyPrice]
+
+
+def _load(path: str) -> _LegacyRecords | None:
+    """Read and parse the legacy file; ``None`` if it does not exist.
+
+    Blocking (file I/O plus building tens of thousands of records), so callers
+    run it in a worker thread to keep the event loop free.
+    """
+
+    if not os.path.isfile(path):
+        return None
+    with open(path, encoding="utf-8") as f:
+        raw = json.load(f)
+    return _LegacyRecords(
+        supplies=[_to_supply(x) for x in raw.get("supplies", [])],
+        contracts=[_to_contract(x) for x in raw.get("contracts", [])],
+        energy=[_to_energy(x) for x in raw.get("consumptions", [])],
+        power=[_to_power(x) for x in raw.get("maximeter", [])],
+        pvpc=[_to_pvpc(x) for x in raw.get("pvpc", [])],
+    )
+
+
 async def apply(db: EdataDB, storage_dir: str, cups: str) -> MigrationResult | None:
     """Import a 1.3.3 JSON export. Returns None (no-op) if the file is absent."""
 
     path = _legacy_path(storage_dir, cups)
-    if not os.path.isfile(path):
+    records = await asyncio.to_thread(_load, path)
+    if records is None:
         return None
 
     _LOGGER.info("Migrating legacy 1.3.3 storage from %s", path)
-    with open(path, encoding="utf-8") as f:
-        raw = json.load(f)
-
     result = MigrationResult(name=NAME)
 
-    supplies = [_to_supply(x) for x in raw.get("supplies", [])]
-    for supply in supplies:
+    for supply in records.supplies:
         await db.add_supply(supply)
-    result.supplies = len(supplies)
+    result.supplies = len(records.supplies)
 
     # Contracts/energy/power reference a supply by cups; take it from the imported
     # supply record rather than the (lower-cased) filename id.
-    target_cups = supplies[0].cups if supplies else cups
+    target_cups = records.supplies[0].cups if records.supplies else cups
 
-    contracts = [_to_contract(x) for x in raw.get("contracts", [])]
-    for contract in contracts:
+    for contract in records.contracts:
         await db.add_contract(target_cups, contract)
-    result.contracts = len(contracts)
+    result.contracts = len(records.contracts)
 
-    energy = [_to_energy(x) for x in raw.get("consumptions", [])]
-    await db.add_energy_list(target_cups, energy)
-    result.energy = len(energy)
+    await db.add_energy_list(target_cups, records.energy)
+    result.energy = len(records.energy)
 
-    power = [_to_power(x) for x in raw.get("maximeter", [])]
-    await db.add_power_list(target_cups, power)
-    result.power = len(power)
+    await db.add_power_list(target_cups, records.power)
+    result.power = len(records.power)
 
-    pvpc = [_to_pvpc(x) for x in raw.get("pvpc", [])]
-    await db.add_pvpc_list(pvpc)
-    result.pvpc = len(pvpc)
+    await db.add_pvpc_list(records.pvpc)
+    result.pvpc = len(records.pvpc)
 
     _LOGGER.info(
         "%s: imported %s supplies, %s contracts, %s energy, %s power, %s pvpc",

{e_data-2.0.2.dev138 → e_data-2.0.2.dev140}/edata/providers/datadis.py

@@ -12,6 +12,7 @@ from datetime import datetime, timedelta
 
 import aiohttp
 import diskcache
+from dateutil.relativedelta import relativedelta
 
 from edata.models import Contract, Energy, Power, Supply
 
@@ -57,6 +58,9 @@ GET_MAX_POWER_MANDATORY_FIELDS = ["time", "date", "maxPower"]
 # Timing constants
 TIMEOUT = 3 * 60  # requests timeout
 QUERY_LIMIT = timedelta(hours=24)  # a datadis limitation, again...
+# Datadis rejects (400) any query starting more than two years ago, which made
+# a first sync of an older supply fail as a whole
+MAX_HISTORY = relativedelta(years=2)
 
 
 # Cache-related constants
@@ -70,6 +74,18 @@ def migrate_storage(storage_dir: str) -> None:
         os.remove(os.path.join(storage_dir, "edata_recent_queries_cache.json"))
 
 
+def earliest_query_start(now: datetime | None = None) -> datetime:
+    """Return the oldest start date Datadis accepts.
+
+    Queries are monthly (``YYYY/MM``), so this is the first month whose first
+    day is still within ``MAX_HISTORY`` of ``now``.
+    """
+
+    limit = (now or datetime.now()) - MAX_HISTORY
+    month = limit.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
+    return month if month >= limit else month + relativedelta(months=1)
+
+
 def _parse_consumptions(
     response: dict[str, typing.Any], start_date: datetime, end_date: datetime
 ) -> list[Energy]:
@@ -463,6 +479,10 @@ class DatadisConnector:
         authorized_nif: str | None = None,
     ) -> list[Energy]:
         """Datadis 'get_consumption_data' query."""
+        start_date = max(start_date, earliest_query_start())
+        if start_date > end_date:
+            _LOGGER.debug("Skipping consumption query older than Datadis history")
+            return []
         data = {
             "cups": cups,
             "distributorCode": distributor_code,
@@ -514,6 +534,10 @@ class DatadisConnector:
         authorized_nif: str | None = None,
     ) -> list[Power]:
         """Datadis 'get_max_power' query."""
+        start_date = max(start_date, earliest_query_start())
+        if start_date > end_date:
+            _LOGGER.debug("Skipping max power query older than Datadis history")
+            return []
         data = {
             "cups": cups,
             "distributorCode": distributor_code,

{e_data-2.0.2.dev138 → e_data-2.0.2.dev140}/edata/tests/test_datadis_connector.py

@@ -7,7 +7,17 @@ from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
-from edata.providers.datadis import DatadisConnector
+from edata.providers.datadis import DatadisConnector, earliest_query_start
+
+
+@pytest.fixture(autouse=True)
+def _fixture_data_within_history():
+    """Keep the 2022 fixture data inside Datadis's two-year query window."""
+    with patch(
+        "edata.providers.datadis.earliest_query_start",
+        return_value=datetime.datetime(2020, 1, 1),
+    ):
+        yield
 
 
 def _json_body(payload) -> AsyncMock:
@@ -377,3 +387,93 @@ async def test_shared_session_is_reused(mock_token, tmp_path, snapshot):
     assert supplies == snapshot
     session.get.assert_called_once()
     session.close.assert_not_awaited()
+
+
+@pytest.mark.parametrize(
+    ("now", "expected"),
+    [
+        # 2024-10-01 is already more than two years before the 9th
+        (datetime.datetime(2026, 10, 9, 18, 0), datetime.datetime(2024, 11, 1)),
+        (datetime.datetime(2026, 10, 1, 0, 0), datetime.datetime(2024, 10, 1)),
+        (datetime.datetime(2026, 3, 1, 0, 0, 1), datetime.datetime(2024, 4, 1)),
+    ],
+)
+def test_earliest_query_start(now, expected):
+    """The oldest accepted start is the first month fully within two years."""
+    assert earliest_query_start(now) == expected
+
+
+@patch("aiohttp.ClientSession.get")
+@patch.object(
+    DatadisConnector, "_async_get_token", new_callable=AsyncMock, return_value=True
+)
+def test_queries_are_clamped_to_datadis_history(mock_token, mock_get, tmp_path):
+    """Older start dates are moved to the oldest month Datadis accepts."""
+    mock_response = MagicMock()
+    mock_response.status = 200
+    mock_response.read = _json_body({"timeCurve": [], "maxPower": []})
+    mock_get.return_value.__aenter__.return_value = mock_response
+    connector = DatadisConnector(
+        MOCK_USERNAME, MOCK_PASSWORD, storage_path=str(tmp_path)
+    )
+
+    with patch(
+        "edata.providers.datadis.earliest_query_start",
+        return_value=datetime.datetime(2022, 10, 1),
+    ):
+        connector.get_consumption_data(
+            "ESXXXXXXXXXXXXXXXXTEST",
+            "2",
+            datetime.datetime(2019, 1, 1),
+            datetime.datetime(2022, 10, 31),
+            "0",
+            5,
+        )
+        connector.get_max_power(
+            "ESXXXXXXXXXXXXXXXXTEST",
+            "2",
+            datetime.datetime(2019, 1, 1),
+            datetime.datetime(2022, 10, 31),
+        )
+
+    urls = [call.args[0] for call in mock_get.call_args_list]
+    assert len(urls) == 2
+    assert all("startDate=2022/10&" in url for url in urls)
+
+
+@patch("aiohttp.ClientSession.get")
+@patch.object(
+    DatadisConnector, "_async_get_token", new_callable=AsyncMock, return_value=True
+)
+def test_queries_entirely_before_history_are_skipped(mock_token, mock_get, tmp_path):
+    """A range Datadis no longer serves is not requested at all."""
+    connector = DatadisConnector(
+        MOCK_USERNAME, MOCK_PASSWORD, storage_path=str(tmp_path)
+    )
+
+    with patch(
+        "edata.providers.datadis.earliest_query_start",
+        return_value=datetime.datetime(2022, 10, 1),
+    ):
+        assert (
+            connector.get_consumption_data(
+                "ESXXXXXXXXXXXXXXXXTEST",
+                "2",
+                datetime.datetime(2019, 1, 1),
+                datetime.datetime(2019, 12, 31),
+                "0",
+                5,
+            )
+            == []
+        )
+        assert (
+            connector.get_max_power(
+                "ESXXXXXXXXXXXXXXXXTEST",
+                "2",
+                datetime.datetime(2019, 1, 1),
+                datetime.datetime(2019, 12, 31),
+            )
+            == []
+        )
+
+    mock_get.assert_not_called()

{e_data-2.0.2.dev138 → e_data-2.0.2.dev140}/edata/tests/test_migrations.py

@@ -5,6 +5,7 @@ from datetime import datetime
 import json
 import os
 import shutil
+import threading
 from unittest.mock import patch
 
 import pytest
@@ -175,3 +176,26 @@ async def test_run_migrations_no_legacy_file(data_service: DataService) -> None:
     """run_migrations is a no-op when there is no legacy file."""
     results = await data_service.run_migrations()
     assert results == []
+
+
+@pytest.mark.asyncio
+async def test_apply_reads_legacy_file_off_the_event_loop(
+    data_service: DataService, tmp_path
+) -> None:
+    """The file is read and parsed in a worker thread, not on the event loop.
+
+    Home Assistant flags blocking ``open`` calls made on its event loop.
+    """
+    _install_legacy_file(str(tmp_path))
+    loop_thread = threading.current_thread()
+    load_threads = []
+    real_load = json.load
+
+    def recording_load(*args, **kwargs):
+        load_threads.append(threading.current_thread())
+        return real_load(*args, **kwargs)
+
+    with patch.object(legacy.json, "load", recording_load):
+        assert await legacy.apply(data_service.db, str(tmp_path), CUPS) is not None
+
+    assert load_threads and all(t is not loop_thread for t in load_threads)

{e_data-2.0.2.dev138 → e_data-2.0.2.dev140}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "e-data"
-version = "2.0.2.dev138"
+version = "2.0.2.dev140"
 description = "Python library for managing spanish energy data from various web providers"
 readme = "README.md"
 authors = [