e-data 2.0.1.dev131__tar.gz → 2.0.1.dev133__tar.gz

{e_data-2.0.1.dev131/e_data.egg-info → e_data-2.0.1.dev133}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: e-data
-Version: 2.0.1.dev131
+Version: 2.0.1.dev133
 Summary: Python library for managing spanish energy data from various web providers
 Author-email: VMG <vmayorg@outlook.es>
 License:                     GNU GENERAL PUBLIC LICENSE

{e_data-2.0.1.dev131 → e_data-2.0.1.dev133/e_data.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: e-data
-Version: 2.0.1.dev131
+Version: 2.0.1.dev133
 Summary: Python library for managing spanish energy data from various web providers
 Author-email: VMG <vmayorg@outlook.es>
 License:                     GNU GENERAL PUBLIC LICENSE

{e_data-2.0.1.dev131 → e_data-2.0.1.dev133}/e_data.egg-info/SOURCES.txt

@@ -18,6 +18,9 @@ edata/database/controller.py
 edata/database/models.py
 edata/database/queries.py
 edata/database/utils.py
+edata/database/migrations/__init__.py
+edata/database/migrations/base.py
+edata/database/migrations/legacy_json_1_3_3.py
 edata/models/__init__.py
 edata/models/bill.py
 edata/models/data.py
@@ -31,6 +34,7 @@ edata/tests/__init__.py
 edata/tests/test_completion.py
 edata/tests/test_datadis_connector.py
 edata/tests/test_incremental.py
+edata/tests/test_migrations.py
 edata/tests/test_redata_connector.py
 edata/tests/test_services.py
 edata/tests/test_utils.py

{e_data-2.0.1.dev131 → e_data-2.0.1.dev133}/edata/cli.py

@@ -141,5 +141,38 @@ def update_pvpc_bill(
     )
 
 
+async def _migrate_legacy(
+    cups: str, storage_path: str, compile_statistics: bool
+) -> None:
+    """Import a 1.3.3 JSON export into the 2.0 store (no datadis account needed)."""
+
+    service = DataService(cups, "", "", storage_path=storage_path)
+    results = await service.run_migrations(compile_statistics=compile_statistics)
+    if not results:
+        typer.echo("No legacy storage found to migrate.")
+        return
+    for result in results:
+        typer.echo(
+            f"{result.name}: {result.supplies} supplies, {result.contracts} contracts, "
+            f"{result.energy} energy, {result.power} power, {result.pvpc} pvpc"
+        )
+
+
+@app.command()
+def migrate_legacy(
+    cups: Annotated[str, typer.Option(help="The identifier of the Supply")],
+    storage_path: Annotated[
+        str,
+        typer.Option(help="Root dir holding the legacy edata/ folder and edata.db"),
+    ] = "./edata_cli",
+    compile_statistics: Annotated[
+        bool, typer.Option(help="Compile day/month statistics after import")
+    ] = True,
+) -> None:
+    """Import a python-edata 1.3.3 JSON export into the 2.0 SQLite store."""
+
+    asyncio.run(_migrate_legacy(cups, storage_path, compile_statistics))
+
+
 if __name__ == "__main__":
     app()

e_data-2.0.1.dev133/edata/database/migrations/__init__.py

@@ -0,0 +1,36 @@
+"""Versioned data migrations for the edata SQLite store.
+
+Each migration is a self-contained module exposing ``NAME`` and an async
+``apply(db, storage_dir, cups)`` that returns a :class:`MigrationResult` (or
+``None`` when it does not apply). ``run_migrations`` runs the ordered registry.
+
+To retire a migration in a future version, delete its module and drop it from
+``MIGRATIONS``.
+"""
+
+import logging
+
+from edata.database.controller import EdataDB
+from edata.database.migrations import legacy_json_1_3_3
+from edata.database.migrations.base import MigrationResult
+
+_LOGGER = logging.getLogger(__name__)
+
+# Ordered registry of migrations to run.
+MIGRATIONS = [legacy_json_1_3_3]
+
+__all__ = ["MIGRATIONS", "MigrationResult", "run_migrations"]
+
+
+async def run_migrations(
+    db: EdataDB, storage_dir: str, cups: str
+) -> list[MigrationResult]:
+    """Run every registered migration; return the results that applied."""
+
+    results: list[MigrationResult] = []
+    for migration in MIGRATIONS:
+        result = await migration.apply(db, storage_dir, cups)
+        if result is not None:
+            _LOGGER.info("Applied migration '%s'", result.name)
+            results.append(result)
+    return results

e_data-2.0.1.dev133/edata/database/migrations/base.py

@@ -0,0 +1,19 @@
+"""Shared types for the migrations package.
+
+Kept separate from ``__init__`` and from the individual migration modules so a
+migration file can be deleted in a future version without breaking imports.
+"""
+
+from dataclasses import dataclass
+
+
+@dataclass
+class MigrationResult:
+    """Summary of what a single migration wrote."""
+
+    name: str
+    supplies: int = 0
+    contracts: int = 0
+    energy: int = 0
+    power: int = 0
+    pvpc: int = 0

e_data-2.0.1.dev133/edata/database/migrations/legacy_json_1_3_3.py

@@ -0,0 +1,134 @@
+"""Migration: import a python-edata 1.3.3 JSON storage export into the 2.0 DB.
+
+1.x persisted everything as a single JSON file per supply at
+``{storage_dir}/edata/edata_{cups}.json`` (ISO datetimes). 2.0 uses a SQLite
+store. This migration imports the raw records only (supplies, contracts,
+consumptions, maximeter, pvpc); the old ``*_sum`` aggregates are ignored because
+2.0 recomputes daily/monthly statistics deterministically.
+
+Self-contained on purpose: everything it needs lives here, so a future version
+can drop this file (and its entry in the registry) once the upgrade window closes.
+"""
+
+from datetime import datetime
+import json
+import logging
+import os
+
+from edata.database.controller import EdataDB
+from edata.database.migrations.base import MigrationResult
+from edata.models import Contract, Energy, Power, Supply
+from edata.models.bill import EnergyPrice
+
+_LOGGER = logging.getLogger(__name__)
+
+NAME = "legacy_json_1_3_3"
+LEGACY_SUBDIR = "edata"
+
+
+def _legacy_path(storage_dir: str, cups: str) -> str:
+    """Return the 1.3.3 storage file path for a supply."""
+
+    return os.path.join(storage_dir, LEGACY_SUBDIR, f"edata_{cups.lower()}.json")
+
+
+def _to_supply(d: dict) -> Supply:
+    return Supply(
+        cups=d["cups"],
+        date_start=datetime.fromisoformat(d["date_start"]),
+        date_end=datetime.fromisoformat(d["date_end"]),
+        address=d.get("address"),
+        postal_code=d.get("postal_code"),
+        province=d.get("province"),
+        municipality=d.get("municipality"),
+        distributor=d.get("distributor"),
+        point_type=d["pointType"],
+        distributor_code=d["distributorCode"],
+    )
+
+
+def _to_contract(d: dict) -> Contract:
+    power = [p for p in (d.get("power_p1"), d.get("power_p2")) if p is not None]
+    return Contract(
+        date_start=datetime.fromisoformat(d["date_start"]),
+        date_end=datetime.fromisoformat(d["date_end"]),
+        marketer=d["marketer"],
+        distributor_code=d["distributorCode"],
+        power=power,
+    )
+
+
+def _to_energy(d: dict) -> Energy:
+    return Energy(
+        datetime=datetime.fromisoformat(d["datetime"]),
+        delta_h=d["delta_h"],
+        consumption_kwh=d["value_kWh"],
+        surplus_kwh=d.get("surplus_kWh", 0),
+        real=d["real"],
+    )
+
+
+def _to_power(d: dict) -> Power:
+    return Power(
+        datetime=datetime.fromisoformat(d["datetime"]),
+        value_kw=d["value_kW"],
+    )
+
+
+def _to_pvpc(d: dict) -> EnergyPrice:
+    return EnergyPrice(
+        datetime=datetime.fromisoformat(d["datetime"]),
+        value_eur_kwh=d["value_eur_kWh"],
+        delta_h=d["delta_h"],
+    )
+
+
+async def apply(db: EdataDB, storage_dir: str, cups: str) -> MigrationResult | None:
+    """Import a 1.3.3 JSON export. Returns None (no-op) if the file is absent."""
+
+    path = _legacy_path(storage_dir, cups)
+    if not os.path.isfile(path):
+        return None
+
+    _LOGGER.info("Migrating legacy 1.3.3 storage from %s", path)
+    with open(path, encoding="utf-8") as f:
+        raw = json.load(f)
+
+    result = MigrationResult(name=NAME)
+
+    supplies = [_to_supply(x) for x in raw.get("supplies", [])]
+    for supply in supplies:
+        await db.add_supply(supply)
+    result.supplies = len(supplies)
+
+    # Contracts/energy/power reference a supply by cups; take it from the imported
+    # supply record rather than the (lower-cased) filename id.
+    target_cups = supplies[0].cups if supplies else cups
+
+    contracts = [_to_contract(x) for x in raw.get("contracts", [])]
+    for contract in contracts:
+        await db.add_contract(target_cups, contract)
+    result.contracts = len(contracts)
+
+    energy = [_to_energy(x) for x in raw.get("consumptions", [])]
+    await db.add_energy_list(target_cups, energy)
+    result.energy = len(energy)
+
+    power = [_to_power(x) for x in raw.get("maximeter", [])]
+    await db.add_power_list(target_cups, power)
+    result.power = len(power)
+
+    pvpc = [_to_pvpc(x) for x in raw.get("pvpc", [])]
+    await db.add_pvpc_list(pvpc)
+    result.pvpc = len(pvpc)
+
+    _LOGGER.info(
+        "%s: imported %s supplies, %s contracts, %s energy, %s power, %s pvpc",
+        NAME,
+        result.supplies,
+        result.contracts,
+        result.energy,
+        result.power,
+        result.pvpc,
+    )
+    return result

{e_data-2.0.1.dev131 → e_data-2.0.1.dev133}/edata/providers/datadis.py

@@ -59,7 +59,7 @@ QUERY_LIMIT = timedelta(hours=24)  # a datadis limitation, again...
 
 
 # Cache-related constants
-RECENT_CACHE_SUBDIR = "cache"
+RECENT_CACHE_SUBDIR = "edata_cache"
 
 
 def migrate_storage(storage_dir: str) -> None:
@@ -403,10 +403,17 @@ class DatadisConnector:
         for i in response.get("timeCurve", []):
             if "consumptionKWh" in i:
                 if all(k in i for k in GET_CONSUMPTION_DATA_MANDATORY_FIELDS):
-                    hour = str(int(i["time"].split(":")[0]) - 1)
+                    raw_hour = int(i["time"].split(":")[0])
+                    # Datadis delivers hours 1..24 (end-of-interval). A sporadic
+                    # i-DE glitch emits an extra "00:00" row on a day already
+                    # carrying its full 24 hours, so drop it -- 01:00..24:00
+                    # already covers the day -- instead of remapping onto 23:00
+                    # and overlapping that day's 24:00 slot.
+                    if raw_hour == 0:
+                        continue
                     date_as_dt = datetime.strptime(
-                        f"{i['date']} {hour.zfill(2)}:00", "%Y/%m/%d %H:%M"
-                    )
+                        i["date"], "%Y/%m/%d"
+                    ) + timedelta(hours=raw_hour - 1)
                     if not (start_date <= date_as_dt <= end_date):
                         continue  # skip element if dt is out of range
 

{e_data-2.0.1.dev131 → e_data-2.0.1.dev133}/edata/services/data_service.py

@@ -19,6 +19,7 @@ from edata.core.utils import (
     redacted_cups,
 )
 from edata.database.controller import EdataDB
+from edata.database.migrations import MigrationResult, run_migrations
 from edata.models import Contract, Energy, Power, Statistics, Supply
 from edata.models.bill import EnergyPrice
 from edata.providers import DatadisConnector, REDataConnector
@@ -55,6 +56,7 @@ class DataService:
             )
             self._authorized_nif = None
 
+        self._storage_path = storage_path
         self.db = EdataDB(get_db_path(storage_path))
 
         # data (in-memory cache)
@@ -317,6 +319,27 @@ class DataService:
         _LOGGER.warning("%s unable to fetch pvpc prices", self._scups)
         return False
 
+    async def run_migrations(
+        self, compile_statistics: bool = True
+    ) -> list[MigrationResult]:
+        """Run pending data migrations (e.g. import a legacy 1.3.3 JSON export).
+
+        Migrations import raw records only; when anything is imported and
+        ``compile_statistics`` is set, the day/month statistics are compiled from
+        the freshly-imported energy so the history is queryable right away. Bills
+        are left to the normal billing flow (they need user rules).
+        """
+
+        results = await run_migrations(self.db, self._storage_path, self._cups)
+        if not results or not compile_statistics:
+            return results
+
+        supply = await self.get_supply()
+        last_energy_dt = await self._get_last_energy_dt()
+        if supply is not None and last_energy_dt is not None:
+            await self.update_statistics(supply.date_start, last_energy_dt)
+        return results
+
     async def update_statistics(self, start: datetime, end: datetime) -> None:
         """Compile the statistics for a range plus any incomplete backlog."""
 

{e_data-2.0.1.dev131 → e_data-2.0.1.dev133}/edata/tests/test_datadis_connector.py

@@ -1,6 +1,7 @@
 """Tests for DatadisConnector (offline)."""
 
 import datetime
+import os
 from unittest.mock import AsyncMock, MagicMock, patch
 
 from edata.providers.datadis import DatadisConnector
@@ -8,6 +9,15 @@ from edata.providers.datadis import DatadisConnector
 MOCK_USERNAME = "USERNAME"
 MOCK_PASSWORD = "PASSWORD"
 
+
+def test_connector_uses_namespaced_cache_dir(tmp_path) -> None:
+    """A storage_path connector creates the namespaced 'edata_cache' dir."""
+    connector = DatadisConnector(
+        MOCK_USERNAME, MOCK_PASSWORD, storage_path=str(tmp_path)
+    )
+    assert os.path.basename(connector._recent_cache_dir) == "edata_cache"
+    assert (tmp_path / "edata_cache").is_dir()
+
 SUPPLIES_RESPONSE = {
     "supplies": [
         {
@@ -56,6 +66,34 @@ CONSUMPTIONS_RESPONSE = {
     ]
 }
 
+CONSUMPTIONS_RESPONSE_WITH_ZERO_HOUR = {
+    "timeCurve": [
+        {
+            "date": "2022/10/22",
+            "time": "01:00",
+            "consumptionKWh": 0.203,
+            "surplusEnergyKWh": 0,
+            "obtainMethod": "Real",
+        },
+        {
+            "date": "2022/10/22",
+            "time": "02:00",
+            "consumptionKWh": 0.163,
+            "surplusEnergyKWh": 0,
+            "obtainMethod": "Real",
+        },
+        # Sporadic i-DE glitch: an extra "00:00" row on a day that already
+        # carries its full 24 hours. Must be dropped, not remapped to 23:00.
+        {
+            "date": "2022/10/22",
+            "time": "00:00",
+            "consumptionKWh": 0.999,
+            "surplusEnergyKWh": 0,
+            "obtainMethod": "Real",
+        },
+    ]
+}
+
 MAXIMETER_RESPONSE = {
     "maxPower": [
         {
@@ -132,6 +170,40 @@ def test_get_consumption_data(mock_token, mock_get, snapshot):
     )
 
 
+@patch("aiohttp.ClientSession.get")
+@patch.object(
+    DatadisConnector, "_async_get_token", new_callable=AsyncMock, return_value=True
+)
+def test_get_consumption_data_skips_zero_hour(mock_token, mock_get):
+    """A stray "00:00" row is dropped, not remapped, and never aborts the fetch."""
+    mock_response = MagicMock()
+    mock_response.status = 200
+    mock_response.text = AsyncMock(return_value="text")
+    mock_response.json = AsyncMock(
+        return_value=CONSUMPTIONS_RESPONSE_WITH_ZERO_HOUR
+    )
+    mock_get.return_value.__aenter__.return_value = mock_response
+    connector = DatadisConnector(MOCK_USERNAME, MOCK_PASSWORD)
+
+    # Range spans the previous day, so a remapped "00:00" -> 2022/10/21 23:00
+    # would fall inside it; its absence proves the explicit skip, not the filter.
+    result = connector.get_consumption_data(
+        "ESXXXXXXXXXXXXXXXXTEST",
+        "2",
+        datetime.datetime(2022, 10, 21, 0, 0, 0),
+        datetime.datetime(2022, 10, 22, 23, 59, 59),
+        "0",
+        5,
+    )
+
+    datetimes = [x.datetime for x in result]
+    assert datetimes == [
+        datetime.datetime(2022, 10, 22, 0, 0),
+        datetime.datetime(2022, 10, 22, 1, 0),
+    ]
+    assert datetime.datetime(2022, 10, 21, 23, 0) not in datetimes
+
+
 @patch("aiohttp.ClientSession.get")
 @patch.object(
     DatadisConnector, "_async_get_token", new_callable=AsyncMock, return_value=True

e_data-2.0.1.dev133/edata/tests/test_migrations.py

@@ -0,0 +1,185 @@
+"""Tests for the 1.3.3 -> 2.0 legacy JSON migration."""
+
+from collections.abc import AsyncIterator
+from datetime import datetime
+import json
+import os
+import shutil
+from unittest.mock import patch
+
+import pytest
+import pytest_asyncio
+
+from edata.database.controller import EdataDB
+from edata.database.migrations import legacy_json_1_3_3 as legacy
+from edata.services.data_service import DataService
+
+ASSETS = os.path.join(os.path.dirname(__file__), "assets")
+CUPS = "ESXXXXXXXXXXXXXXXXTEST"
+
+
+def _reset_singleton() -> None:
+    EdataDB._instance = None
+    EdataDB._engine = None
+    EdataDB._db_url = None
+
+
+def _raw() -> dict:
+    with open(os.path.join(ASSETS, "legacy_1.3.3.json"), encoding="utf-8") as f:
+        return json.load(f)
+
+
+def _install_legacy_file(storage_dir: str, cups: str = CUPS) -> str:
+    """Copy the asset into the 1.3.3 storage location for ``storage_dir``."""
+    dest = legacy._legacy_path(storage_dir, cups)
+    os.makedirs(os.path.dirname(dest), exist_ok=True)
+    shutil.copy(os.path.join(ASSETS, "legacy_1.3.3.json"), dest)
+    return dest
+
+
+@pytest_asyncio.fixture
+async def data_service(tmp_path) -> AsyncIterator[DataService]:
+    """A DataService on an isolated on-disk database, singleton reset around it."""
+    _reset_singleton()
+    with (
+        patch("edata.services.data_service.DatadisConnector"),
+        patch("edata.services.data_service.REDataConnector"),
+    ):
+        service = DataService(CUPS, "user", "pwd", storage_path=str(tmp_path))
+    yield service
+    if EdataDB._engine is not None:
+        await EdataDB._engine.dispose()
+    _reset_singleton()
+
+
+# --- pure mappers ---
+
+
+def test_to_supply_renames_fields() -> None:
+    """Supply maps camelCase fields to the 2.0 snake_case model."""
+    supply = legacy._to_supply(_raw()["supplies"][0])
+    assert supply.cups == CUPS
+    assert supply.point_type == 5
+    assert supply.distributor_code == "2"
+    assert supply.date_start == datetime(2023, 1, 1)
+
+
+def test_to_contract_builds_power_list_dropping_none() -> None:
+    """Contract power list is [p1, p2], dropping a None p2."""
+    contracts = [legacy._to_contract(c) for c in _raw()["contracts"]]
+    assert contracts[0].power == [4.6, 4.6]
+    assert contracts[0].distributor_code == "2"
+    # second contract has power_p2=None -> single-element power list
+    assert contracts[1].power == [3.3]
+
+
+def test_to_energy_renames_and_defaults() -> None:
+    """Energy renames value_kWh and defaults generation/selfconsumption."""
+    energy = legacy._to_energy(_raw()["consumptions"][2])
+    assert energy.consumption_kwh == 0.3
+    assert energy.surplus_kwh == 0.1
+    assert energy.generation_kwh == 0
+    assert energy.selfconsumption_kwh == 0
+    assert energy.real is True
+
+
+def test_to_power_and_pvpc_rename() -> None:
+    """Power and pvpc rename their value fields to snake_case."""
+    power = legacy._to_power(_raw()["maximeter"][0])
+    assert power.value_kw == 3.2
+    pvpc = legacy._to_pvpc(_raw()["pvpc"][0])
+    assert pvpc.value_eur_kwh == 0.12
+    assert pvpc.delta_h == 1
+
+
+# --- migration apply ---
+
+
+@pytest.mark.asyncio
+async def test_apply_imports_raw_records_only(
+    data_service: DataService, tmp_path
+) -> None:
+    """Import the raw records and ignores old *_sum aggregates."""
+    _install_legacy_file(str(tmp_path))
+    result = await legacy.apply(data_service.db, str(tmp_path), CUPS)
+
+    assert result is not None
+    assert result.name == "legacy_json_1_3_3"
+    assert (
+        result.supplies,
+        result.contracts,
+        result.energy,
+        result.power,
+        result.pvpc,
+    ) == (1, 2, 5, 2, 2)
+
+    supply = await data_service.get_supply()
+    assert supply is not None and supply.cups == CUPS
+    assert len(await data_service.get_contracts()) == 2
+    assert len(await data_service.get_energy()) == 5
+    assert len(await data_service.get_power()) == 2
+    assert len(await data_service.get_pvpc()) == 2
+
+    # The old *_sum aggregates in the file must be ignored by the migration.
+    assert await data_service.get_statistics("day") == []
+    assert await data_service.get_statistics("month") == []
+
+
+@pytest.mark.asyncio
+async def test_apply_missing_file_is_noop(data_service: DataService, tmp_path) -> None:
+    """Return None and write nothing when no legacy file exists."""
+    result = await legacy.apply(data_service.db, str(tmp_path), CUPS)
+    assert result is None
+    assert await data_service.get_supply() is None
+
+
+@pytest.mark.asyncio
+async def test_apply_is_idempotent(data_service: DataService, tmp_path) -> None:
+    """Re-running apply does not duplicate rows (upsert)."""
+    _install_legacy_file(str(tmp_path))
+    await legacy.apply(data_service.db, str(tmp_path), CUPS)
+    await legacy.apply(data_service.db, str(tmp_path), CUPS)
+
+    assert len(await data_service.get_energy()) == 5
+    assert len(await data_service.get_contracts()) == 2
+    assert len(await data_service.get_power()) == 2
+    assert len(await data_service.get_pvpc()) == 2
+
+
+# --- orchestrator (DataService.run_migrations) ---
+
+
+@pytest.mark.asyncio
+async def test_run_migrations_compiles_statistics(
+    data_service: DataService, tmp_path
+) -> None:
+    """run_migrations imports and compiles day/month statistics."""
+    _install_legacy_file(str(tmp_path))
+    results = await data_service.run_migrations(compile_statistics=True)
+
+    assert [r.name for r in results] == ["legacy_json_1_3_3"]
+    # Raw records imported and day/month statistics compiled immediately.
+    assert len(await data_service.get_energy()) == 5
+    assert len(await data_service.get_statistics("day")) > 0
+    assert len(await data_service.get_statistics("month")) > 0
+
+
+@pytest.mark.asyncio
+async def test_run_migrations_can_skip_compilation(
+    data_service: DataService, tmp_path
+) -> None:
+    """compile_statistics=False imports raw records without statistics."""
+    _install_legacy_file(str(tmp_path))
+    results = await data_service.run_migrations(compile_statistics=False)
+
+    assert len(results) == 1
+    assert len(await data_service.get_energy()) == 5
+    assert await data_service.get_statistics("day") == []
+    assert await data_service.get_statistics("month") == []
+
+
+@pytest.mark.asyncio
+async def test_run_migrations_no_legacy_file(data_service: DataService) -> None:
+    """run_migrations is a no-op when there is no legacy file."""
+    results = await data_service.run_migrations()
+    assert results == []

{e_data-2.0.1.dev131 → e_data-2.0.1.dev133}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "e-data"
-version = "2.0.1.dev131"
+version = "2.0.1.dev133"
 description = "Python library for managing spanish energy data from various web providers"
 readme = "README.md"
 authors = [