python-esios 2.3.0__py3-none-any.whl → 2.4.1__py3-none-any.whl

esios/.agents/skills/esios/SKILL.md

@@ -60,8 +60,8 @@ print(sheet.frequency)    # "hourly" or "hourly-quarterly"
 
 | ID | Name | Description | Geos |
 |----|------|-------------|------|
-| 600 | Precio mercado spot | OMIE spot market price | ES, PT, FR, DE, BE, NL |
-| 1001 | Precio mercado diario | Day-ahead market price | ES |
+| 600 | Precio mercado SPOT Diario | OMIE spot / day-ahead market price | ES, PT, FR, DE, BE, NL |
+| 1001 | PVPC T. 2.0TD | Término de facturación de energía activa del PVPC (voluntary price for small consumers) | Península, Canarias, Baleares, Ceuta, Melilla |
 | 10033 | Demanda real | Real-time electricity demand | ES |
 | 10034 | Generación eólica | Real-time wind generation | ES |
 | 10035 | Generación solar FV | Real-time solar PV generation | ES |

esios/constants.py

@@ -14,7 +14,17 @@ MAX_RETRIES = 3
 RETRY_MIN_WAIT = 2  # seconds
 RETRY_MAX_WAIT = 10  # seconds
 
-# ESIOS API limits responses to ~3 weeks of data per request
-CHUNK_SIZE_DAYS = 21
+# ESIOS API chunk sizes for historical data fetching.
+# High-geo indicators (40+ geos) timeout (504) at >21 days.
+# Low-geo indicators handle 6+ months per request in <0.1s.
+CHUNK_SIZE_DAYS = 21  # Legacy default, kept for backward compat
+CHUNK_SIZE_DAYS_LOW_GEO = 180  # 6 months for indicators with few geos
+CHUNK_SIZE_DAYS_HIGH_GEO = 21  # Conservative for indicators with many geos
+HIGH_GEO_THRESHOLD = 15  # Indicators with >= this many geos use smaller chunks
+
+# Concurrent chunk fetching within a single indicator.
+# 4 workers gives ~17-95x speedup over sequential with no errors.
+# Diminishing returns past 4 (ESIOS server becomes the bottleneck).
+DEFAULT_CHUNK_WORKERS = 4
 
 TIMEZONE = "Europe/Madrid"

esios/managers/indicators.py

@@ -3,13 +3,20 @@
 from __future__ import annotations
 
 import logging
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import timedelta
 from typing import Any
 
 import pandas as pd
 
 from esios.cache import CacheStore
-from esios.constants import CHUNK_SIZE_DAYS, TIMEZONE
+from esios.constants import (
+    CHUNK_SIZE_DAYS_HIGH_GEO,
+    CHUNK_SIZE_DAYS_LOW_GEO,
+    DEFAULT_CHUNK_WORKERS,
+    HIGH_GEO_THRESHOLD,
+    TIMEZONE,
+)
 from esios.managers.base import BaseManager
 from esios.models.indicator import Indicator
 from esios.processing.dataframes import to_dataframe
@@ -131,6 +138,88 @@ class IndicatorHandle:
             f"Available: {', '.join(available)}"
         )
 
+    @property
+    def _chunk_days(self) -> int:
+        """Choose chunk size based on indicator's geo count.
+
+        ESIOS API times out (504) for high-geo indicators (40+ geos) with
+        windows larger than ~3 weeks. Low-geo indicators handle 6+ months
+        per request in <0.1s.
+
+        When geos are unknown (empty metadata), uses the conservative
+        chunk size to avoid timeouts on first fetch.
+        """
+        geo_count = len(self.geos)
+        if geo_count == 0:
+            # Unknown geo count — be conservative
+            return CHUNK_SIZE_DAYS_HIGH_GEO
+        if geo_count >= HIGH_GEO_THRESHOLD:
+            return CHUNK_SIZE_DAYS_HIGH_GEO
+        return CHUNK_SIZE_DAYS_LOW_GEO
+
+    def _fetch_one(
+        self, start: str, end: str, base_params: dict[str, Any],
+    ) -> list[dict]:
+        """Fetch a single date-range chunk from the ESIOS API."""
+        params = {
+            **base_params,
+            "start_date": start,
+            "end_date": end + "T23:59:59",
+        }
+        logger.debug("Fetch %s → %s", start, end)
+        data = self._manager._get(f"indicators/{self.id}", params=params)
+        return data.get("indicator", {}).get("values", [])
+
+    def _fetch_chunks(
+        self,
+        gaps: list,
+        base_params: dict[str, Any],
+        max_workers: int = DEFAULT_CHUNK_WORKERS,
+    ) -> list[dict]:
+        """Fetch all gap chunks concurrently, return values in order.
+
+        Builds a list of (start, end) chunks from the gaps, then fetches
+        them in parallel using a thread pool. Results are reassembled in
+        chronological order.
+        """
+        chunk_delta = timedelta(days=self._chunk_days)
+
+        # Build chunk list
+        chunks: list[tuple[str, str]] = []
+        for gap in gaps:
+            current = gap.start
+            while current <= gap.end:
+                chunk_end = min(current + chunk_delta, gap.end)
+                chunks.append((
+                    current.strftime("%Y-%m-%d"),
+                    chunk_end.strftime("%Y-%m-%d"),
+                ))
+                current = chunk_end + timedelta(days=1)
+
+        if not chunks:
+            return []
+
+        if len(chunks) == 1:
+            return self._fetch_one(chunks[0][0], chunks[0][1], base_params)
+
+        # Fetch concurrently, preserve order
+        results: list[list[dict] | None] = [None] * len(chunks)
+        with ThreadPoolExecutor(max_workers=max_workers) as pool:
+            futures = {
+                pool.submit(self._fetch_one, s, e, base_params): i
+                for i, (s, e) in enumerate(chunks)
+            }
+            for future in as_completed(futures):
+                idx = futures[future]
+                results[idx] = future.result()
+
+        # Flatten in chronological order
+        all_values: list[dict] = []
+        for chunk_values in results:
+            if chunk_values:
+                all_values.extend(chunk_values)
+        return all_values
+
     def historical(
         self,
         start: str,
@@ -143,11 +232,15 @@ class IndicatorHandle:
         time_trunc: str | None = None,
         geo_trunc: str | None = None,
         column_name: str | None = None,
+        chunk_workers: int = DEFAULT_CHUNK_WORKERS,
     ) -> pd.DataFrame:
         """Fetch historical values as a DataFrame with DatetimeIndex.
 
         Uses local parquet cache when enabled. Only fetches missing date ranges
-        from the API. Automatically chunks requests exceeding ~3 weeks.
+        from the API. Automatically chunks requests and fetches concurrently.
+
+        Chunk size adapts to the indicator's geo count: 180 days for low-geo
+        indicators, 21 days for high-geo (≥15 geos) to avoid ESIOS timeouts.
 
         When multiple geo_ids are present (e.g. indicator 600 returns data for
         several countries), the result is pivoted so each geo becomes a column
@@ -158,6 +251,8 @@ class IndicatorHandle:
                 Useful for single-column results where a stable name like
                 ``"value"`` is preferred over the default geo_name or
                 indicator ID.
+            chunk_workers: Number of concurrent threads for fetching chunks.
+                Defaults to 4. Set to 1 for sequential fetching.
         """
         base_params: dict[str, Any] = {
             "locale": locale,
@@ -211,24 +306,8 @@ class IndicatorHandle:
             from esios.cache import DateRange
             gaps = [DateRange(start_date, end_date)]
 
-        # -- Fetch missing ranges ----------------------------------------------
-        all_values: list[dict] = []
-        chunk_delta = timedelta(days=CHUNK_SIZE_DAYS)
-
-        for gap in gaps:
-            current = gap.start
-            gap_end = gap.end
-            while current <= gap_end:
-                chunk_end = min(current + chunk_delta, gap_end)
-                params = {
-                    **base_params,
-                    "start_date": current.strftime("%Y-%m-%d"),
-                    "end_date": chunk_end.strftime("%Y-%m-%d") + "T23:59:59",
-                }
-                logger.debug("Fetch %s → %s", params["start_date"], params["end_date"])
-                data = self._manager._get(f"indicators/{self.id}", params=params)
-                all_values.extend(data.get("indicator", {}).get("values", []))
-                current = chunk_end + timedelta(days=1)
+        # -- Fetch missing ranges (concurrent + adaptive chunk size) -----------
+        all_values = self._fetch_chunks(gaps, base_params, max_workers=chunk_workers)
 
         # Learn any new geo mappings from the response
         self._enrich_geo_map(all_values)

esios/processing/i90.py

@@ -33,6 +33,43 @@ def _any_value_greater_than_30(series: np.ndarray) -> bool:
     return any(v > 30 for v in series if isinstance(v, (int, float, np.integer, np.floating)) and not np.isnan(v))
 
 
+# Labels that REE uses for the cells sitting between the index columns and the
+# per-period value columns of an I90 sheet. Match is exact (case-insensitive,
+# trimmed). The count of these cells per sheet has varied across REE format
+# revisions — historically 2 ("Hora" / "Cuarto de Hora del dia" + "Total");
+# from Oct 2025 the MTU 15-min transition dropped "Total" on several sheets,
+# leaving 1. Counting them dynamically keeps the parser resilient to either.
+_SEPARATOR_LABELS = frozenset({
+    "cuarto de hora del dia",
+    "hora del dia",
+    "hora",
+    "total",
+    "indicadores",
+})
+
+
+def _count_header_separators(row: np.ndarray, idx_col_start: int) -> int:
+    """Count separator cells immediately preceding the time-value block.
+
+    Walks ``row`` backwards from ``idx_col_start - 1``, incrementing on each
+    cell whose text matches a known separator label and stopping at the first
+    non-matching cell or NaN. A NaN cell signals the start of the index-column
+    placeholder zone in double-header layouts (where index labels live on the
+    other header row, leaving the date row blank under each index position).
+    """
+    n = 0
+    for i in range(idx_col_start - 1, -1, -1):
+        cell = row[i]
+        if cell is None or (isinstance(cell, float) and np.isnan(cell)):
+            break
+        text = str(cell).strip().lower()
+        if text in _SEPARATOR_LABELS:
+            n += 1
+            continue
+        break
+    return n
+
+
 class I90Book:
     """Represents an I90DIA workbook (XLS) with lazy sheet preprocessing.
 
@@ -221,6 +258,11 @@ class I90Sheet:
             return pd.DataFrame()
 
         columns_date = self._normalize_datetime_columns(columns_prior[idx_col_start:])
+        # _normalize_datetime_columns sets _n_columns_totals from the time-block
+        # content (NaN-filler vs sequential). Override with a header-label count
+        # so the index slice survives REE format revisions that add or drop a
+        # "Total" column without touching the time-axis encoding.
+        self._n_columns_totals = _count_header_separators(columns_prior, idx_col_start)
         columns_variable = columns[idx_col_start:]
         columns_index = columns[: idx_col_start - self._n_columns_totals]
 
@@ -230,6 +272,7 @@ class I90Sheet:
         self, idx_col_start: int, columns: np.ndarray
     ) -> tuple[np.ndarray, np.ndarray, np.ndarray, None]:
         columns_date = self._normalize_datetime_columns(columns[idx_col_start:])
+        self._n_columns_totals = _count_header_separators(columns, idx_col_start)
         columns_index = columns[: idx_col_start - self._n_columns_totals]
         return columns, columns_index, columns_date, None
 

{python_esios-2.3.0.dist-info → python_esios-2.4.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-esios
-Version: 2.3.0
+Version: 2.4.1
 Summary: A Python wrapper for the ESIOS API (Spanish electricity market)
 Project-URL: Homepage, https://github.com/datons/python-esios
 Project-URL: Repository, https://github.com/datons/python-esios
@@ -85,7 +85,7 @@ from esios import ESIOSClient
 client = ESIOSClient()
 
 # Get indicator data as DataFrame
-handle = client.indicators.get(600)  # PVPC price
+handle = client.indicators.get(600)  # Day-ahead spot price (OMIE)
 df = handle.historical("2025-01-01", "2025-01-31")
 
 # Search indicators
@@ -99,8 +99,8 @@ client.archives.download(1, start="2025-01-01", end="2025-01-31", output_dir="./
 
 | ID | Name | Description |
 |----|------|-------------|
-| 600 | PVPC | Voluntary price for small consumers |
-| 1001 | Day-ahead price | OMIE spot market price |
+| 600 | Day-ahead price | OMIE spot market price |
+| 1001 | PVPC | Voluntary price for small consumers (2.0TD) |
 | 10033 | Demand | Real-time electricity demand |
 | 10034 | Wind generation | Real-time wind generation |
 | 10035 | Solar PV generation | Real-time solar generation |

{python_esios-2.3.0.dist-info → python_esios-2.4.1.dist-info}/RECORD

@@ -3,9 +3,9 @@ esios/async_client.py,sha256=OVNNZwFbvPyUnu7LVr7X5MdXlk_-AJ1lfkUE0OODlbQ,3452
 esios/cache.py,sha256=GgbrL9Rc9aLrEWHvXtQOCGQRgq2T4m6VBJDvBJfWMTk,18920
 esios/catalog.py,sha256=xWwMx5I32m34npjAXHh-Ua4e_0pfG89yxUC_Vy9VlAA,16811
 esios/client.py,sha256=rLgdyPFII6CC_TJwgkHaScJ7nBUpt85N94mujKAn0d0,5825
-esios/constants.py,sha256=pwB2UlBI96zYBA8wAbcCSHcm_E-aIj2hBarDA8t1Vp8,474
+esios/constants.py,sha256=yfxSNG37i4dkpa7x0CBvXTroyddn5jhNTuWGDhAq3-0,1074
 esios/exceptions.py,sha256=AiWLdRDWj50JEsld9CvVBsfLnZZKFmW62_bZmZ7Z_eA,899
-esios/.agents/skills/esios/SKILL.md,sha256=_5wCzMMB8FHWcAPeMA5vGklZFEGBEvU5wBOryNIogzM,6252
+esios/.agents/skills/esios/SKILL.md,sha256=D1wXiKyk7HoFw6CapccoORrtMXUpS2BuAVEChLu3AJE,6375
 esios/cli/__init__.py,sha256=9gd5ZDIH1-yNP_xcd60ethOFXm9w6un0CJ9CX0Qvb2A,256
 esios/cli/app.py,sha256=j1d8QWtKTTsWozSqqQitTkzzRjBE6OXY0ZZWYdS19wE,1524
 esios/cli/archives.py,sha256=Re9ZMauTiJlHdmiE7F3ZlV2wfaEyShS0C7Z4M2X4Ra8,7715
@@ -24,7 +24,7 @@ esios/data/time_periods.yaml,sha256=oyisKYYyOGA57eEAqkFFx6B3x9rdSl0DokZe5gNZfMw,
 esios/managers/__init__.py,sha256=-1AwL7arUf7WEZn1RSiK_DZhY3j6U4GE9_dqjbukCJc,268
 esios/managers/archives.py,sha256=PG-1gQYEiJUVQQtTKIZeEoWIsS-gkWT3ZHy89c8tTW8,9293
 esios/managers/base.py,sha256=7XcdrUtUOPuqfHYlz4w562TD8o9cNdBWOgs4CHHonoo,835
-esios/managers/indicators.py,sha256=nbmKsvBTPO2w3FlcVYv9WOtCTU8xMjFvT_d1AcA2sbg,17506
+esios/managers/indicators.py,sha256=4f1wLhT33Fc93ixHr51DIzIBqzznJSaoeLfWOT-2EQ0,20260
 esios/managers/offer_indicators.py,sha256=0MjEKkj77YC2fRSHVTEc7FW6E8AuwwciAXK-bOVEL5Q,4187
 esios/models/__init__.py,sha256=oppuTASpf0Dh2KbGMXInULT0F4sELjeo-9UhPiPOZiA,289
 esios/models/archive.py,sha256=P2LaT7_ff4ujwqVn_ofgQP3dbpf7jqON0R22dKwSJ_w,1062
@@ -32,10 +32,10 @@ esios/models/indicator.py,sha256=u1AJyEA3YeOqQFjV08_lzyMaofuCiMoLPjvosls9gfE,111
 esios/models/offer_indicator.py,sha256=nA80Y7Yp0utDaDOdZ-ObcWTsAdhvuXlfJjJBpdVQ7Lo,758
 esios/processing/__init__.py,sha256=1kLt_gO_wDhXM1BbY0zTyfAYo-CjYKW1ljgRRDZ7USM,278
 esios/processing/dataframes.py,sha256=OitzBvAerssGP2VXNC-sSO48XsHdIB2nKTUgByN5eYQ,2524
-esios/processing/i90.py,sha256=fI8DfY8CD2kF1_ZrAzuEDxN0m7Vh3CV3dIn32lxKffA,11687
+esios/processing/i90.py,sha256=UZEI6f0pG6mbMaVTh_L3KtUp9PuvxvfXYaG9-ePNIxg,13622
 esios/processing/zip.py,sha256=12LbFHJTdX_h3JG-clEgQ4Haj-kw0UjfopGLlCRXfGM,1913
-python_esios-2.3.0.dist-info/METADATA,sha256=KtWOIGA-o9z7mxZD8vryWTOntVZbxNm_LIVaXYhFD3g,3169
-python_esios-2.3.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
-python_esios-2.3.0.dist-info/entry_points.txt,sha256=7ngseyIyvJ4buTHFL9htaZ4tTFHpG4zzJNkc8B5Jr8U,40
-python_esios-2.3.0.dist-info/licenses/LICENSE,sha256=LorLs1-VeBW70Wo9fLAtLJN7nNd6Poy0xzvqdWVqFlE,35128
-python_esios-2.3.0.dist-info/RECORD,,
+python_esios-2.4.1.dist-info/METADATA,sha256=_qFX68pzna9JIBW1zm57X8WNxZ70BO8zaxC4g1nbmTM,3194
+python_esios-2.4.1.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+python_esios-2.4.1.dist-info/entry_points.txt,sha256=7ngseyIyvJ4buTHFL9htaZ4tTFHpG4zzJNkc8B5Jr8U,40
+python_esios-2.4.1.dist-info/licenses/LICENSE,sha256=LorLs1-VeBW70Wo9fLAtLJN7nNd6Poy0xzvqdWVqFlE,35128
+python_esios-2.4.1.dist-info/RECORD,,