tesorotools-python 0.0.52__tar.gz → 0.0.53__tar.gz

{tesorotools_python-0.0.52 → tesorotools_python-0.0.53}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tesorotools-python
-Version: 0.0.52
+Version: 0.0.53
 Requires-Python: >=3.13
 Requires-Dist: babel>=2.17
 Requires-Dist: matplotlib>=3.10

tesorotools_python-0.0.53/src/tesorotools/providers/bde.py

@@ -0,0 +1,479 @@
+"""Bank of Spain (Banco de Espana, BdE) data provider.
+
+Downloads historical time series from the BdE public REST API.
+No authentication required.
+
+Install with::
+
+    pip install tesorotools-python[bde]
+
+API reference
+-------------
+Endpoints:
+    https://app.bde.es/bierest/resources/srdatosapp/listaSeries
+    https://app.bde.es/bierest/resources/srdatosapp/favoritas
+
+Query parameters (names are fixed by the BdE API):
+    ``idioma``  : ``"es"`` | ``"en"``
+    ``series``  : comma-separated series codes
+    ``rango``   : temporal range; **the valid values depend on
+                  the frequency of the series** (see below).
+
+The ``rango`` parameter
+-----------------------
+The BdE API rejects a ``rango`` value that does not match the
+series frequency (``codFrecuencia``).  The official table is:
+
+    ==========  ==============================
+    Frequency   Valid ``rango`` values
+    ==========  ==============================
+    Daily (D)   ``3M``, ``12M``, ``36M``
+    Monthly (M) ``30M``, ``60M``, ``MAX``
+    Quarter (Q) ``30M``, ``60M``, ``MAX``
+    Annual (A)  ``60M``, ``MAX``
+    ==========  ==============================
+
+Every frequency additionally accepts a specific calendar year
+(e.g. ``rango=2024``), which is the only frequency-agnostic
+value.  Daily series have **no** ``MAX``: their largest window
+is 36 months, so longer histories are paginated year by year.
+
+Because ``rango`` applies to the whole request, series of
+different frequencies cannot share a request.  This provider
+therefore detects each code's frequency first (via the cheap
+``favoritas`` endpoint) and downloads one frequency at a time.
+
+Notes
+-----
+- ECB DCF series (credit stock data) are also available
+  through this API because the BdE redistributes them.
+- The API accepts multiple series per request, but may
+  time out with too many.  Downloads are batched.
+- Timestamps include timezone info (UTC).  They are
+  normalized to tz-naive midnight for pandas compatibility.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import ClassVar, NotRequired, TypedDict, cast
+
+import pandas as pd
+import requests
+
+from tesorotools.providers.base import DataProvider
+
+logger = logging.getLogger(__name__)
+
+
+class _BdeSeriesEntry(TypedDict):
+    """Shape of a single series object in the ``listaSeries``
+    JSON response.  ``informacion`` is optional metadata."""
+
+    serie: str
+    fechas: list[str]
+    valores: list[int | float | None]
+    decimales: int
+    codFrecuencia: str
+    informacion: NotRequired[list[dict[str, str]]]
+
+
+_BASE_URL = "https://app.bde.es/bierest/resources/srdatosapp"
+_LIST_URL = f"{_BASE_URL}/listaSeries"
+_FAV_URL = f"{_BASE_URL}/favoritas"
+
+DEFAULT_TIMEOUT = 30
+BATCH_SIZE = 10
+
+#: Frequency code the API treats specially: no ``MAX`` and a
+#: distinct set of month-window ranges.
+_DAILY = "D"
+#: Largest month window the API offers for daily series.
+_MAX_DAILY_MONTHS = 36
+#: Extra months added when translating a start date into a
+#: month-window range, so publication lag never truncates it.
+_RANGE_BUFFER_MONTHS = 3
+
+
+class BdeProvider(DataProvider):
+    """Provider that downloads series from the Bank of Spain.
+
+    The BdE API is public and requires no credentials.
+
+    Parameters
+    ----------
+    language
+        Language for metadata.  ``"es"`` for Spanish,
+        ``"en"`` for English.
+    timeout
+        Maximum seconds to wait per HTTP request.
+    """
+
+    PROVIDER_NAME: ClassVar[str] = "bde"
+
+    def __init__(
+        self,
+        *,
+        language: str = "es",
+        timeout: int = DEFAULT_TIMEOUT,
+    ) -> None:
+        self._language = language
+        self._timeout = timeout
+
+    def fetch(
+        self,
+        codes: list[str],
+        start: str | None = None,
+        end: str | None = None,
+    ) -> pd.DataFrame:
+        """Download one or more series from the BdE.
+
+        Detects the frequency of every code, groups the codes
+        by frequency (series of different frequencies cannot
+        share a request), downloads each group with a valid
+        ``rango`` for that frequency, and merges everything
+        into one DataFrame.
+
+        Parameters
+        ----------
+        codes
+            BdE series codes.
+        start
+            Earliest date to include (ISO format).
+            If ``None``, fetches the full history.
+        end
+            Ignored by this provider.  The BdE API always
+            returns data up to the latest observation.
+        """
+        if not codes:
+            return pd.DataFrame()
+
+        frequencies = self._detect_frequencies(codes)
+        missing = [c for c in codes if c not in frequencies]
+        if missing:
+            logger.warning(
+                "BdE: series not found: %s",
+                ", ".join(missing),
+            )
+
+        start_ts = pd.Timestamp(start) if start else None
+        now = pd.Timestamp.now()
+        frames: list[pd.DataFrame] = []
+
+        for freq, group in _group_by_frequency(codes, frequencies).items():
+            df = self._fetch_group(group, freq, start_ts, now)
+            if not df.empty:
+                frames.append(df)
+
+        if not frames:
+            return pd.DataFrame()
+
+        result = pd.concat(frames, axis=1)
+        result.sort_index(inplace=True)
+        result = result[[c for c in codes if c in result.columns]]
+
+        if start_ts is not None:
+            result = result.loc[result.index >= start_ts]
+
+        return result
+
+    def is_available(self) -> bool:
+        """Ping the BdE API to check connectivity."""
+        try:
+            resp = requests.get(
+                _FAV_URL,
+                params={
+                    "idioma": self._language,
+                    "series": "DN_1TI2TIE42",
+                },
+                timeout=5,
+            )
+            return resp.status_code == 200
+        except requests.RequestException:
+            return False
+
+    # -- frequency detection ---------------------------------
+
+    def _detect_frequencies(self, codes: list[str]) -> dict[str, str]:
+        """Return ``{code: codFrecuencia}`` for the codes the
+        API knows about.
+
+        Uses the lightweight ``favoritas`` endpoint (last datum
+        only), which returns the frequency of every series and
+        works for a mixed-frequency batch.
+        """
+        frequencies: dict[str, str] = {}
+        for batch in _batches(codes, BATCH_SIZE):
+            resp = requests.get(
+                _FAV_URL,
+                params={
+                    "idioma": self._language,
+                    "series": ",".join(batch),
+                },
+                timeout=self._timeout,
+            )
+            resp.raise_for_status()
+            payload = resp.json()
+            _check_api_error(payload)
+            for entry in cast(list[dict[str, object]], payload):
+                code = entry.get("serie")
+                freq = entry.get("codFrecuencia")
+                if isinstance(code, str) and isinstance(freq, str):
+                    frequencies[code] = freq
+        return frequencies
+
+    # -- per-frequency download ------------------------------
+
+    def _fetch_group(
+        self,
+        codes: list[str],
+        freq: str,
+        start: pd.Timestamp | None,
+        now: pd.Timestamp,
+    ) -> pd.DataFrame:
+        """Download a set of same-frequency codes."""
+        if freq == _DAILY and start is None:
+            first_year = self._earliest_year(codes, now)
+            ranges = _year_ranges(first_year, now.year)
+        else:
+            ranges = _plan_ranges(freq, start, now)
+        return self._download_ranges(codes, ranges)
+
+    def _earliest_year(self, codes: list[str], now: pd.Timestamp) -> int:
+        """First observation year across a daily batch.
+
+        Reads the ``"Primer valor"`` metadata (format
+        ``[DD/MM/YYYY]``) returned by ``listaSeries``.  Falls
+        back to the last ``_MAX_DAILY_MONTHS`` when no year can
+        be parsed, so a metadata change never breaks the fetch.
+        """
+        params = {
+            "idioma": self._language,
+            "series": ",".join(codes[:BATCH_SIZE]),
+            "rango": f"{_MAX_DAILY_MONTHS}M",
+        }
+        resp = requests.get(_LIST_URL, params=params, timeout=self._timeout)
+        resp.raise_for_status()
+        payload = resp.json()
+        _check_api_error(payload)
+
+        years = [
+            year
+            for entry in cast(list[_BdeSeriesEntry], payload)
+            if (year := _first_observation_year(entry)) is not None
+        ]
+        if not years:
+            logger.warning(
+                "BdE: could not determine daily history start; "
+                "limiting to last %d months",
+                _MAX_DAILY_MONTHS,
+            )
+            return now.year - _MAX_DAILY_MONTHS // 12
+        return min(years)
+
+    def _download_ranges(
+        self,
+        codes: list[str],
+        ranges: list[str],
+    ) -> pd.DataFrame:
+        """Download ``codes`` for every ``rango`` in ``ranges``
+        and stack the results by date."""
+        per_range: list[pd.DataFrame] = []
+        for time_range in ranges:
+            batch_frames: list[pd.DataFrame] = []
+            for batch in _batches(codes, BATCH_SIZE):
+                df = self._download_batch(batch, time_range)
+                if not df.empty:
+                    batch_frames.append(df)
+            if batch_frames:
+                per_range.append(pd.concat(batch_frames, axis=1))
+
+        if not per_range:
+            return pd.DataFrame()
+
+        combined = pd.concat(per_range, axis=0)
+        combined = combined[~combined.index.duplicated(keep="last")]
+        combined.sort_index(inplace=True)
+        return combined
+
+    def _download_batch(
+        self,
+        codes: list[str],
+        time_range: str,
+    ) -> pd.DataFrame:
+        params = {
+            "idioma": self._language,
+            "series": ",".join(codes),
+            "rango": time_range,
+        }
+        logger.info(
+            "BdE: downloading %d series (rango=%s)...",
+            len(codes),
+            time_range,
+        )
+        resp = requests.get(
+            _LIST_URL,
+            params=params,
+            timeout=self._timeout,
+        )
+        resp.raise_for_status()
+
+        payload = resp.json()
+        _check_api_error(payload)
+        return _parse_series_list(cast(list[_BdeSeriesEntry], payload))
+
+
+# ----------------------------------------------------------
+# Module-level helpers
+# ----------------------------------------------------------
+
+
+class _BdeErrorResponse(TypedDict, total=False):
+    errNum: int  # noqa: N815
+    errMsgUsr: str  # noqa: N815
+    errMsgDebug: str  # noqa: N815
+
+
+def _check_api_error(payload: object) -> None:
+    """Raise RuntimeError if the API returned an error."""
+    if not isinstance(payload, dict):
+        return
+    if "errNum" not in payload:
+        return
+    err = cast(_BdeErrorResponse, payload)
+    raise RuntimeError(
+        f"BdE API error {err.get('errNum', '?')}: {err.get('errMsgUsr', '')}"
+    )
+
+
+def _group_by_frequency(
+    codes: list[str],
+    frequencies: dict[str, str],
+) -> dict[str, list[str]]:
+    """Group *codes* by their frequency, preserving order.
+
+    Codes absent from *frequencies* (unknown to the API) are
+    dropped; ``fetch`` has already warned about them.
+    """
+    groups: dict[str, list[str]] = {}
+    for code in codes:
+        freq = frequencies.get(code)
+        if freq is None:
+            continue
+        groups.setdefault(freq, []).append(code)
+    return groups
+
+
+def _plan_ranges(
+    freq: str,
+    start: pd.Timestamp | None,
+    now: pd.Timestamp,
+) -> list[str]:
+    """Pick the ``rango`` value(s) needed to cover *start* for a
+    series of frequency *freq*.
+
+    Returns a list because daily series older than
+    ``_MAX_DAILY_MONTHS`` must be paginated one calendar year
+    at a time.  Every other case is a single keyword range.
+    """
+    if freq == _DAILY:
+        # Daily start is never None here: that case is resolved
+        # in _fetch_group, which knows the first observation.
+        months = _months_between(cast(pd.Timestamp, start), now)
+        if months <= 3:
+            return ["3M"]
+        if months <= 12:
+            return ["12M"]
+        if months <= _MAX_DAILY_MONTHS:
+            return [f"{_MAX_DAILY_MONTHS}M"]
+        return _year_ranges(cast(pd.Timestamp, start).year, now.year)
+
+    if start is None:
+        return ["MAX"]
+
+    months = _months_between(start, now)
+    # Annual series reject "30M"; their smallest window is 60M.
+    if freq != "A" and months <= 30:
+        return ["30M"]
+    if months <= 60:
+        return ["60M"]
+    return ["MAX"]
+
+
+def _year_ranges(first_year: int, last_year: int) -> list[str]:
+    """Calendar-year ``rango`` values from *first_year* to
+    *last_year* (inclusive)."""
+    return [str(year) for year in range(first_year, last_year + 1)]
+
+
+def _months_between(start: pd.Timestamp, now: pd.Timestamp) -> int:
+    """Whole months from *start* to *now*, plus a lag buffer."""
+    return (
+        (now.year - start.year) * 12
+        + (now.month - start.month)
+        + _RANGE_BUFFER_MONTHS
+    )
+
+
+_FIRST_YEAR_RE = re.compile(r"\[\d{1,2}\D\d{1,2}\D(\d{4})\]")
+
+
+def _first_observation_year(entry: _BdeSeriesEntry) -> int | None:
+    """Parse the first-observation year from a series' metadata.
+
+    The ``"Primer valor"`` entry looks like ``"[04/01/1999] ..."``
+    or ``"[18.07.1989] ..."`` -- the day/month separator varies
+    by series, so any non-digit separator is accepted.  Only
+    daily series (this helper's only caller) use that format.
+    """
+    for info in entry.get("informacion", []):
+        if info.get("titulo") == "Primer valor":
+            match = _FIRST_YEAR_RE.search(info.get("descripcion", ""))
+            if match:
+                return int(match.group(1))
+    return None
+
+
+def _parse_series_list(
+    series_list: list[_BdeSeriesEntry],
+) -> pd.DataFrame:
+    """Convert the JSON series list into a DataFrame."""
+    columns: dict[str, pd.Series[float]] = {}
+
+    for entry in series_list:
+        code = entry["serie"]
+        raw_dates = entry["fechas"]
+        raw_values = entry["valores"]
+
+        if not raw_dates:
+            continue
+
+        dates = pd.to_datetime(raw_dates, utc=True)
+        dates = dates.tz_localize(None).normalize()
+        values = [_to_float(v) for v in raw_values]
+
+        columns[code] = pd.Series(values, index=dates, name=code, dtype=float)
+
+    if not columns:
+        return pd.DataFrame()
+
+    df = pd.DataFrame(columns)
+    df.index.name = "date"
+    df.sort_index(inplace=True)
+    return df
+
+
+def _to_float(raw: int | float | None) -> float | None:
+    """Convert a raw API value to float, or None."""
+    if raw is None:
+        return None
+    try:
+        return float(raw)
+    except (ValueError, TypeError):
+        return None
+
+
+def _batches(items: list[str], size: int) -> list[list[str]]:
+    """Split a list into sublists of at most ``size``."""
+    return [items[i : i + size] for i in range(0, len(items), size)]

tesorotools_python-0.0.52/src/tesorotools/providers/bde.py

@@ -1,269 +0,0 @@
-"""Bank of Spain (Banco de Espana, BdE) data provider.
-
-Downloads historical time series from the BdE public REST API.
-No authentication required.
-
-Install with::
-
-    pip install tesorotools-python[bde]
-
-API reference
--------------
-Endpoint:
-    https://app.bde.es/bierest/resources/srdatosapp/listaSeries
-
-Query parameters (names are fixed by the BdE API):
-    ``idioma``  : ``"es"`` | ``"en"``
-    ``series``  : comma-separated series codes
-    ``rango``   : ``"MAX"`` | ``"30M"`` | ``"60M"``
-
-Notes
------
-- ECB DCF series (credit stock data) are also available
-  through this API because the BdE redistributes them.
-- The API accepts multiple series per request, but may
-  time out with too many.  Downloads are batched.
-- Timestamps include timezone info (UTC).  They are
-  normalized to tz-naive midnight for pandas compatibility.
-"""
-
-from __future__ import annotations
-
-import logging
-from typing import ClassVar, TypedDict, cast
-
-import pandas as pd
-import requests
-
-from tesorotools.providers.base import DataProvider
-
-logger = logging.getLogger(__name__)
-
-
-class _BdeSeriesEntry(TypedDict):
-    """Shape of a single series object in the BdE API
-    JSON response."""
-
-    serie: str
-    fechas: list[str]
-    valores: list[int | float | None]
-    decimales: int
-    codFrecuencia: str
-
-
-_LIST_URL = "https://app.bde.es/bierest/resources/srdatosapp/listaSeries"
-_PING_URL = "https://app.bde.es/bierest/resources/srdatosapp/favoritas"
-
-DEFAULT_TIMEOUT = 30
-BATCH_SIZE = 10
-
-
-class BdeProvider(DataProvider):
-    """Provider that downloads series from the Bank of Spain.
-
-    The BdE API is public and requires no credentials.
-
-    Parameters
-    ----------
-    language
-        Language for metadata.  ``"es"`` for Spanish,
-        ``"en"`` for English.
-    timeout
-        Maximum seconds to wait per HTTP request.
-    """
-
-    PROVIDER_NAME: ClassVar[str] = "bde"
-
-    def __init__(
-        self,
-        *,
-        language: str = "es",
-        timeout: int = DEFAULT_TIMEOUT,
-    ) -> None:
-        self._language = language
-        self._timeout = timeout
-
-    def fetch(
-        self,
-        codes: list[str],
-        start: str | None = None,
-        end: str | None = None,
-    ) -> pd.DataFrame:
-        """Download one or more series from the BdE.
-
-        Splits the series into batches of ``BATCH_SIZE``,
-        downloads each batch in a single HTTP request, and
-        merges the results into one DataFrame.
-
-        Parameters
-        ----------
-        codes
-            BdE series codes.
-        start
-            Earliest date to include (ISO format).
-            If ``None``, fetches the full history.
-        end
-            Ignored by this provider.  The BdE API always
-            returns data up to the latest observation.
-        """
-        if not codes:
-            return pd.DataFrame()
-
-        time_range = _compute_time_range(start)
-        frames: list[pd.DataFrame] = []
-
-        for batch in _batches(codes, BATCH_SIZE):
-            df = self._download_batch(batch, time_range)
-            if not df.empty:
-                frames.append(df)
-
-        if not frames:
-            return pd.DataFrame()
-
-        result = pd.concat(frames, axis=1)
-        result.sort_index(inplace=True)
-
-        if start:
-            cutoff = pd.Timestamp(start)
-            result = result.loc[result.index >= cutoff]
-
-        return result
-
-    def is_available(self) -> bool:
-        """Ping the BdE API to check connectivity."""
-        try:
-            resp = requests.get(
-                _PING_URL,
-                params={
-                    "idioma": self._language,
-                    "series": "DN_1TI2TIE42",
-                },
-                timeout=5,
-            )
-            return resp.status_code == 200
-        except requests.RequestException:
-            return False
-
-    def _download_batch(
-        self,
-        codes: list[str],
-        time_range: str,
-    ) -> pd.DataFrame:
-        params = {
-            "idioma": self._language,
-            "series": ",".join(codes),
-            "rango": time_range,
-        }
-        logger.info("BdE: downloading %d series...", len(codes))
-        resp = requests.get(
-            _LIST_URL,
-            params=params,
-            timeout=self._timeout,
-        )
-        resp.raise_for_status()
-
-        payload = resp.json()
-        _check_api_error(payload)
-        return _parse_series_list(cast(list[_BdeSeriesEntry], payload), codes)
-
-
-# ----------------------------------------------------------
-# Module-level helpers
-# ----------------------------------------------------------
-
-
-class _BdeErrorResponse(TypedDict, total=False):
-    errNum: int  # noqa: N815
-    errMsgUsr: str  # noqa: N815
-    errMsgDebug: str  # noqa: N815
-
-
-def _check_api_error(payload: object) -> None:
-    """Raise RuntimeError if the API returned an error."""
-    if not isinstance(payload, dict):
-        return
-    if "errNum" not in payload:
-        return
-    err = cast(_BdeErrorResponse, payload)
-    raise RuntimeError(
-        f"BdE API error {err.get('errNum', '?')}: {err.get('errMsgUsr', '')}"
-    )
-
-
-def _compute_time_range(start: str | None) -> str:
-    """Translate a start date into a BdE API range value.
-
-    The BdE API only accepts ``"30M"``, ``"60M"``, or
-    ``"MAX"`` for the ``rango`` parameter.  This function
-    picks the smallest value that covers the requested
-    start date.
-    """
-    if start is None:
-        return "MAX"
-
-    start_ts = pd.Timestamp(start)
-    now = pd.Timestamp.now()
-    months_needed = (
-        (now.year - start_ts.year) * 12
-        + (now.month - start_ts.month)
-        + 3  # buffer for publication lag
-    )
-
-    if months_needed <= 30:
-        return "30M"
-    if months_needed <= 60:
-        return "60M"
-    return "MAX"
-
-
-def _parse_series_list(
-    series_list: list[_BdeSeriesEntry],
-    requested: list[str],
-) -> pd.DataFrame:
-    """Convert the JSON series list into a DataFrame."""
-    columns: dict[str, pd.Series[float]] = {}
-
-    for entry in series_list:
-        code = entry["serie"]
-        raw_dates = entry["fechas"]
-        raw_values = entry["valores"]
-
-        if not raw_dates:
-            logger.warning("BdE: %s returned no data", code)
-            continue
-
-        dates = pd.to_datetime(raw_dates, utc=True)
-        dates = dates.tz_localize(None).normalize()
-        values = [_to_float(v) for v in raw_values]
-
-        columns[code] = pd.Series(values, index=dates, name=code, dtype=float)
-
-    missing = set(requested) - set(columns)
-    if missing:
-        logger.warning(
-            "BdE: series not found: %s",
-            ", ".join(sorted(missing)),
-        )
-
-    if not columns:
-        return pd.DataFrame()
-
-    df = pd.DataFrame(columns)
-    df.index.name = "date"
-    df.sort_index(inplace=True)
-    return df
-
-
-def _to_float(raw: int | float | None) -> float | None:
-    """Convert a raw API value to float, or None."""
-    if raw is None:
-        return None
-    try:
-        return float(raw)
-    except (ValueError, TypeError):
-        return None
-
-
-def _batches(items: list[str], size: int) -> list[list[str]]:
-    """Split a list into sublists of at most ``size``."""
-    return [items[i : i + size] for i in range(0, len(items), size)]