pse-opendata 0.2.0__py3-none-any.whl

pse_opendata/__init__.py

@@ -0,0 +1,110 @@
+"""pse-opendata: unofficial client for the public data of PSE, the Polish transmission system operator.
+
+Quick start:
+    import pse_opendata as pse
+    rce = pse.rce("2025-01-01", "2025-01-31")          # RCE price, 15-minute rows
+    sys = pse.get("his-wlk-cal", "2025-06-01", fields=["demand", "wi", "pv"])
+    demand = pse.history("demand", "2019-01-01", "2025-12-31")   # legacy export + API, hourly
+
+Module-level functions use a shared default :class:`Client` (default cache directory, see
+:mod:`pse_opendata.cache`); pass ``client=Client(...)`` to any of them to change the settings.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import pandas as pd
+
+from pse_opendata._version import __version__
+from pse_opendata.catalog import LEGACY_REPORTS, REPORTS, Field, LegacyReport, Report, get_legacy_report, get_report
+from pse_opendata.client import Client, DateLike, MonthLoad
+from pse_opendata.exceptions import (
+    CacheMissError,
+    LegacyFormatError,
+    PseApiError,
+    PseError,
+    UnknownFieldError,
+    UnknownReportError,
+)
+from pse_opendata.frames import TIME_COLUMNS, to_hourly
+from pse_opendata.stitch import splice_factor
+
+logging.getLogger("pse_opendata").addHandler(logging.NullHandler())
+
+__all__ = [
+    "LEGACY_REPORTS", "REPORTS", "TIME_COLUMNS", "CacheMissError", "Client", "Field", "LegacyFormatError",
+    "LegacyReport", "MonthLoad", "PseApiError", "PseError", "Report", "UnknownFieldError", "UnknownReportError",
+    "__version__", "balancing_capacity", "default_client", "fundamentals", "get", "get_legacy_report", "get_report",
+    "history", "imbalance_prices", "legacy", "rce", "reports", "sdac", "splice_factor", "to_hourly",
+]
+
+_default: Client | None = None
+
+
+def default_client() -> Client:
+    """The shared client used by the module-level functions (created on first use)."""
+    global _default
+    if _default is None:
+        _default = Client()
+    return _default
+
+
+def get(report: str, start: DateLike, end: DateLike | None = None, *, client: Client | None = None,
+        **kwargs: Any) -> pd.DataFrame:
+    """Data of a report of the reports API; see :meth:`Client.get`."""
+    return (client or default_client()).get(report, start, end, **kwargs)
+
+
+def legacy(code: str, start: DateLike, end: DateLike | None = None, *, client: Client | None = None,
+           **kwargs: Any) -> pd.DataFrame:
+    """Hourly data of the legacy CSV export; see :meth:`Client.legacy`."""
+    return (client or default_client()).legacy(code, start, end, **kwargs)
+
+
+def history(series: str, start: DateLike, end: DateLike | None = None, *, client: Client | None = None,
+            **kwargs: Any) -> pd.DataFrame:
+    """Hourly series stitched from the legacy export and the API; see :meth:`Client.history`."""
+    return (client or default_client()).history(series, start, end, **kwargs)
+
+
+def rce(start: DateLike, end: DateLike | None = None, *, client: Client | None = None, **kwargs: Any) -> pd.DataFrame:
+    """RCE price (``rce-pln``); see :meth:`Client.rce`."""
+    return (client or default_client()).rce(start, end, **kwargs)
+
+
+def sdac(start: DateLike, end: DateLike | None = None, *, client: Client | None = None, **kwargs: Any) -> pd.DataFrame:
+    """SDAC price for Poland (``csdac-pln``); see :meth:`Client.sdac`."""
+    return (client or default_client()).sdac(start, end, **kwargs)
+
+
+def fundamentals(start: DateLike, end: DateLike | None = None, *, client: Client | None = None,
+                 **kwargs: Any) -> pd.DataFrame:
+    """Demand, generation and exchange (``his-wlk-cal``); see :meth:`Client.fundamentals`."""
+    return (client or default_client()).fundamentals(start, end, **kwargs)
+
+
+def imbalance_prices(start: DateLike, end: DateLike | None = None, *, client: Client | None = None,
+                     **kwargs: Any) -> pd.DataFrame:
+    """Imbalance and balancing energy prices (``crb-rozl``); see :meth:`Client.imbalance_prices`."""
+    return (client or default_client()).imbalance_prices(start, end, **kwargs)
+
+
+def balancing_capacity(start: DateLike, end: DateLike | None = None, *, client: Client | None = None,
+                       **kwargs: Any) -> pd.DataFrame:
+    """Balancing capacity prices and volumes; see :meth:`Client.balancing_capacity`."""
+    return (client or default_client()).balancing_capacity(start, end, **kwargs)
+
+
+def reports(legacy: bool = False) -> pd.DataFrame:
+    """The catalogue as a frame: one row per report of the reports API (or of the legacy export)."""
+    if legacy:
+        rows = [{"code": r.code, "title": r.title, "title_pl": r.title_pl, "resolution": r.resolution,
+                 "first_date": r.first_date, "last_date": r.last_date, "fields": ", ".join(r.field_names),
+                 "page": r.page} for r in LEGACY_REPORTS.values()]
+    else:
+        rows = [{"name": r.name, "title": r.title, "title_pl": r.title_pl, "group": r.group,
+                 "resolution": r.resolution, "first_date": r.first_date, "revision_days": r.revision_days,
+                 "fields": ", ".join(r.field_names), "verified": r.verified, "url": r.url} for r in REPORTS.values()]
+    return pd.DataFrame(rows)

pse_opendata/__main__.py

@@ -0,0 +1,3 @@
+from pse_opendata.cli import main
+
+raise SystemExit(main())

pse_opendata/_http.py

@@ -0,0 +1,126 @@
+"""HTTP with retries, exponential backoff and a minimum pause between requests."""
+
+from __future__ import annotations
+
+import logging
+import time
+from collections.abc import Callable
+from typing import Any, TypeVar
+
+import requests
+
+from pse_opendata._version import __version__
+from pse_opendata.exceptions import PseApiError, PseError
+
+log = logging.getLogger("pse_opendata")
+
+USER_AGENT = f"pse-opendata/{__version__} (+https://github.com/kwwaleszczynski-del/pse-opendata)"
+# network failures worth another attempt; other RequestException subclasses (bad URL, ...) are raised at once
+RETRY_ERRORS = (requests.ConnectionError, requests.Timeout, requests.exceptions.ChunkedEncodingError,
+                requests.exceptions.ContentDecodingError)
+T = TypeVar("T")
+
+
+def _retry_after(response: requests.Response) -> float | None:
+    value = response.headers.get("Retry-After")
+    try:
+        return float(value) if value is not None else None
+    except ValueError:
+        return None
+
+
+def _error_message(response: requests.Response) -> tuple[str, str | None]:
+    """Message and code of an error response; the reports API answers ``{"error": {"code", "message"}}``."""
+    try:
+        err = response.json().get("error", {})
+        return str(err.get("message") or response.reason), err.get("code")
+    except (ValueError, AttributeError):
+        return f"{response.reason or 'HTTP error'}", None
+
+
+class HttpClient:
+    """GET requests with retries on network errors (also a connection dropped mid-body), HTTP 429 and 5xx.
+
+    Waits ``backoff * 2**attempt`` seconds (at most ``max_backoff``) between attempts, or what ``Retry-After`` asks
+    for. ``min_interval`` is the pause between the end of one request and the start of the next one.
+    ``sleep`` and ``clock`` can be replaced in tests.
+    """
+
+    def __init__(
+        self,
+        session: requests.Session | None = None,
+        *,
+        timeout: float = 60.0,
+        retries: int = 4,
+        backoff: float = 2.0,
+        max_backoff: float = 120.0,
+        min_interval: float = 0.0,
+        user_agent: str = USER_AGENT,
+        sleep: Callable[[float], None] = time.sleep,
+        clock: Callable[[], float] = time.monotonic,
+    ):
+        self.session = session if session is not None else requests.Session()
+        self.timeout = timeout
+        self.retries = retries
+        self.backoff = backoff
+        self.max_backoff = max_backoff
+        self.min_interval = min_interval
+        self.user_agent = user_agent
+        self._sleep = sleep
+        self._clock = clock
+        self._last_done: float | None = None
+
+    def _wait_for_slot(self) -> None:
+        if self._last_done is None or self.min_interval <= 0:
+            return
+        wait = self._last_done + self.min_interval - self._clock()
+        if wait > 0:
+            self._sleep(wait)
+
+    def _delay(self, attempt: int, response: requests.Response | None) -> float:
+        hinted = _retry_after(response) if response is not None else None
+        if hinted is not None:
+            return min(hinted, self.max_backoff)
+        return min(self.backoff * 2**attempt, self.max_backoff)
+
+    def get(self, url: str, params: dict[str, Any] | None = None,
+            decode: Callable[[requests.Response], T] | None = None) -> Any:
+        """GET ``url`` and return ``decode(response)`` (the response itself without ``decode``).
+
+        A plain ``ValueError`` from ``decode`` (e.g. truncated JSON) is retried like a network error; errors of
+        this package raised by ``decode`` are not. Raises :class:`PseApiError` when the request finally fails or the
+        server answers a non-retryable error.
+        """
+        last_error: PseApiError | None = None
+        for attempt in range(self.retries + 1):
+            self._wait_for_slot()
+            response = None
+            try:
+                response = self.session.get(url, params=params, timeout=self.timeout,
+                                            headers={"User-Agent": self.user_agent})
+            except RETRY_ERRORS as e:
+                last_error = PseApiError(f"request failed: {e}", url=url)
+            except requests.RequestException as e:
+                raise PseApiError(f"request failed: {e}", url=url) from e
+            finally:
+                self._last_done = self._clock()
+            if response is not None:
+                status = response.status_code
+                if status >= 400:
+                    message, code = _error_message(response)
+                    last_error = PseApiError(f"HTTP {status} from {url}: {message}", status=status, code=code, url=url)
+                    if status != 429 and status < 500:
+                        raise last_error
+                else:
+                    try:
+                        return decode(response) if decode is not None else response
+                    except PseError:
+                        raise
+                    except ValueError as e:
+                        last_error = PseApiError(f"invalid response from {url}: {e}", status=status, url=url)
+            if attempt < self.retries:
+                delay = self._delay(attempt, response)
+                log.warning("%s; retry %d/%d in %.0f s", last_error, attempt + 1, self.retries, delay)
+                self._sleep(delay)
+        assert last_error is not None
+        raise last_error

pse_opendata/_version.py

@@ -0,0 +1 @@
+__version__ = "0.2.0"

pse_opendata/cache.py

@@ -0,0 +1,108 @@
+"""Monthly Parquet cache.
+
+One file per report and calendar month (business dates): ``<cache_dir>/<source>/<report>/<YYYY-MM>.parquet``,
+with ``source`` = ``api`` or ``legacy``. Each file stores when it was downloaded, which decides whether the month is
+final (see :attr:`pse_opendata.catalog.Report.revision_days`). Files are written atomically.
+
+The default directory is ``$PSE_OPENDATA_CACHE`` if set, otherwise the user cache directory:
+``%LOCALAPPDATA%\\pse-opendata\\cache`` on Windows, ``~/Library/Caches/pse-opendata`` on macOS and
+``$XDG_CACHE_HOME/pse-opendata`` (``~/.cache/pse-opendata``) elsewhere.
+
+Usage:
+    cache = MonthCache(default_cache_dir())
+    cache.write("api", "rce-pln", date(2025, 1, 1), frame, fetched_utc=now)
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from dataclasses import dataclass
+from datetime import date, datetime
+from pathlib import Path
+
+import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from pse_opendata._version import __version__
+
+META_KEY = b"pse_opendata"
+SOURCES = ("api", "legacy")
+DATETIME_COLUMNS = ["start_utc", "end_utc", "start_local", "end_local"]
+
+
+def default_cache_dir() -> Path:
+    env = os.environ.get("PSE_OPENDATA_CACHE")
+    if env:
+        return Path(env)
+    if sys.platform == "win32":
+        base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local"))
+        return base / "pse-opendata" / "cache"
+    if sys.platform == "darwin":
+        return Path.home() / "Library" / "Caches" / "pse-opendata"
+    return Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "pse-opendata"
+
+
+@dataclass(frozen=True)
+class CachedMonth:
+    frame: pd.DataFrame
+    fetched_utc: datetime
+    first_day: date
+    last_day: date
+
+
+class MonthCache:
+    def __init__(self, root: Path | str):
+        self.root = Path(root)
+
+    def path(self, source: str, report: str, month: date) -> Path:
+        return self.root / source / report / f"{month:%Y-%m}.parquet"
+
+    def read(self, source: str, report: str, month: date) -> CachedMonth | None:
+        """The cached month, or ``None`` when absent or unreadable."""
+        path = self.path(source, report, month)
+        if not path.exists():
+            return None
+        try:
+            table = pq.read_table(path)
+            meta = json.loads((table.schema.metadata or {})[META_KEY])
+            fetched_utc = datetime.fromisoformat(meta["fetched_utc"])
+            first_day = date.fromisoformat(meta["first_day"])
+            last_day = date.fromisoformat(meta["last_day"])
+            frame = table.to_pandas()
+        except (OSError, KeyError, TypeError, ValueError, pa.ArrowException):
+            return None
+        for col in DATETIME_COLUMNS:
+            if col in frame:
+                frame[col] = frame[col].dt.as_unit("ns")
+        return CachedMonth(frame=frame, fetched_utc=fetched_utc, first_day=first_day, last_day=last_day)
+
+    def write(self, source: str, report: str, month: date, frame: pd.DataFrame, *, fetched_utc: datetime,
+              first_day: date, last_day: date) -> Path:
+        """Store the frame of one month; ``first_day``..``last_day`` are the business dates it covers."""
+        path = self.path(source, report, month)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        table = pa.Table.from_pandas(frame, preserve_index=False)
+        meta = {"fetched_utc": fetched_utc.isoformat(), "first_day": first_day.isoformat(),
+                "last_day": last_day.isoformat(), "report": report, "version": __version__}
+        table = table.replace_schema_metadata({**(table.schema.metadata or {}), META_KEY: json.dumps(meta)})
+        tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp")
+        pq.write_table(table, tmp)
+        os.replace(tmp, path)
+        return path
+
+    def clear(self, source: str | None = None, report: str | None = None) -> int:
+        """Delete cached files (all, one source, or one report of a source); returns how many were deleted.
+
+        Only the ``api`` and ``legacy`` sub-folders are touched, so other files in the directory are safe.
+        """
+        sources = [source] if source else list(SOURCES)
+        files = []
+        for src in sources:
+            base = self.root / src / report if report else self.root / src
+            files += [f for f in base.glob("**/*.parquet") if f.is_file()] if base.exists() else []
+        for f in files:
+            f.unlink()
+        return len(files)