pyecos 0.1.0__py3-none-any.whl

pyecos/__init__.py

@@ -0,0 +1,56 @@
+"""pyecos -- a Python client for the Bank of Korea ECOS Open API."""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+
+from . import catalog
+from .client import ECOS
+from .curation import Indicator, IndicatorSpec
+from .exceptions import (
+    ECOSAuthError,
+    ECOSConfigError,
+    ECOSError,
+    ECOSNetworkError,
+    ECOSRateLimitError,
+    ECOSResponseError,
+)
+from .types import (
+    CatalogRow,
+    Cycle,
+    ItemRow,
+    KeyStatRow,
+    Language,
+    MetaRow,
+    StatRow,
+    TableRow,
+    WordRow,
+)
+
+try:
+    __version__ = version("pyecos")
+except PackageNotFoundError:  # running from a source tree without an install
+    __version__ = "0.0.0"
+
+__all__ = [
+    "ECOS",
+    "catalog",
+    "Indicator",
+    "IndicatorSpec",
+    "Cycle",
+    "Language",
+    "StatRow",
+    "CatalogRow",
+    "TableRow",
+    "ItemRow",
+    "KeyStatRow",
+    "WordRow",
+    "MetaRow",
+    "ECOSError",
+    "ECOSConfigError",
+    "ECOSAuthError",
+    "ECOSResponseError",
+    "ECOSRateLimitError",
+    "ECOSNetworkError",
+    "__version__",
+]

pyecos/__main__.py

@@ -0,0 +1,8 @@
+"""Run the pyecos CLI, so ``python -m pyecos`` matches the ``pyecos`` console script.
+
+Importing this module runs the CLI and terminates the process via ``SystemExit``.
+"""
+
+from .cli import main
+
+raise SystemExit(main())

pyecos/_cache.py

@@ -0,0 +1,75 @@
+"""An opt-in, TTL-bounded store of prior fetch results, keyed by the logical query.
+
+Off unless :class:`ECOS` is given a ``cache_ttl``. When on, a repeated query returns
+the stored rows without a network round trip -- cutting calls so a burst stays under
+the ECOS rate cap (~300 in three minutes), and repeating work for free.
+
+The store holds *complete* results (a whole paginated, mapped series), never a single
+page: a page is a partial intermediate, and caching answers rather than ingredients is
+what keeps a cache honest. Entries expire after ``ttl`` seconds, so the staleness a
+caller accepts is exactly the bound they chose -- a series whose latest observation has
+since updated is re-fetched once its entry expires. Least-recently-used entries are
+evicted past ``maxsize`` so the store cannot grow without bound.
+
+Each entry is copied on store and on retrieval, row by row, so a caller mutating a
+returned row cannot corrupt the stored entry or another caller's copy; the rows carry
+only scalar values, so a per-row ``dict`` copy fully isolates them.
+
+Not thread-safe: the entries are mutable instance state with no lock, like the client
+that owns it -- use one client per thread.
+"""
+
+from __future__ import annotations
+
+import time
+from collections import OrderedDict
+from typing import Any
+
+# (service, resolved language, request tail) -- the logical query, minus the paging
+# window and the API key, so kr and en cache apart and two clients never collide.
+_CacheKey = tuple[str, str, tuple[str, ...]]
+
+_Rows = list[dict[str, Any]]
+
+_DEFAULT_MAXSIZE = 256
+
+
+def _isolate_rows(rows: _Rows) -> _Rows:
+    """A copy of ``rows`` sharing none of its dicts (values are scalars)."""
+    return [dict(row) for row in rows]
+
+
+class _Cache:
+    """A TTL + LRU store mapping a query key to its rows."""
+
+    def __init__(self, *, ttl: float, maxsize: int = _DEFAULT_MAXSIZE) -> None:
+        self._ttl = ttl
+        self._maxsize = maxsize
+        self._entries: OrderedDict[_CacheKey, tuple[float, _Rows]] = OrderedDict()
+
+    def get(self, key: _CacheKey) -> _Rows | None:
+        """The cached rows for ``key`` if present and unexpired, else ``None``.
+
+        Returns an isolated copy, so a caller mutating the rows cannot corrupt the
+        entry.
+        """
+        entry = self._entries.get(key)
+        if entry is None:
+            return None
+        expires_at, rows = entry
+        if time.monotonic() >= expires_at:
+            del self._entries[key]
+            return None
+        self._entries.move_to_end(key)  # mark most-recently-used
+        return _isolate_rows(rows)
+
+    def set(self, key: _CacheKey, rows: _Rows) -> None:
+        """Store an isolated copy of ``rows``, evicting the LRU entry past maxsize."""
+        self._entries[key] = (time.monotonic() + self._ttl, _isolate_rows(rows))
+        self._entries.move_to_end(key)
+        while len(self._entries) > self._maxsize:
+            self._entries.popitem(last=False)  # drop the least-recently-used
+
+    def clear(self) -> None:
+        """Drop every entry."""
+        self._entries.clear()

pyecos/_config.py

@@ -0,0 +1,63 @@
+"""Resolve the ECOS API key from the caller, the environment, or the config file.
+
+The key is looked up in a fixed order, so an explicit value always wins and a set
+environment variable beats a file on disk:
+
+1. the ``api_key`` passed to ``ECOS(...)``
+2. the ``ECOS_API_KEY`` environment variable
+3. ``"ECOS_API_KEY"`` in ``$XDG_CONFIG_HOME/pyecos/credentials.json``
+   (``$XDG_CONFIG_HOME`` defaults to ``~/.config``)
+
+The file is optional -- its absence just means "no key here." But a file that is
+present and unreadable, not JSON, or not a JSON object is an error, because a caller
+who wrote one meant it to be used and a silent skip would hide the mistake.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+from .exceptions import ECOSConfigError
+
+_ENV_VAR = "ECOS_API_KEY"
+_CONFIG_DIR = "pyecos"
+_CONFIG_FILE = "credentials.json"
+
+
+def resolve_api_key(explicit: str | None) -> str:
+    """Return the first key found across the three sources, or raise if none exists."""
+    key = explicit or os.environ.get(_ENV_VAR) or _key_from_file()
+    if not key:
+        raise ECOSConfigError(
+            f"no ECOS API key: pass api_key=, set the {_ENV_VAR} environment "
+            f"variable, or put it in {credentials_path()}"
+        )
+    return key
+
+
+def credentials_path() -> Path:
+    """The path pyecos reads a stored key from (honoring ``$XDG_CONFIG_HOME``)."""
+    config_home = os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config")
+    return Path(config_home) / _CONFIG_DIR / _CONFIG_FILE
+
+
+def _key_from_file() -> str | None:
+    path = credentials_path()
+    try:
+        text = path.read_text(encoding="utf-8")
+    except FileNotFoundError:
+        return None
+    except OSError as err:
+        raise ECOSConfigError(f"could not read {path}: {err}") from err
+
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError as err:
+        raise ECOSConfigError(f"{path} is not valid JSON: {err}") from err
+    if not isinstance(data, dict):
+        raise ECOSConfigError(f"{path} must contain a JSON object")
+
+    key = data.get(_ENV_VAR)
+    return key if isinstance(key, str) and key else None

pyecos/_parse.py

@@ -0,0 +1,110 @@
+"""Turn ECOS's paged, upper-cased rows into a full list of snake_cased dicts.
+
+Two jobs live here, both above a single wire request (``_transport``):
+
+* **paginate** -- ECOS caps a response at :data:`PAGE_SIZE` rows, so walk the
+  ``start_row``/``end_row`` window until ``list_total_count`` is exhausted and
+  return every row, never a first page silently mistaken for the whole answer.
+* **map** -- rename the vendor's ``UPPER_CASE`` keys to the snake_case fields the
+  ``TypedDict``s document, and parse the one genuinely numeric field
+  (``DATA_VALUE``) to ``float``. Every key the vendor sends is kept; an
+  unrecognized one simply lowercases and passes through.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ._transport import PAGE_SIZE, _Transport
+from .exceptions import ECOSResponseError
+
+# No real ECOS series approaches a million rows; the cap turns a broken server
+# that returns full pages forever into a clear error instead of an endless loop.
+_MAX_PAGES = 10_000
+
+# Vendor keys whose snake_case is not a plain lowercase of the original.
+_FIELD_BY_VENDOR_KEY = {
+    "P_STAT_CODE":  "parent_stat_code",
+    "P_ITEM_CODE":  "parent_item_code",
+    "P_ITEM_NAME":  "parent_item_name",
+    "GRP_CODE":     "group_code",
+    "GRP_NAME":     "group_name",
+    "ORG_NAME":     "org_name",
+    "SRCH_YN":      "searchable",
+    "WGT":          "weight",
+    "DATA_CNT":     "data_count",
+    "DATA_VALUE":   "data_value",
+    "KEYSTAT_NAME": "keystat_name",
+    "CLASS_NAME":   "class_name",
+    "LVL":          "level",
+    "P_CONT_CODE":  "parent_content_code",
+    "CONT_CODE":    "content_code",
+    "CONT_NAME":    "content_name",
+}
+
+
+def collect(
+    transport: _Transport,
+    *,
+    service: str,
+    api_key: str,
+    lang: str,
+    tail: list[str],
+) -> list[dict[str, Any]]:
+    """Fetch every page of a service call and return the mapped rows."""
+    return [_map_row(row) for row in _paginate(
+        transport, service=service, api_key=api_key, lang=lang, tail=tail,
+    )]
+
+
+def _paginate(
+    transport: _Transport,
+    *,
+    service: str,
+    api_key: str,
+    lang: str,
+    tail: list[str],
+) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    start_row = 1
+    for _ in range(_MAX_PAGES):
+        page = transport.request_page(
+            service=service,
+            api_key=api_key,
+            lang=lang,
+            start_row=start_row,
+            end_row=start_row + PAGE_SIZE - 1,
+            tail=tail,
+        )
+        batch = page.get("row") or []
+        rows.extend(batch)
+        try:
+            total = int(page.get("list_total_count") or 0)
+        except (TypeError, ValueError):
+            # A garbage total must not truncate the series: push it past what we
+            # have so the loop keeps paging and the empty-batch guard ends it.
+            total = len(rows) + PAGE_SIZE
+        if not batch or len(rows) >= total:
+            return rows
+        start_row += PAGE_SIZE
+    raise ECOSResponseError("UNKNOWN", f"pagination exceeded {_MAX_PAGES} pages")
+
+
+def _map_row(raw: dict[str, Any]) -> dict[str, Any]:
+    row = {_FIELD_BY_VENDOR_KEY.get(key, key.lower()): value
+           for key, value in raw.items()}
+    if "data_value" in row:
+        row["data_value"] = _to_float(row["data_value"])
+    return row
+
+
+def _to_float(text: Any) -> float | None:
+    # ECOS marks a missing observation with an empty string or a lone dash; any
+    # other unparseable value also becomes None (the vendor sends plain numbers,
+    # so this only fires on genuinely malformed data, reported as missing).
+    if text in (None, "", "-"):
+        return None
+    try:
+        return float(text)
+    except (TypeError, ValueError):
+        return None

pyecos/_transport.py

@@ -0,0 +1,176 @@
+"""One request over the wire: build the ECOS path, GET it, surface errors.
+
+ECOS is a path-positional API -- every argument is a slash segment in a fixed
+order, up to fifteen of them:
+
+    /api/{service}/{key}/{format}/{lang}/{start_row}/{end_row}/{tail...}
+
+This module is the only place that order is written down. ``_Transport`` holds the
+HTTP client and the pacing clock: it spaces consecutive requests (``delay_seconds``)
+and retries a transient failure (timeout, connection reset, 5xx) with backoff, but a
+rate limit (ERROR-602) is an answer to respect, raised straight through.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from typing import Any
+from urllib.parse import quote
+
+import httpx
+
+from .exceptions import (
+    ECOSAuthError,
+    ECOSNetworkError,
+    ECOSRateLimitError,
+    ECOSResponseError,
+)
+
+BASE_URL = "https://ecos.bok.or.kr/api"
+
+# ECOS serves at most this many rows per request; the parser pages past it.
+PAGE_SIZE = 100
+
+# The vendor's rate-limit code -- in the RESULT body, and mirrored for an HTTP 429.
+_RATE_LIMIT_CODE = "ERROR-602"
+
+# A transient failure (timeout, reset, 5xx) is a glitch worth retrying; a rate
+# limit is not (see _Transport.request_page). This counts total attempts, not
+# retries -- 3 is one try plus two retries.
+_MAX_ATTEMPTS = 3
+_RETRY_BACKOFF_SECONDS = 1.0
+_RETRY_BACKOFF_FACTOR = 2  # each retry waits this many times the last
+
+
+class _Transport:
+    """The HTTP client plus its pacing clock -- one per :class:`ECOS`.
+
+    ``delay_seconds`` spaces consecutive requests so a burst (a long paginated
+    series, or many indicators in a loop) stays under the ECOS rate cap of ~300
+    calls in three minutes; the default is 0 because a handful of calls never
+    reaches it, and pacing every page would only slow the common case. A bulk
+    caller sets it (0.6s keeps one client under the cap indefinitely).
+
+    Not thread-safe: the pacing clock (``_next_request_at``) is shared mutable
+    state, so use one client -- hence one transport -- per thread.
+    """
+
+    def __init__(
+        self,
+        client: httpx.Client,
+        *,
+        delay_seconds: float = 0.0,
+        max_attempts: int = _MAX_ATTEMPTS,
+    ) -> None:
+        self._client = client
+        self._delay_seconds = delay_seconds
+        self._max_attempts = max_attempts
+        self._next_request_at = 0.0
+
+    def request_page(
+        self,
+        *,
+        service: str,
+        api_key: str,
+        lang: str,
+        start_row: int,
+        end_row: int,
+        tail: list[str],
+    ) -> dict[str, Any]:
+        """Fetch one page and return its ``{list_total_count, row}`` body.
+
+        Retries a transient transport failure (timeout, connection reset, 5xx) with
+        backoff. Raises :class:`ECOSNetworkError` if it never completes,
+        :class:`ECOSRateLimitError` on a rate limit (ERROR-602, not retried),
+        :class:`ECOSAuthError` on a rejected key, and :class:`ECOSResponseError` on
+        any other vendor error. A "no data" response (INFO-200) returns as empty.
+        """
+        url = _build_url(service, api_key, lang, start_row, end_row, tail)
+        last_error: ECOSNetworkError | None = None
+        last_cause: Exception | None = None
+        for attempt in range(self._max_attempts):
+            self._wait_for_next_slot()
+            try:
+                response = self._client.get(url)
+                response.raise_for_status()
+                payload = response.json()
+            except httpx.HTTPStatusError as err:
+                status = err.response.status_code
+                if status == 429:  # an HTTP-level rate limit, should ECOS send one
+                    raise ECOSRateLimitError(_RATE_LIMIT_CODE, str(err)) from err
+                if status < 500:  # any other 4xx is the server's answer
+                    raise ECOSNetworkError(str(err)) from err
+                last_error, last_cause = ECOSNetworkError(str(err)), err  # 5xx -- retry
+            except httpx.HTTPError as err:  # timeout, connection reset, ...
+                last_error, last_cause = ECOSNetworkError(str(err)), err
+            except json.JSONDecodeError as err:
+                # A 200 whose body is not JSON (a proxy/maintenance HTML page) must
+                # surface through the ECOSError hierarchy, not as a raw decode error.
+                raise ECOSResponseError(
+                    "UNKNOWN", f"non-JSON response from ECOS: {err}") from err
+            else:
+                return _extract_body(payload, service)
+            if attempt + 1 < self._max_attempts:
+                time.sleep(_RETRY_BACKOFF_SECONDS * _RETRY_BACKOFF_FACTOR**attempt)
+        if last_error is not None:
+            # Chain the httpx cause the ECOSNetworkError docstring promises.
+            raise last_error from last_cause
+        raise ECOSNetworkError("request failed")
+
+    def _wait_for_next_slot(self) -> None:
+        if self._delay_seconds <= 0:
+            return
+        now = time.monotonic()
+        if now < self._next_request_at:
+            time.sleep(self._next_request_at - now)
+        self._next_request_at = time.monotonic() + self._delay_seconds
+
+
+def _build_url(
+    service: str,
+    api_key: str,
+    lang: str,
+    start_row: int,
+    end_row: int,
+    tail: list[str],
+) -> str:
+    segments = [service, api_key, "json", lang, str(start_row), str(end_row), *tail]
+    while segments and segments[-1] == "":  # trailing optional args ECOS omits
+        segments.pop()
+    if "" in segments:
+        # A remaining empty is an interior positional gap (end without start, or
+        # item_code2 without item_code1); it would shift every later argument.
+        raise ValueError(
+            "cannot build a request with a gap between positional arguments"
+        )
+    path = "/".join(quote(segment, safe="") for segment in segments)
+    return f"{BASE_URL}/{path}"
+
+
+def _extract_body(payload: Any, service: str) -> dict[str, Any]:
+    if not isinstance(payload, dict):
+        raise ECOSResponseError("UNKNOWN", f"unexpected ECOS response: {payload!r}")
+    if service in payload:
+        body = payload[service]
+        if not isinstance(body, dict):  # a non-object under the service key is bad
+            raise ECOSResponseError("UNKNOWN", f"unexpected ECOS response: {payload!r}")
+        if "row" not in body and "list_total_count" not in body:
+            # A dict under the service key that carries no page fields (e.g. a
+            # nested RESULT error) must surface, not read as an empty series.
+            raise ECOSResponseError("UNKNOWN", f"unexpected ECOS response: {payload!r}")
+        return body
+
+    result = payload.get("RESULT")
+    if not isinstance(result, dict):
+        raise ECOSResponseError("UNKNOWN", f"unexpected ECOS response: {payload!r}")
+
+    code = result.get("CODE", "UNKNOWN")
+    message = result.get("MESSAGE", "")
+    if code == "INFO-200":  # no matching data -- an empty result, not a failure
+        return {"list_total_count": "0", "row": []}
+    if code == "INFO-100":  # invalid authentication key
+        raise ECOSAuthError(message or "invalid ECOS API key")
+    if code == _RATE_LIMIT_CODE:  # too many calls -- rate limited, back off
+        raise ECOSRateLimitError(code, message)
+    raise ECOSResponseError(code, message)

pyecos/catalog.py

@@ -0,0 +1,81 @@
+"""Offline search over a bundled snapshot of the ECOS table catalog.
+
+The hard part of using ECOS is finding the stat code you need -- and
+:meth:`~pyecos.ECOS.fetch_tables` costs a network call every time. This module
+ships a snapshot of the table hierarchy inside the package, so you can search it
+by name or code with **no key and no network**::
+
+    from pyecos import catalog
+
+    for table in catalog.search("소비자물가"):
+        print(table["stat_code"], table["stat_name"])   # 901Y009 소비자물가지수
+
+    catalog.table("901Y009")   # one table's row, or None
+
+The snapshot is a point-in-time copy (refresh it with ``tools/gen_catalog.py``);
+for the live hierarchy, or a table's detail items, use ``ECOS.fetch_tables`` /
+``ECOS.fetch_items``.
+"""
+
+from __future__ import annotations
+
+import csv
+import gzip
+import io
+from collections.abc import Iterable
+from functools import cache
+from importlib.resources import files
+from typing import cast
+
+from .types import CatalogRow
+
+_DATA = files("pyecos").joinpath("data", "catalog.tsv.gz")
+
+
+@cache
+def _tables() -> tuple[CatalogRow, ...]:
+    text = gzip.decompress(_DATA.read_bytes()).decode("utf-8")
+    return tuple(
+        CatalogRow(
+            stat_code=row["stat_code"],
+            stat_name=row["stat_name"],
+            cycle=row["cycle"],
+            searchable=row["searchable"] == "Y",
+        )
+        for row in csv.DictReader(io.StringIO(text), delimiter="\t")
+    )
+
+
+def _copies(rows: Iterable[CatalogRow]) -> list[CatalogRow]:
+    # Hand out fresh row dicts so a caller mutating a result can't corrupt the
+    # shared snapshot -- the same isolation _Cache applies to fetched rows.
+    return [cast("CatalogRow", dict(row)) for row in rows]
+
+
+def tables() -> list[CatalogRow]:
+    """Every table in the bundled catalog snapshot."""
+    return _copies(_tables())
+
+
+def table(stat_code: str) -> CatalogRow | None:
+    """The catalog row for ``stat_code``, or ``None`` if the snapshot lacks it."""
+    row = next((row for row in _tables() if row["stat_code"] == stat_code), None)
+    return cast("CatalogRow", dict(row)) if row is not None else None
+
+
+def search(query: str, *, searchable_only: bool = True) -> list[CatalogRow]:
+    """Tables whose code or name contains ``query``, case-insensitively, offline.
+
+    ``searchable_only`` (the default) keeps only tables you can actually query
+    with :meth:`~pyecos.ECOS.fetch_series`, dropping the category headers that
+    organize the hierarchy; pass ``False`` to search those too.
+    """
+    needle = query.strip().lower()
+    hits = [
+        row
+        for row in _tables()
+        if needle in row["stat_code"].lower() or needle in row["stat_name"].lower()
+    ]
+    if searchable_only:
+        hits = [row for row in hits if row["searchable"]]
+    return _copies(hits)