undata 0.1.0__py3-none-any.whl

undata/__init__.py

@@ -0,0 +1,86 @@
+"""undata: UN data in Python, without the guessing.
+
+A friendly client for the UN System Data Commons (data.un.org): statistics from 26
+UN System entities, no API key needed. Unofficial, and part of Nerd Lab
+(https://github.com/nerd-factory). Not affiliated with or endorsed by the United
+Nations.
+
+Start here::
+
+    import undata as un
+
+    un.search("maternal mortality")            # find an indicator
+    un.about("undata/sdg/SH_STA_MORT.SEX--F")   # what it is and how much data exists
+    table = un.get("undata/sdg/SH_STA_MORT.SEX--F", ["Bangladesh", "Nepal"])
+    un.latest("undata/sdg/EG_ACS_ELEC", "Western Africa")
+    un.sdg.series("16.1.2")                     # from an SDG indicator to its data
+    un.save(table, "maternal-mortality.xlsx")   # data plus a sheet of citations
+
+Every function has a docstring with an example: ``help(un.get)``.
+"""
+
+import warnings
+
+from . import places, sdg
+from ._http import DEFAULT_BASE_URL, Transport
+from ._session import configure
+from ._version import __version__
+from .ask import Answer, ask
+from .data import get, latest
+from .dcid import build_dcid, child_countries_of, describe_dcid, parse_dcid
+from .errors import (
+    AmbiguousPlaceError,
+    APIError,
+    BlockedRequestError,
+    ConnectionFailedError,
+    IndicatorNotFoundError,
+    MissingDataWarning,
+    MixedYearsWarning,
+    PlaceGuessedWarning,
+    PlaceNotFoundError,
+    UNDataError,
+    UNDataWarning,
+)
+from .export import cite, save
+from .indicators import IndicatorInfo, about, search
+from .places import Place, countries, regions
+
+# Each warning is about the table just returned, so show it every time rather than
+# only the first time Python sees it.
+warnings.filterwarnings("always", category=UNDataWarning)
+
+__all__ = [
+    "DEFAULT_BASE_URL",
+    "APIError",
+    "AmbiguousPlaceError",
+    "Answer",
+    "BlockedRequestError",
+    "ConnectionFailedError",
+    "IndicatorInfo",
+    "IndicatorNotFoundError",
+    "MissingDataWarning",
+    "MixedYearsWarning",
+    "Place",
+    "PlaceGuessedWarning",
+    "PlaceNotFoundError",
+    "Transport",
+    "UNDataError",
+    "UNDataWarning",
+    "__version__",
+    "about",
+    "ask",
+    "build_dcid",
+    "child_countries_of",
+    "cite",
+    "configure",
+    "countries",
+    "describe_dcid",
+    "get",
+    "latest",
+    "parse_dcid",
+    "places",
+    "regions",
+    "save",
+    "sdg",
+    "search",
+]

undata/_endpoints.py

@@ -0,0 +1,167 @@
+"""One thin wrapper per API route. No interpretation happens here.
+
+These return the API's JSON as-is; the public modules built on top of them are
+where missing data, provenance and place resolution get handled. Keeping this
+layer dumb means a change on the platform shows up in exactly one place.
+
+The routes split across two surfaces:
+
+* ``/core/api/v2/*`` is the Data Commons REST v2 API (graph, observations, resolve).
+* ``/api/*`` is the website's own API, which carries search and variable metadata
+  that REST v2 does not expose.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any, Final
+
+from ._http import Transport
+
+#: Fields requested from the observation endpoint. Without ``select`` it returns
+#: an empty shell, and with ``select`` in a POST body the firewall answers 403,
+#: which is why :func:`observation` is GET-only.
+OBSERVATION_FIELDS: Final = ("date", "value", "variable", "entity", "facet")
+
+#: Graph walks return at most this many values per page and set ``nextToken``.
+NODE_PAGE_SIZE: Final = 500
+
+#: Upper bound on pages followed for one graph walk; protects against a cycle.
+MAX_NODE_PAGES: Final = 200
+
+JSON = dict[str, Any]
+
+
+def _as_list(value: str | Sequence[str]) -> list[str]:
+    return [value] if isinstance(value, str) else list(value)
+
+
+def observation(
+    transport: Transport,
+    variables: str | Sequence[str],
+    *,
+    entities: str | Sequence[str] | None = None,
+    entity_expression: str | None = None,
+    date: str = "LATEST",
+) -> JSON:
+    """Fetch observations from ``/core/api/v2/observation``, always by GET.
+
+    Pass either explicit ``entities`` or an ``entity_expression`` such as
+    ``africa<-containedInPlace+{typeOf:Country}``. ``date`` is ``"LATEST"``, ``""``
+    for every date, or a single date like ``"2020"``. Note that ``"LATEST"`` means
+    each entity's own most recent date, which differs between countries.
+    """
+    if (entities is None) == (entity_expression is None):
+        raise ValueError("Pass exactly one of entities or entity_expression.")
+    result: JSON = transport.get(
+        "/core/api/v2/observation",
+        {
+            "date": date,
+            "variable.dcids": _as_list(variables),
+            "entity.dcids": None if entities is None else _as_list(entities),
+            "entity.expression": entity_expression,
+            "select": OBSERVATION_FIELDS,
+        },
+    )
+    return result
+
+
+def node(transport: Transport, nodes: str | Sequence[str], prop: str) -> JSON:
+    """Follow a relation expression from ``/core/api/v2/node``, across every page.
+
+    The API caps each page at 500 values and returns a ``nextToken``; this follows
+    the tokens and merges the pages, so the result is the complete answer.
+    """
+    params: dict[str, str | list[str] | None] = {"nodes": _as_list(nodes), "property": prop}
+    merged: JSON = transport.get("/core/api/v2/node", params)
+    token = merged.pop("nextToken", None)
+    pages = 1
+    while token:
+        if pages >= MAX_NODE_PAGES:
+            raise RuntimeError(f"Stopped after {MAX_NODE_PAGES} pages of {prop!r}.")
+        page: JSON = transport.get("/core/api/v2/node", {**params, "nextToken": token})
+        _merge_node_page(merged, page)
+        token = page.get("nextToken")
+        pages += 1
+    return merged
+
+
+def _merge_node_page(merged: JSON, page: JSON) -> None:
+    """Append one page's arc values onto the accumulated result, in place."""
+    for dcid, entry in page.get("data", {}).items():
+        target = merged.setdefault("data", {}).setdefault(dcid, {})
+        for arc, values in entry.get("arcs", {}).items():
+            existing = target.setdefault("arcs", {}).setdefault(arc, {"nodes": []})
+            existing.setdefault("nodes", []).extend(values.get("nodes", []))
+        if "properties" in entry:
+            props = target.setdefault("properties", [])
+            props.extend(p for p in entry["properties"] if p not in props)
+
+
+def resolve(
+    transport: Transport,
+    nodes: str | Sequence[str],
+    prop: str = "<-description->dcid",
+) -> JSON:
+    """Resolve names or other identifiers to dcids via ``/core/api/v2/resolve``."""
+    result: JSON = transport.get(
+        "/core/api/v2/resolve", {"nodes": _as_list(nodes), "property": prop}
+    )
+    return result
+
+
+def series(
+    transport: Transport,
+    variables: str | Sequence[str],
+    entities: str | Sequence[str],
+) -> JSON:
+    """Fetch full time series for every variable/entity pair from the website API.
+
+    Entities with no data are absent from the response rather than empty.
+    """
+    result: JSON = transport.post(
+        "/api/observations/series",
+        {"variables": _as_list(variables), "entities": _as_list(entities)},
+    )
+    return result
+
+
+def stat_var_search(transport: Transport, query: str) -> JSON:
+    """Search statistical variables by text.
+
+    Results include Data Commons' base variables (``Count_Person``) as well as the
+    UN collections (``undata/...``).
+    """
+    result: JSON = transport.get("/api/stats/stat-var-search", {"query": query})
+    return result
+
+
+def variable_info(transport: Transport, dcids: str | Sequence[str]) -> JSON:
+    """Coverage and provenance summary per variable, from ``/api/variable/info``."""
+    result: JSON = transport.get("/api/variable/info", {"dcids": _as_list(dcids)})
+    return result
+
+
+def place_names(transport: Transport, dcids: str | Sequence[str]) -> dict[str, str]:
+    """Display names for place dcids."""
+    result: dict[str, str] = transport.get("/api/place/name", {"dcids": _as_list(dcids)})
+    return result
+
+
+def variable_group_info(transport: Transport, dcid: str) -> JSON:
+    """Children and indicator counts for a node in the variable-group tree."""
+    result: JSON = transport.post("/api/variable-group/info", {"dcid": dcid, "entities": []})
+    return result
+
+
+def detect_and_fulfill(transport: Transport, question: str) -> JSON:
+    """Run a question through the site's natural-language resolver.
+
+    The resolver always commits to a place. When the question names none it
+    recognises, it supplies one silently, so the result must be inspected before
+    it is trusted.
+    """
+    result: JSON = transport.post(
+        "/api/explore/detect-and-fulfill", {"contextHistory": []}, params={"q": question}
+    )
+    return result

undata/_http.py

@@ -0,0 +1,210 @@
+"""Transport: the only module that talks to the network.
+
+It exists to make the platform's request traps impossible to trip over:
+
+* A line break anywhere in a POST body is answered with 403, so bodies are always
+  serialised compactly here rather than left to the caller.
+* A POST body containing a ``"select"`` key is also answered with 403, so the
+  observation endpoint is only ever called with GET (see ``_endpoints``).
+* Relation expressions (``->``, ``<-``, ``+``, ``{typeOf:Country}``) must be
+  percent-encoded or the API answers 400. Query strings are built here with every
+  reserved character percent-encoded, ``+`` included.
+* Any GET whose query string contains the word ``ESH`` (Western Sahara's country
+  code) is answered with 403: the firewall mistakes it for the ``esh`` shell. The
+  friendly functions therefore fetch data by POST (see ``data._collect``).
+* The server does not decode ``+`` as a space in a query string: ``nodes=Viet+Nam``
+  reaches it as ``Viet+Nam``. Spaces are therefore sent as ``%20``, never ``+``,
+  which is what httpx would do by default.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import time
+from collections.abc import Mapping, Sequence
+from typing import Any, Final
+from urllib.parse import quote, urlencode
+
+import httpx
+
+from ._version import __version__
+from .errors import APIError, BlockedRequestError, ConnectionFailedError
+
+DEFAULT_BASE_URL: Final = "https://unsd-datacommons.gcp.un-icc.cloud"
+DEFAULT_TIMEOUT: Final = 30.0
+DEFAULT_RETRIES: Final = 3
+
+USER_AGENT: Final = f"undata/{__version__} (+https://github.com/nerd-factory/undata)"
+
+#: Statuses worth retrying: rate limiting and transient upstream failures.
+RETRY_STATUSES: Final = frozenset({429, 502, 503, 504})
+
+#: A query value is a string, or a sequence of strings sent as repeated keys.
+QueryValue = str | Sequence[str]
+
+_BLOCKED_MESSAGE: Final = (
+    "The data.un.org firewall refused this request. No API key exists or is needed, "
+    "so this is not an authentication problem. The firewall is known to refuse a POST "
+    'body containing a line break or a "select" key, and any GET request whose query '
+    "contains the word ESH (Western Sahara's country code), which it mistakes for a "
+    "shell command. undata's own functions avoid all three; if one of them raised this, "
+    "the rules have changed. Please report it at "
+    "https://github.com/nerd-factory/undata/issues with the URL above."
+)
+
+
+def encode_body(body: Mapping[str, Any]) -> bytes:
+    """Serialise a JSON body on a single line, which this deployment requires."""
+    return json.dumps(body, separators=(",", ":"), ensure_ascii=False).encode()
+
+
+def query_string(params: Mapping[str, QueryValue | None]) -> str:
+    """Encode params as a query string the way this deployment needs.
+
+    Sequence values become repeated keys, ``None`` values are dropped, and every
+    reserved character is percent-encoded. Spaces become ``%20``: the server reads
+    ``+`` literally.
+
+    Example:
+        >>> query_string({"nodes": ["Viet Nam", "Chad"], "property": "<-a+", "x": None})
+        'nodes=Viet%20Nam&nodes=Chad&property=%3C-a%2B'
+    """
+    pairs: list[tuple[str, str]] = []
+    for key, value in params.items():
+        if value is None:
+            continue
+        if isinstance(value, str):
+            pairs.append((key, value))
+        else:
+            pairs.extend((key, item) for item in value)
+    return urlencode(pairs, quote_via=quote, safe="")
+
+
+class Transport:
+    """A synchronous HTTP transport with retries and explanatory errors.
+
+    Args:
+        base_url: Root of the Data Commons deployment.
+        timeout: Seconds to wait for each response.
+        retries: How many times to retry a 429, 502, 503 or 504, or a dropped
+            connection, before giving up.
+        http_client: An ``httpx.Client`` to use instead of creating one. Useful for
+            proxies, custom certificates, and tests.
+    """
+
+    def __init__(
+        self,
+        *,
+        base_url: str = DEFAULT_BASE_URL,
+        timeout: float = DEFAULT_TIMEOUT,
+        retries: int = DEFAULT_RETRIES,
+        http_client: httpx.Client | None = None,
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.retries = retries
+        self._owns_client = http_client is None
+        self._client = http_client or httpx.Client(
+            timeout=timeout,
+            headers={"User-Agent": USER_AGENT, "Accept": "application/json"},
+            follow_redirects=True,
+        )
+
+    def get(self, path: str, params: Mapping[str, QueryValue | None] | None = None) -> Any:
+        """GET ``path`` and return the decoded JSON body."""
+        request = self._client.build_request("GET", self._url(path, params))
+        return self._send(request)
+
+    def post(
+        self,
+        path: str,
+        body: Mapping[str, Any],
+        params: Mapping[str, QueryValue | None] | None = None,
+    ) -> Any:
+        """POST a single-line JSON ``body`` to ``path`` and return the decoded JSON."""
+        request = self._client.build_request(
+            "POST",
+            self._url(path, params),
+            content=encode_body(body),
+            headers={"Content-Type": "application/json"},
+        )
+        return self._send(request)
+
+    def close(self) -> None:
+        """Close the underlying connection pool, if this transport created it."""
+        if self._owns_client:
+            self._client.close()
+
+    def __enter__(self) -> Transport:
+        return self
+
+    def __exit__(self, *exc_info: object) -> None:
+        self.close()
+
+    def _url(self, path: str, params: Mapping[str, QueryValue | None] | None) -> str:
+        url = f"{self.base_url}/{path.lstrip('/')}"
+        query = query_string(params or {})
+        return f"{url}?{query}" if query else url
+
+    def _send(self, request: httpx.Request) -> Any:
+        url = str(request.url)
+        for attempt in range(self.retries + 1):
+            last_attempt = attempt == self.retries
+            try:
+                response = self._client.send(request)
+            except httpx.TransportError as exc:
+                if last_attempt:
+                    raise ConnectionFailedError(
+                        f"Could not reach data.un.org: {exc}", url=url
+                    ) from exc
+                _sleep_before_retry(attempt, None)
+                continue
+
+            if response.status_code in RETRY_STATUSES and not last_attempt:
+                _sleep_before_retry(attempt, response.headers.get("Retry-After"))
+                continue
+            return _decode(response, url)
+
+        raise AssertionError("unreachable: the final attempt always returns or raises")
+
+
+def _decode(response: httpx.Response, url: str) -> Any:
+    status = response.status_code
+    body = response.text[:500]
+    if status == httpx.codes.FORBIDDEN:
+        raise BlockedRequestError(_BLOCKED_MESSAGE, status=status, url=url, body=body)
+    if status >= httpx.codes.BAD_REQUEST:
+        raise APIError(_status_message(status), status=status, url=url, body=body)
+    try:
+        return response.json()
+    except ValueError as exc:
+        raise APIError(
+            "The API answered successfully but the body was not JSON.",
+            status=status,
+            url=url,
+            body=body,
+        ) from exc
+
+
+def _status_message(status: int) -> str:
+    if status == httpx.codes.BAD_REQUEST:
+        return (
+            "The API rejected the request as malformed. Check identifiers for typos; "
+            "an unknown property in a relation expression also causes this."
+        )
+    if status == httpx.codes.NOT_FOUND:
+        return "No such endpoint on this deployment."
+    if status == httpx.codes.METHOD_NOT_ALLOWED:
+        return "This endpoint does not accept that HTTP method."
+    if status >= httpx.codes.INTERNAL_SERVER_ERROR:
+        return "The API failed while answering. Retrying later usually works."
+    return "The API answered with an error."
+
+
+def _sleep_before_retry(attempt: int, retry_after: str | None) -> None:
+    delay = 0.5 * 2**attempt
+    if retry_after is not None:
+        # An HTTP-date Retry-After is ignored; the exponential delay is close enough.
+        with contextlib.suppress(ValueError):
+            delay = max(delay, float(retry_after))
+    time.sleep(min(delay, 30.0))

undata/_session.py

@@ -0,0 +1,58 @@
+"""The shared connection and in-memory cache behind the module-level functions.
+
+Most people never touch this: ``un.get(...)`` opens a connection the first time it
+is needed and reuses it. Call :func:`configure` to change the timeout or point at
+another deployment.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any, TypeVar
+
+from ._http import DEFAULT_BASE_URL, DEFAULT_RETRIES, DEFAULT_TIMEOUT, Transport
+
+T = TypeVar("T")
+
+
+class _State:
+    transport: Transport | None = None
+    cache: dict[str, Any] = {}  # noqa: RUF012 — one process-wide cache, deliberately
+
+
+def transport() -> Transport:
+    """Return the shared transport, opening it on first use."""
+    if _State.transport is None:
+        _State.transport = Transport()
+    return _State.transport
+
+
+def configure(
+    *,
+    base_url: str = DEFAULT_BASE_URL,
+    timeout: float = DEFAULT_TIMEOUT,
+    retries: int = DEFAULT_RETRIES,
+) -> None:
+    """Change how undata connects, and forget anything it has cached.
+
+    Args:
+        base_url: Root of the Data Commons deployment.
+        timeout: Seconds to wait for each response. Raise it on a slow connection.
+        retries: How many times to retry a request that failed for a temporary reason.
+
+    Example:
+        >>> import undata as un
+        >>> un.configure(timeout=60)
+    """
+    if _State.transport is not None:
+        _State.transport.close()
+    _State.transport = Transport(base_url=base_url, timeout=timeout, retries=retries)
+    _State.cache.clear()
+
+
+def cached(key: str, factory: Callable[[], T]) -> T:
+    """Return the cached value for ``key``, computing it with ``factory`` the first time."""
+    if key not in _State.cache:
+        _State.cache[key] = factory()
+    value: T = _State.cache[key]
+    return value

undata/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

undata/ask.py

@@ -0,0 +1,98 @@
+"""See what data.un.org's search box makes of a question, before trusting its chart.
+
+The search box always picks a place. When a question names no place it knows, it
+uses a default one and says so only in small print. :func:`ask` shows the place it
+chose, whether it chose it itself, and the indicators it picked.
+"""
+
+from __future__ import annotations
+
+import warnings
+from dataclasses import dataclass, field
+
+from . import _endpoints as api
+from . import _session
+from .errors import PlaceGuessedWarning
+
+__all__ = ["Answer", "ask"]
+
+
+@dataclass(frozen=True, slots=True)
+class Answer:
+    """What the search box decided.
+
+    Attributes:
+        question: The question as asked.
+        place: The place it answered about, by name.
+        place_id: The place's id.
+        place_was_guessed: True when the question named no place the search box
+            recognised and it supplied one of its own.
+        indicators: ``(id, name)`` for each indicator it chose.
+        messages: Any notes the search box returned.
+    """
+
+    question: str
+    place: str | None
+    place_id: str | None
+    place_was_guessed: bool
+    indicators: list[tuple[str, str]] = field(default_factory=list)
+    messages: list[str] = field(default_factory=list)
+
+    def __repr__(self) -> str:
+        if self.place is None:
+            where = "  place         none"
+        elif self.place_was_guessed:
+            where = f"  place         {self.place}  <- GUESSED: your question did not name it"
+        else:
+            where = f"  place         {self.place}  (from your question)"
+        lines = [f"Question: {self.question}", where]
+        lines += [f"  indicator     {name}  [{dcid}]" for dcid, name in self.indicators[:8]]
+        if len(self.indicators) > 8:  # noqa: PLR2004
+            lines.append(f"  ...and {len(self.indicators) - 8} more")
+        lines += [f"  note          {m}" for m in self.messages]
+        return "\n".join(lines)
+
+
+def ask(question: str) -> Answer:
+    """Ask data.un.org's search box a question and see what it decided.
+
+    Args:
+        question: A question as you would type it on data.un.org.
+
+    Returns:
+        An :class:`Answer`. A :class:`~undata.PlaceGuessedWarning` is shown when the
+        search box chose a place your question did not name.
+
+    Example:
+        >>> import undata as un
+        >>> un.ask("child mortality in Bengal")  # doctest: +SKIP
+        Question: child mortality in Bengal
+          place         United States of America  <- GUESSED: your question did not name it
+        ...
+    """
+    body = api.detect_and_fulfill(_session.transport(), question)
+    place = body.get("place") or {}
+    guessed = bool(place) and body.get("placeSource") != "CURRENT_QUERY"
+
+    chosen: dict[str, str] = {}
+    for category in (body.get("config") or {}).get("categories", []):
+        for key, spec in (category.get("statVarSpec") or {}).items():
+            dcid = spec.get("statVar") or key.removesuffix("_multiple_place_bar_block")
+            chosen.setdefault(dcid, spec.get("name", dcid))
+
+    answer = Answer(
+        question=question,
+        place=place.get("name"),
+        place_id=place.get("dcid"),
+        place_was_guessed=guessed,
+        indicators=list(chosen.items()),
+        messages=list(body.get("userMessages") or []),
+    )
+    if guessed:
+        warnings.warn(
+            f"data.un.org answered {question!r} about {answer.place}, a place your question "
+            "did not name. Name the country exactly, or use un.get() with a country you choose.",
+            PlaceGuessedWarning,
+            stacklevel=2,
+        )
+    return answer