misocialpy 0.4.0__py3-none-any.whl

misocialpy/__init__.py

@@ -0,0 +1,66 @@
+"""misocialpy — access the MDS MI Social API from Python.
+
+A pandas-friendly port of the R package `misocialr`. Monthly municipal series
+of the Cadastro Unico, the Bolsa Familia programme and its conditionalities,
+the BPC, the social assistance network and its funding, and food security
+programmes, from the Social Information Matrix of the
+Brazilian Ministry of Development and Social Assistance (MDS/SAGI).
+
+Quick start
+-----------
+>>> import misocialpy
+>>> misocialpy.mds_bpc_suas(periods=2025, localities="PE")      # doctest: +SKIP
+>>> misocialpy.mds_periods("bpc_ben_i", localities="PE")        # doctest: +SKIP
+"""
+
+from __future__ import annotations
+
+from ._cache import clear_cache as mds_clear_cache
+from ._cache import set_cache_ttl
+from ._client import DataConflictError, HttpError, MdsError, ResponseError, set_timeout
+from ._msg import set_verbose
+from .data import (
+    mds_bolsa_familia,
+    mds_bpc_suas,
+    mds_cadunico,
+    mds_data,
+    mds_pbf_condicionalidades,
+    mds_seguranca_alimentar,
+    mds_suas_financiamento,
+    mds_suas_rede,
+)
+from .fields import mds_api_fields, mds_fields, mds_themes
+from .localities import ibge_code7, mds_localities
+from .periods import mds_periods
+from .publication import mds_drop_unpublished
+from .validation import ValidationError
+
+__version__ = "0.4.0"
+
+__all__ = [
+    "DataConflictError",
+    "HttpError",
+    "MdsError",
+    "ResponseError",
+    "ValidationError",
+    "__version__",
+    "ibge_code7",
+    "mds_api_fields",
+    "mds_bolsa_familia",
+    "mds_bpc_suas",
+    "mds_cadunico",
+    "mds_clear_cache",
+    "mds_data",
+    "mds_drop_unpublished",
+    "mds_fields",
+    "mds_localities",
+    "mds_pbf_condicionalidades",
+    "mds_periods",
+    "mds_seguranca_alimentar",
+    "mds_suas_financiamento",
+    "mds_suas_rede",
+    "mds_themes",
+    "set_cache_ttl",
+    "set_timeout",
+    "set_verbose",
+]

misocialpy/_cache.py

@@ -0,0 +1,83 @@
+"""In-memory, per-session cache (equivalent to the R package's environment).
+
+Holds localities, field lists, period facets and fields already confirmed to
+exist.
+"""
+
+from __future__ import annotations
+
+import math
+import time
+from typing import Any
+
+_CACHE: dict[str, Any] = {}
+_STORED_AT: dict[str, float] = {}
+_TTL = math.inf
+
+
+def set_cache_ttl(seconds: float = math.inf) -> None:
+    """Set how long, in seconds, a cached result is used (default: the whole
+    session). Must be a non-negative number; on error the previous value is
+    kept."""
+    global _TTL
+    if (
+        not isinstance(seconds, (int, float))
+        or isinstance(seconds, bool)
+        or math.isnan(seconds)
+        or seconds < 0
+    ):
+        raise ValueError("`seconds` must be a non-negative number.")
+    _TTL = float(seconds)
+
+
+def _copy(value: Any) -> Any:
+    """Independent copy of a cached table, with its attrs.
+
+    Pandas objects are mutable: handing out the stored instance would let an
+    edit of a result change the metadata used by later calls (periods feed
+    the inference of publication).
+    """
+    if hasattr(value, "copy") and hasattr(value, "attrs"):
+        out = value.copy(deep=True)
+        out.attrs = dict(value.attrs)
+        return out
+    return value
+
+
+def cache_get(key: str, refresh: bool = False) -> Any:
+    """Read a copy of a cached value, unless a refresh is requested."""
+    if refresh or key not in _CACHE:
+        return None
+    if time.monotonic() - _STORED_AT.get(key, 0.0) > _TTL:
+        return None
+    return _copy(_CACHE[key])
+
+
+def cache_set(key: str, value: Any) -> Any:
+    """Store a copy of a value and return another (only complete, validated
+    responses reach here)."""
+    from datetime import datetime, timezone
+
+    if hasattr(value, "attrs"):
+        value.attrs["retrieved_at"] = datetime.now(timezone.utc).isoformat(timespec="seconds")
+    _CACHE[key] = _copy(value)
+    _STORED_AT[key] = time.monotonic()
+    return _copy(value)
+
+
+def locality_key(loc: dict) -> str:
+    """Canonical key: ['PE', 'PB'] and ['PB', 'PE'] share the cached result."""
+    return f"uf={','.join(sorted(loc['states']))};mun={','.join(sorted(loc['municipalities']))}"
+
+
+def clear_cache() -> None:
+    """Remove all cached localities, field lists, periods and field checks.
+
+    Forces fresh API calls on subsequent requests. Public alias:
+    :func:`misocialpy.mds_clear_cache`.
+    """
+    _CACHE.clear()
+    _STORED_AT.clear()
+    from . import _msg
+
+    _msg.success("MI Social cache cleared.")

misocialpy/_client.py

@@ -0,0 +1,162 @@
+"""HTTP layer for the MI Social API.
+
+The MI Social API is an open Apache Solr ``select`` handler: every call is a
+``q=*:*`` query narrowed with filter queries (``fq``), with paging
+(``start``, ``rows``), field selection (``fl``) and facets.
+
+Transient failures are retried (mirroring ``httr2::req_retry``) and errors
+raise :class:`MdsError` with the Solr message when there is one.
+"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Mapping
+from typing import Any
+
+import requests
+from requests.adapters import HTTPAdapter
+
+try:  # urllib3 ships with requests; import path is stable
+    from urllib3.util.retry import Retry
+except ImportError:  # pragma: no cover
+    from requests.packages.urllib3.util.retry import Retry  # type: ignore
+
+from . import _msg
+
+MDS_BASE_URL = "https://aplicacoes.mds.gov.br/sagi/servicos/misocial"
+
+_TIMEOUT = 180.0
+
+
+def _user_agent() -> str:
+    """User agent with the package version."""
+    from . import __version__
+
+    return f"misocialpy/{__version__} (Python package; {MDS_BASE_URL})"
+
+
+# Seconds between retries grow as _BACKOFF * 2 ** (retry - 1); the first
+# retry is immediate (urllib3).
+_BACKOFF = 2.0
+
+
+def set_timeout(seconds: float) -> None:
+    """Set the timeout, in seconds, of every request (default 180).
+
+    Must be a finite, strictly positive number; on error the previous timeout
+    is kept.
+    """
+    global _TIMEOUT
+    if (
+        not isinstance(seconds, (int, float))
+        or isinstance(seconds, bool)
+        or not math.isfinite(seconds)
+        or seconds <= 0
+    ):
+        raise ValueError("`seconds` must be a finite, positive number.")
+    _TIMEOUT = float(seconds)
+
+
+class MdsError(RuntimeError):
+    """Base class of every misocialpy error."""
+
+
+class HttpError(MdsError):
+    """The request failed (transport or HTTP status). ``status`` is the HTTP
+    status code, or ``None`` when no response arrived."""
+
+    def __init__(self, message: str, status: int | None = None):
+        super().__init__(message)
+        self.status = status
+
+
+class ResponseError(MdsError):
+    """The API answered, but the answer is incomplete or malformed."""
+
+
+class DataConflictError(MdsError):
+    """The API contradicts itself (two values for the same observation)."""
+
+
+# A Retry-After header is obeyed, but never for more than this many seconds
+# per try (the same cap as the R package).
+_RETRY_AFTER_MAX = 30.0
+
+
+class _CappedRetry(Retry):
+    """urllib3 Retry whose Retry-After wait is capped."""
+
+    def get_retry_after(self, response):
+        wait = super().get_retry_after(response)
+        return None if wait is None else min(wait, _RETRY_AFTER_MAX)
+
+
+def _session() -> requests.Session:
+    session = requests.Session()
+    # 400 means the query itself is wrong (e.g. an undefined field): retrying
+    # would only repeat the same answer. Connection failures (DNS, reset,
+    # timeout) are retried too, within the same total.
+    # urllib3 counts retries after the first request: 2 retries = 3 tries,
+    # the same as max_tries = 3 in the R package.
+    retry = _CappedRetry(
+        total=2,
+        connect=2,
+        read=2,
+        backoff_factor=_BACKOFF,
+        status_forcelist=(429, 500, 502, 503, 504),
+        # The query is sent by POST but only reads: retrying it is safe.
+        allowed_methods=frozenset(["GET", "POST"]),
+        raise_on_status=False,
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount("https://", adapter)
+    session.mount("http://", adapter)
+    session.headers.update({"User-Agent": _user_agent()})
+    return session
+
+
+def _http_error_hint(resp: requests.Response) -> str:
+    """Explain an HTTP error, surfacing Solr's ``error.msg`` when present."""
+    try:
+        msg = resp.json()["error"]["msg"]
+    except (ValueError, KeyError, TypeError):
+        msg = None
+    if msg:
+        return f"HTTP error {resp.status_code}: {msg}"
+    return f"HTTP error {resp.status_code}"
+
+
+def mds_request(query: Mapping[str, Any] | None = None, label: str = "data") -> Any:
+    """Build and execute a request to the MI Social API.
+
+    Repeated parameters (several ``fq``) are given as a list under one key.
+    """
+    params = {"q": "*:*", "wt": "json"}
+    params.update({k: v for k, v in (query or {}).items() if v is not None})
+
+    _msg.step(f"Fetching {label} from MDS MI Social API...")
+    try:
+        # The query goes in the body (POST form): with many months, fields and
+        # codes a GET URL passes the server limit (HTTP 414 measured at about
+        # 8,000 characters). The Solr select handler answers both the same way.
+        resp = _session().post(MDS_BASE_URL, data=params, timeout=_TIMEOUT)
+    except requests.RequestException as exc:
+        raise HttpError(
+            f"Failed to access the MDS MI Social API.\n  x {exc}\n  i URL: {MDS_BASE_URL}"
+        ) from exc
+
+    if resp.status_code >= 400:
+        raise HttpError(
+            "Failed to access the MDS MI Social API.\n"
+            f"  x {_http_error_hint(resp)}\n  i URL: {resp.url}",
+            status=resp.status_code,
+        )
+
+    # The server labels its JSON as text/plain; requests does not check it.
+    try:
+        return resp.json()
+    except ValueError as exc:
+        raise ResponseError(
+            "Unexpected response from the MI Social API.\n  x The body is not valid JSON."
+        ) from exc

misocialpy/_format.py

@@ -0,0 +1,91 @@
+"""Solr filter builders and small helpers shared by the modules."""
+
+from __future__ import annotations
+
+import math
+import numbers
+from collections.abc import Iterable, Sequence
+from datetime import date
+from typing import Any
+
+# Columns that identify a document (municipality x month). Everything else in
+# a document is a measure.
+ID_FIELDS = ["codigo_ibge", "municipio", "sigla_uf", "anomes_s"]
+
+
+def as_list(x: Any) -> list:
+    """A scalar (text, number, numpy scalar) becomes a one-element list; any
+    other iterable becomes a list."""
+    if isinstance(x, (str, bytes, numbers.Number)) or not hasattr(x, "__iter__"):
+        return [x]
+    return list(x)
+
+
+def is_number(x: Any) -> bool:
+    """A real number (int, float, numpy number), but not a bool."""
+    return isinstance(x, numbers.Real) and not isinstance(x, bool)
+
+
+def whole_text(x: Any) -> str:
+    """Text of a value; a number with a whole value is written without
+    decimals (26.0 -> "26"), as in the R package."""
+    if is_number(x) and math.isfinite(x) and x == int(x):
+        return str(int(x))
+    return str(x).strip()
+
+
+# Years accepted in `periods`: the API starts in 01/2003; anything outside
+# 2000 to next year is a typo, not a query.
+MIN_YEAR = 2000
+
+
+def solr_in(field: str, values: Iterable) -> str:
+    """Solr disjunction of terms for one field: ``field:(a OR b OR c)``."""
+    return f"{field}:({' OR '.join(str(v) for v in values)})"
+
+
+def solr_has(field: str) -> str:
+    """Solr filter matching documents where a field has a value."""
+    return f"{field}:[* TO *]"
+
+
+def chunk_vector(x: Sequence, size: int) -> list[list]:
+    """Split a sequence into chunks of at most ``size`` elements."""
+    x = list(x)
+    if not x:
+        return [x]
+    return [x[i : i + size] for i in range(0, len(x), size)]
+
+
+def unit_from_suffix(field: str) -> str:
+    """Unit implied by the field-name suffix.
+
+    MI Social field names end with the Solr dynamic-field type: ``_i``/``_l``
+    integer count, ``_f`` float (currency in most cases), ``_d`` double
+    (percentages and rates). ``_s`` is how the value is stored (a string),
+    not what it measures, so its unit is ``"unknown"``.
+    """
+    for suffix, unit in (("_i", "count"), ("_l", "count"), ("_f", "value"), ("_d", "rate")):
+        if field.endswith(suffix):
+            return unit
+    return "unknown"
+
+
+def storage_from_suffix(field: str) -> str:
+    """Solr storage type implied by the field-name suffix."""
+    for suffix, kind in (
+        ("_i", "integer"),
+        ("_l", "long"),
+        ("_f", "float"),
+        ("_d", "double"),
+        ("_s", "string"),
+    ):
+        if field.endswith(suffix):
+            return kind
+    return "untyped"
+
+
+def period_of(today: date | None = None) -> int:
+    """Month of ``today`` (default: now) as a ``YYYYMM`` integer."""
+    today = today or date.today()
+    return today.year * 100 + today.month

misocialpy/_msg.py

@@ -0,0 +1,31 @@
+"""Lightweight console messages, mirroring the R package's cli alerts.
+
+Messages go to stderr and can be silenced with :func:`set_verbose`.
+"""
+
+from __future__ import annotations
+
+import sys
+
+_VERBOSE = True
+
+
+def set_verbose(verbose: bool) -> None:
+    """Enable or disable informational messages (success/step/inform)."""
+    global _VERBOSE
+    _VERBOSE = bool(verbose)
+
+
+def success(message: str) -> None:
+    if _VERBOSE:
+        print(f"✔ {message}", file=sys.stderr)
+
+
+def step(message: str) -> None:
+    if _VERBOSE:
+        print(f"ℹ {message}", file=sys.stderr)
+
+
+def inform(message: str) -> None:
+    if _VERBOSE:
+        print(message, file=sys.stderr)

misocialpy/_response.py

@@ -0,0 +1,170 @@
+"""Response contract, document identity, scalar values and conflicts.
+
+Every response is checked before it is interpreted or cached: a partial,
+truncated or malformed answer is an error, never an empty result (deep review
+of 0.2.1, items B1, B2, B4, B5, B8 and M5). Mirrors ``R/resposta.R``.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from typing import Any
+
+from ._client import DataConflictError, ResponseError
+
+
+def _is_count(x: Any) -> bool:
+    return (
+        isinstance(x, (int, float))
+        and not isinstance(x, bool)
+        and math.isfinite(x)
+        and x >= 0
+        and x == int(x)
+    )
+
+
+def check_response(res: Any, start: int = 0, kind: str = "docs") -> None:
+    """Check the shape of a Solr response before using it.
+
+    ``kind`` is ``"docs"`` (a page of documents), ``"count"`` (``rows=0``,
+    only ``numFound`` is used) or ``"facet"`` (month counts).
+    """
+    where = f"page after {start} documents" if kind == "docs" else kind
+    resp = res.get("response") if isinstance(res, dict) else None
+    if not isinstance(resp, dict):
+        raise ResponseError(
+            f"Unexpected response from the MI Social API.\n  x No `response` element ({where})."
+        )
+    header = res.get("responseHeader") or {}
+    status = header.get("status")
+    if status is not None and status != 0:
+        raise ResponseError(
+            f"The MI Social API reported an error.\n  x responseHeader status {status} ({where})."
+        )
+    # Solr flags results cut short by a time limit.
+    if header.get("partialResults"):
+        raise ResponseError(
+            "Partial results from the MI Social API.\n"
+            f"  x The server stopped the query before the end ({where}). "
+            "Try a smaller selection."
+        )
+    if not _is_count(resp.get("numFound")):
+        raise ResponseError(
+            "Unexpected response from the MI Social API.\n"
+            f"  x `numFound` must be a non-negative whole number ({where})."
+        )
+    if kind == "docs":
+        docs = resp.get("docs")
+        if not isinstance(docs, list) or not all(isinstance(d, dict) for d in docs):
+            raise ResponseError(
+                "Unexpected response from the MI Social API.\n"
+                f"  x `docs` must be a list of documents ({where})."
+            )
+    if kind == "facet":
+        f = ((res.get("facet_counts") or {}).get("facet_fields") or {}).get("anomes_s")
+        ok = isinstance(f, list) and len(f) % 2 == 0
+        if ok and f:
+            months = f[0::2]
+            ok = (
+                all(is_month(p) for p in months)
+                and len(set(months)) == len(months)
+                and all(_is_count(n) for n in f[1::2])
+            )
+        if not ok:
+            raise ResponseError(
+                "Unexpected response from the MI Social API.\n"
+                f"  x Missing or malformed `facet_counts` ({where})."
+            )
+
+
+def is_month(x: Any) -> bool:
+    """A month as 'YYYYMM' with the month between 01 and 12 (the one calendar
+    rule used for input, documents and facets)."""
+    return isinstance(x, str) and bool(re.fullmatch(r"[0-9]{6}", x)) and 1 <= int(x[4:]) <= 12
+
+
+def check_doc_identity(docs: list[dict[str, Any]]) -> None:
+    """Every data document must identify its municipality and month."""
+    bad = [
+        d
+        for d in docs
+        if not re.fullmatch(r"[0-9]{6}", str(d.get("codigo_ibge") or ""))
+        or not is_month(d.get("anomes_s"))
+    ]
+    if bad:
+        raise ResponseError(
+            "Unexpected document in the MI Social response.\n"
+            f"  x {len(bad)} document(s) without a valid `codigo_ibge` (6 digits) "
+            "or `anomes_s` (YYYYMM, month 01 to 12)."
+        )
+
+
+def doc_value(field: str, value: Any) -> float:
+    """One numeric value per document; a list or a text is an error."""
+    from .validation import ValidationError
+
+    if isinstance(value, (list, tuple, dict, set)):
+        raise ResponseError(
+            f"Field `{field}` is not a single value per document.\n"
+            "  x A document carries a list or several values."
+        )
+    try:
+        number = float(value)
+    except (TypeError, ValueError):
+        raise ValidationError(
+            f"Field `{field}` is not numeric.\n"
+            f"  x The API returned {value!r}.\n"
+            "  i misocialpy returns numeric values only."
+        ) from None
+    # "NaN", "Inf" or "1e999" are numbers that cannot be a measure; accepting
+    # them would create missing or infinite values the source never meant.
+    if not math.isfinite(number):
+        raise ResponseError(
+            f"Field `{field}` has non-finite values.\n  x The API returned {value!r}."
+        )
+    return number
+
+
+def check_ids(docs: list[dict[str, Any]]) -> None:
+    """A document id read twice means the source changed while paging."""
+    seen: set = set()
+    dup = []
+    for d in docs:
+        i = d.get("id")
+        if i is None:
+            continue
+        if i in seen:
+            dup.append(i)
+        seen.add(i)
+    if dup:
+        raise ResponseError(
+            "The MI Social API returned the same document twice.\n"
+            f"  x Repeated id(s): {dup[:3]}.\n"
+            "  i The source probably changed while it was being read; run the query again."
+        )
+
+
+def dedupe_observations(out):
+    """Drop repeated observations; two different values for the same key are an error."""
+    key = ["locality_id", "period", "field"]
+    dup = out.duplicated(key, keep=False)
+    if not dup.any():
+        return out
+    # A missing value never replaces a known one, whatever the order of the
+    # rows: when a key has a known value, its missing rows are dropped first.
+    has_known = out[dup & out["value"].notna()].set_index(key).index.unique()
+    keys = out.set_index(key).index
+    out = out[~(dup & out["value"].isna() & keys.isin(has_known))].reset_index(drop=True)
+    dup = out.duplicated(key, keep=False)
+    if not dup.any():
+        return out
+    n_values = out[dup].groupby(key)["value"].nunique(dropna=True)
+    conflicts = n_values[n_values > 1]
+    if len(conflicts):
+        shown = [" / ".join(k) for k in conflicts.index[:3]]
+        raise DataConflictError(
+            "The MI Social API returned conflicting values.\n"
+            f"  x Different values for the same municipality / month / field: {shown}."
+        )
+    return out.drop_duplicates(key).reset_index(drop=True)