condepe 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- condepe/__init__.py +53 -0
- condepe/_client.py +167 -0
- condepe/_msg.py +31 -0
- condepe/data.py +226 -0
- condepe-0.1.0.dist-info/METADATA +133 -0
- condepe-0.1.0.dist-info/RECORD +8 -0
- condepe-0.1.0.dist-info/WHEEL +4 -0
- condepe-0.1.0.dist-info/licenses/LICENSE +21 -0
condepe/__init__.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""condepe — open data of the Pernambuco Observatory from Python.
|
|
2
|
+
|
|
3
|
+
Municipal series of Pernambuco, Brazil, published by Condepe/Fidem (the State
|
|
4
|
+
Agency for Planning and Research) in the Observatory of Pernambuco: population,
|
|
5
|
+
economy, public finance, labour, education, health, security, transport and
|
|
6
|
+
more, every source in the same long format.
|
|
7
|
+
|
|
8
|
+
Data come from https://cf.pe.gov.br/observatorio/dados/atual/, with a mirror
|
|
9
|
+
on Hugging Face as fallback, and are cached on disk per catalogue version.
|
|
10
|
+
|
|
11
|
+
Quick start
|
|
12
|
+
-----------
|
|
13
|
+
>>> import condepe
|
|
14
|
+
>>> condepe.condepe_catalog() # doctest: +SKIP
|
|
15
|
+
>>> condepe.condepe_series("sidra_populacao_estimada",
|
|
16
|
+
... "Populacao residente estimada", code="2611606") # doctest: +SKIP
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from ._client import (
|
|
22
|
+
CondepeError,
|
|
23
|
+
HttpError,
|
|
24
|
+
IntegrityError,
|
|
25
|
+
set_timeout,
|
|
26
|
+
)
|
|
27
|
+
from ._client import cache_dir as condepe_cache_dir
|
|
28
|
+
from ._client import clear_cache as condepe_cache_clear
|
|
29
|
+
from ._msg import set_verbose
|
|
30
|
+
from .data import (
|
|
31
|
+
condepe_catalog,
|
|
32
|
+
condepe_data,
|
|
33
|
+
condepe_indicators,
|
|
34
|
+
condepe_localities,
|
|
35
|
+
condepe_series,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
__version__ = "0.1.0"
|
|
39
|
+
|
|
40
|
+
__all__ = [
|
|
41
|
+
"CondepeError",
|
|
42
|
+
"HttpError",
|
|
43
|
+
"IntegrityError",
|
|
44
|
+
"condepe_cache_clear",
|
|
45
|
+
"condepe_cache_dir",
|
|
46
|
+
"condepe_catalog",
|
|
47
|
+
"condepe_data",
|
|
48
|
+
"condepe_indicators",
|
|
49
|
+
"condepe_localities",
|
|
50
|
+
"condepe_series",
|
|
51
|
+
"set_timeout",
|
|
52
|
+
"set_verbose",
|
|
53
|
+
]
|
condepe/_client.py
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""HTTP layer: the canonical Observatory endpoint, with the Hugging Face
|
|
2
|
+
mirror as fallback, and a disk cache keyed by catalogue version.
|
|
3
|
+
|
|
4
|
+
Both backends serve the *same relative paths* (``catalogo.json``,
|
|
5
|
+
``<source>/<source>.parquet``, ``MANIFESTO.sha256``), so a file is always
|
|
6
|
+
addressed by its path in the catalogue. The mirror exists for availability:
|
|
7
|
+
the canonical server is a single machine in Pernambuco, and its network may
|
|
8
|
+
one day restrict access from outside Brazil.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import hashlib
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import shutil
|
|
17
|
+
import time
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any
|
|
20
|
+
from urllib.parse import quote
|
|
21
|
+
|
|
22
|
+
import requests
|
|
23
|
+
from platformdirs import user_cache_dir
|
|
24
|
+
|
|
25
|
+
from . import _msg
|
|
26
|
+
|
|
27
|
+
CANONICAL_URL = "https://cf.pe.gov.br/observatorio/dados/atual"
|
|
28
|
+
MIRROR_URL = "https://huggingface.co/datasets/mlkwy/observatorio-pernambuco/resolve/main"
|
|
29
|
+
USER_AGENT = "condepe Python package (https://github.com/StrategicProjects/condepe-py)"
|
|
30
|
+
|
|
31
|
+
_TIMEOUT = 60.0
|
|
32
|
+
_CATALOG_MAX_AGE = 6 * 3600 # seconds a cached catalogue is trusted without asking
|
|
33
|
+
_TRANSIENT = {429, 500, 502, 503, 504}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class CondepeError(RuntimeError):
|
|
37
|
+
"""Base class for errors raised by condepe."""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class HttpError(CondepeError):
|
|
41
|
+
"""Neither the canonical endpoint nor the mirror answered."""
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class IntegrityError(CondepeError):
|
|
45
|
+
"""A downloaded file does not match the sha256 in the catalogue."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def set_timeout(seconds: float = 60.0) -> None:
|
|
49
|
+
"""Set the timeout, in seconds, of each HTTP request."""
|
|
50
|
+
global _TIMEOUT
|
|
51
|
+
if not isinstance(seconds, (int, float)) or isinstance(seconds, bool) or seconds <= 0:
|
|
52
|
+
raise ValueError("`seconds` must be a positive number.")
|
|
53
|
+
_TIMEOUT = float(seconds)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def base_urls() -> list[str]:
|
|
57
|
+
"""Backends in the order they are tried: canonical, then mirror.
|
|
58
|
+
|
|
59
|
+
``CONDEPE_URL`` and ``CONDEPE_MIRROR_URL`` override them (an empty
|
|
60
|
+
``CONDEPE_MIRROR_URL`` disables the fallback).
|
|
61
|
+
"""
|
|
62
|
+
canonical = os.environ.get("CONDEPE_URL", CANONICAL_URL).rstrip("/")
|
|
63
|
+
mirror = os.environ.get("CONDEPE_MIRROR_URL", MIRROR_URL).rstrip("/")
|
|
64
|
+
return [u for u in (canonical, mirror) if u]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def cache_dir() -> Path:
|
|
68
|
+
"""Directory of the disk cache (``CONDEPE_CACHE_DIR`` overrides it)."""
|
|
69
|
+
return Path(os.environ.get("CONDEPE_CACHE_DIR") or user_cache_dir("condepe"))
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def clear_cache() -> Path:
|
|
73
|
+
"""Delete every cached file and return the cache directory."""
|
|
74
|
+
d = cache_dir()
|
|
75
|
+
if d.exists():
|
|
76
|
+
shutil.rmtree(d)
|
|
77
|
+
_msg.success(f"Cleared condepe cache at {d}.")
|
|
78
|
+
else:
|
|
79
|
+
_msg.inform(f"No condepe cache found at {d}.")
|
|
80
|
+
return d
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _get(url: str) -> bytes:
|
|
84
|
+
"""One GET with up to three tries on transient failures."""
|
|
85
|
+
last: Exception | None = None
|
|
86
|
+
for attempt in range(3):
|
|
87
|
+
try:
|
|
88
|
+
resp = requests.get(url, timeout=_TIMEOUT, headers={"User-Agent": USER_AGENT})
|
|
89
|
+
except requests.RequestException as exc: # connection, DNS, timeout
|
|
90
|
+
last = exc
|
|
91
|
+
else:
|
|
92
|
+
if resp.status_code == 200:
|
|
93
|
+
return resp.content
|
|
94
|
+
last = HttpError(f"HTTP {resp.status_code} from {url}")
|
|
95
|
+
if resp.status_code not in _TRANSIENT:
|
|
96
|
+
break
|
|
97
|
+
if attempt < 2:
|
|
98
|
+
time.sleep(2**attempt)
|
|
99
|
+
raise HttpError(str(last))
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def fetch(path: str) -> bytes:
|
|
103
|
+
"""Fetch a relative path from the first backend that answers."""
|
|
104
|
+
errors = []
|
|
105
|
+
for base in base_urls():
|
|
106
|
+
url = f"{base}/{quote(path)}"
|
|
107
|
+
try:
|
|
108
|
+
return _get(url)
|
|
109
|
+
except HttpError as exc:
|
|
110
|
+
errors.append(f"{base}: {exc}")
|
|
111
|
+
if base != base_urls()[-1]:
|
|
112
|
+
_msg.step("Canonical endpoint unavailable; trying the mirror.")
|
|
113
|
+
raise HttpError("Could not reach the Observatory data. " + "; ".join(errors))
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def catalog_json(refresh: bool = False) -> dict[str, Any]:
|
|
117
|
+
"""The published catalogue, from the disk cache when recent enough."""
|
|
118
|
+
path = cache_dir() / "catalogo.json"
|
|
119
|
+
if not refresh and path.exists() and time.time() - path.stat().st_mtime < _CATALOG_MAX_AGE:
|
|
120
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
121
|
+
try:
|
|
122
|
+
raw = fetch("catalogo.json")
|
|
123
|
+
except HttpError:
|
|
124
|
+
# A stale catalogue is better than none: the files it lists are
|
|
125
|
+
# immutable per version and stay valid in the cache.
|
|
126
|
+
if path.exists():
|
|
127
|
+
_msg.step("Using the cached catalogue; the Observatory could not be reached.")
|
|
128
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
129
|
+
raise
|
|
130
|
+
data = json.loads(raw)
|
|
131
|
+
if "versao" not in data or "fontes" not in data:
|
|
132
|
+
raise CondepeError("Unexpected catalogue format (no `versao`/`fontes`).")
|
|
133
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
134
|
+
path.write_bytes(raw)
|
|
135
|
+
return data
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def cached_file(path: str, sha256: str, version: str, refresh: bool = False) -> Path:
|
|
139
|
+
"""Path to a verified local copy of a catalogue file, downloading if needed."""
|
|
140
|
+
dest = cache_dir() / version / path
|
|
141
|
+
if not refresh and dest.exists() and _sha256(dest.read_bytes()) == sha256:
|
|
142
|
+
return dest
|
|
143
|
+
raw = fetch(path)
|
|
144
|
+
got = _sha256(raw)
|
|
145
|
+
if got != sha256:
|
|
146
|
+
raise IntegrityError(
|
|
147
|
+
f"{path}: sha256 {got[:12]}… differs from the catalogue ({sha256[:12]}…). "
|
|
148
|
+
"The publication may be in progress; try again in a few minutes."
|
|
149
|
+
)
|
|
150
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
151
|
+
tmp = dest.with_suffix(dest.suffix + ".tmp")
|
|
152
|
+
tmp.write_bytes(raw)
|
|
153
|
+
tmp.replace(dest)
|
|
154
|
+
_prune_old_versions(version)
|
|
155
|
+
return dest
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _prune_old_versions(keep: str) -> None:
|
|
159
|
+
"""Drop cached files of other catalogue versions (they are superseded)."""
|
|
160
|
+
root = cache_dir()
|
|
161
|
+
for child in root.iterdir():
|
|
162
|
+
if child.is_dir() and child.name != keep:
|
|
163
|
+
shutil.rmtree(child, ignore_errors=True)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _sha256(raw: bytes) -> str:
|
|
167
|
+
return hashlib.sha256(raw).hexdigest()
|
condepe/_msg.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Lightweight console messages, mirroring the R package's cli alerts.
|
|
2
|
+
|
|
3
|
+
Messages go to stderr and can be silenced with :func:`set_verbose`.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import sys
|
|
9
|
+
|
|
10
|
+
_VERBOSE = True
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def set_verbose(verbose: bool) -> None:
|
|
14
|
+
"""Enable or disable informational messages."""
|
|
15
|
+
global _VERBOSE
|
|
16
|
+
_VERBOSE = bool(verbose)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def success(message: str) -> None:
|
|
20
|
+
if _VERBOSE:
|
|
21
|
+
print(f"✔ {message}", file=sys.stderr)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def step(message: str) -> None:
|
|
25
|
+
if _VERBOSE:
|
|
26
|
+
print(f"ℹ {message}", file=sys.stderr)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def inform(message: str) -> None:
|
|
30
|
+
if _VERBOSE:
|
|
31
|
+
print(message, file=sys.stderr)
|
condepe/data.py
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""Public functions: catalogue, indicators, full sources, series, localities.
|
|
2
|
+
|
|
3
|
+
Every source of the Observatory is published in the same long format (one
|
|
4
|
+
observation per row, 15 columns). Column names are kept in Portuguese, as
|
|
5
|
+
published, so results stay traceable to the official files.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Iterable
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
import pandas as pd
|
|
14
|
+
import pyarrow.parquet as pq
|
|
15
|
+
|
|
16
|
+
from . import _client, _msg
|
|
17
|
+
|
|
18
|
+
CITATION = (
|
|
19
|
+
"Condepe/Fidem, Observatório de Pernambuco — Base de Dados do Estado, "
|
|
20
|
+
"version {version}, https://cf.pe.gov.br/observatorio (CC BY 4.0)."
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
_CATALOG_COLUMNS = [
|
|
24
|
+
"id_fonte",
|
|
25
|
+
"eixo",
|
|
26
|
+
"tema",
|
|
27
|
+
"assunto",
|
|
28
|
+
"descricao",
|
|
29
|
+
"fonte_original",
|
|
30
|
+
"linhas",
|
|
31
|
+
"indicadores",
|
|
32
|
+
"localidades",
|
|
33
|
+
"ano_inicio",
|
|
34
|
+
"ano_fim",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def condepe_catalog(refresh: bool = False) -> pd.DataFrame:
|
|
39
|
+
"""List the sources published by the Observatory.
|
|
40
|
+
|
|
41
|
+
Parameters
|
|
42
|
+
----------
|
|
43
|
+
refresh:
|
|
44
|
+
Ignore the cached catalogue (trusted for 6 hours) and ask the server.
|
|
45
|
+
|
|
46
|
+
Returns
|
|
47
|
+
-------
|
|
48
|
+
pandas.DataFrame
|
|
49
|
+
One row per source (``id_fonte``) with axis, theme, description,
|
|
50
|
+
original publisher, number of rows, indicators and localities, and the
|
|
51
|
+
years covered. ``attrs["version"]`` holds the catalogue version.
|
|
52
|
+
"""
|
|
53
|
+
cat = _client.catalog_json(refresh=refresh)
|
|
54
|
+
df = pd.DataFrame([{k: f.get(k) for k in _CATALOG_COLUMNS} for f in cat["fontes"]])
|
|
55
|
+
df.attrs["version"] = cat["versao"]
|
|
56
|
+
df.attrs["citation"] = CITATION.format(version=cat["versao"])
|
|
57
|
+
return df
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _source_entry(source: str, cat: dict[str, Any]) -> dict[str, Any]:
|
|
61
|
+
for f in cat["fontes"]:
|
|
62
|
+
if f["id_fonte"] == source:
|
|
63
|
+
return f
|
|
64
|
+
raise ValueError(
|
|
65
|
+
f"Unknown source {source!r}. See condepe_catalog() for the {len(cat['fontes'])} available."
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def condepe_data(source: str, refresh: bool = False) -> pd.DataFrame:
|
|
70
|
+
"""Download a whole source (all indicators, localities and periods).
|
|
71
|
+
|
|
72
|
+
The Parquet file is kept in the disk cache for the current catalogue
|
|
73
|
+
version and verified against the sha256 published in the catalogue.
|
|
74
|
+
|
|
75
|
+
Parameters
|
|
76
|
+
----------
|
|
77
|
+
source:
|
|
78
|
+
Source id, e.g. ``"sidra_populacao_estimada"`` (see
|
|
79
|
+
:func:`condepe_catalog`).
|
|
80
|
+
refresh:
|
|
81
|
+
Download again even if a verified copy is cached.
|
|
82
|
+
|
|
83
|
+
Returns
|
|
84
|
+
-------
|
|
85
|
+
pandas.DataFrame
|
|
86
|
+
The 15 columns of the Observatory long format: ``cd_municipio_ibge``,
|
|
87
|
+
``nm_municipio``, ``ano``, ``mes``, ``indicador``, ``indicador_base``,
|
|
88
|
+
``valor``, ``unidade``, ``natureza``, ``id_fonte``, ``eixo``, ``tema``,
|
|
89
|
+
``assunto``, ``fonte_original``, ``arquivo_origem``.
|
|
90
|
+
"""
|
|
91
|
+
cat = _client.catalog_json(refresh=refresh)
|
|
92
|
+
entry = _source_entry(source, cat)
|
|
93
|
+
parquet = next((a for a in entry.get("arquivos", []) if a.get("formato") == "parquet"), None)
|
|
94
|
+
if parquet is None:
|
|
95
|
+
raise _client.CondepeError(f"Source {source!r} has no Parquet file in the catalogue.")
|
|
96
|
+
path = _client.cached_file(
|
|
97
|
+
parquet["caminho"], parquet["sha256"], cat["versao"], refresh=refresh
|
|
98
|
+
)
|
|
99
|
+
df = pq.read_table(path).to_pandas()
|
|
100
|
+
df.attrs["version"] = cat["versao"]
|
|
101
|
+
df.attrs["source"] = source
|
|
102
|
+
_msg.success(f"{source}: {len(df):,} observations (version {cat['versao']}).")
|
|
103
|
+
return df
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def condepe_indicators(source: str, refresh: bool = False) -> pd.DataFrame:
|
|
107
|
+
"""List the indicators of a source, with unit, coverage and periodicity.
|
|
108
|
+
|
|
109
|
+
Parameters
|
|
110
|
+
----------
|
|
111
|
+
source:
|
|
112
|
+
Source id (see :func:`condepe_catalog`).
|
|
113
|
+
refresh:
|
|
114
|
+
Download the source again even if a verified copy is cached.
|
|
115
|
+
|
|
116
|
+
Returns
|
|
117
|
+
-------
|
|
118
|
+
pandas.DataFrame
|
|
119
|
+
One row per ``indicador_base`` with ``unidade``, ``natureza``,
|
|
120
|
+
``mensal`` (monthly series), number of observations and localities,
|
|
121
|
+
and first/last year.
|
|
122
|
+
"""
|
|
123
|
+
df = condepe_data(source, refresh=refresh)
|
|
124
|
+
out = (
|
|
125
|
+
df.groupby("indicador_base", sort=True)
|
|
126
|
+
.agg(
|
|
127
|
+
unidade=("unidade", "first"),
|
|
128
|
+
natureza=("natureza", "first"),
|
|
129
|
+
mensal=("mes", lambda s: bool(s.notna().any())),
|
|
130
|
+
observacoes=("valor", "size"),
|
|
131
|
+
localidades=("cd_municipio_ibge", "nunique"),
|
|
132
|
+
ano_inicio=("ano", "min"),
|
|
133
|
+
ano_fim=("ano", "max"),
|
|
134
|
+
)
|
|
135
|
+
.reset_index()
|
|
136
|
+
)
|
|
137
|
+
out.attrs = dict(df.attrs)
|
|
138
|
+
return out
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _as_list(x: str | Iterable[str] | None) -> list[str] | None:
|
|
142
|
+
if x is None:
|
|
143
|
+
return None
|
|
144
|
+
return [x] if isinstance(x, str) else [str(v) for v in x]
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def condepe_series(
|
|
148
|
+
source: str,
|
|
149
|
+
indicator: str | Iterable[str],
|
|
150
|
+
code: str | Iterable[str] | None = None,
|
|
151
|
+
years: int | Iterable[int] | None = None,
|
|
152
|
+
refresh: bool = False,
|
|
153
|
+
) -> pd.DataFrame:
|
|
154
|
+
"""Series of one or more indicators, optionally filtered by place and year.
|
|
155
|
+
|
|
156
|
+
Parameters
|
|
157
|
+
----------
|
|
158
|
+
source:
|
|
159
|
+
Source id (see :func:`condepe_catalog`).
|
|
160
|
+
indicator:
|
|
161
|
+
One or more ``indicador_base`` values, exactly as listed by
|
|
162
|
+
:func:`condepe_indicators`.
|
|
163
|
+
code:
|
|
164
|
+
IBGE codes: 7 digits for municipalities, ``"26"`` for the State.
|
|
165
|
+
``None`` keeps every locality.
|
|
166
|
+
years:
|
|
167
|
+
A year or a collection of years; ``None`` keeps all.
|
|
168
|
+
|
|
169
|
+
Returns
|
|
170
|
+
-------
|
|
171
|
+
pandas.DataFrame
|
|
172
|
+
The matching observations, sorted by locality, year and month.
|
|
173
|
+
"""
|
|
174
|
+
df = condepe_data(source, refresh=refresh)
|
|
175
|
+
wanted = _as_list(indicator) or []
|
|
176
|
+
missing = sorted(set(wanted) - set(df["indicador_base"].unique()))
|
|
177
|
+
if missing:
|
|
178
|
+
raise ValueError(
|
|
179
|
+
f"Indicator(s) not in {source!r}: {missing}. See condepe_indicators({source!r})."
|
|
180
|
+
)
|
|
181
|
+
keep = df["indicador_base"].isin(wanted)
|
|
182
|
+
codes = _as_list(code)
|
|
183
|
+
if codes is not None:
|
|
184
|
+
keep &= df["cd_municipio_ibge"].isin(codes)
|
|
185
|
+
if years is not None:
|
|
186
|
+
ys = [years] if isinstance(years, int) else list(years)
|
|
187
|
+
keep &= df["ano"].isin(ys)
|
|
188
|
+
out = df[keep].sort_values(["cd_municipio_ibge", "ano", "mes"], na_position="first")
|
|
189
|
+
out = out.reset_index(drop=True)
|
|
190
|
+
out.attrs = dict(df.attrs)
|
|
191
|
+
if out.empty:
|
|
192
|
+
_msg.step("No observations match these filters.")
|
|
193
|
+
return out
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def condepe_localities(refresh: bool = False) -> pd.DataFrame:
|
|
197
|
+
"""The 185 municipalities of Pernambuco and the State, with IBGE codes.
|
|
198
|
+
|
|
199
|
+
Taken from the population estimates source, which covers every
|
|
200
|
+
municipality (Fernando de Noronha included); the State (``"26"``) comes
|
|
201
|
+
first.
|
|
202
|
+
|
|
203
|
+
Parameters
|
|
204
|
+
----------
|
|
205
|
+
refresh:
|
|
206
|
+
Download the source again even if a verified copy is cached.
|
|
207
|
+
|
|
208
|
+
Returns
|
|
209
|
+
-------
|
|
210
|
+
pandas.DataFrame
|
|
211
|
+
``cd_municipio_ibge`` and ``nm_municipio``.
|
|
212
|
+
"""
|
|
213
|
+
df = condepe_data("sidra_populacao_estimada", refresh=refresh)
|
|
214
|
+
out = (
|
|
215
|
+
df[["cd_municipio_ibge", "nm_municipio"]]
|
|
216
|
+
.drop_duplicates()
|
|
217
|
+
.sort_values("cd_municipio_ibge")
|
|
218
|
+
.reset_index(drop=True)
|
|
219
|
+
)
|
|
220
|
+
if not (out["cd_municipio_ibge"] == "26").any():
|
|
221
|
+
out = pd.concat(
|
|
222
|
+
[pd.DataFrame({"cd_municipio_ibge": ["26"], "nm_municipio": ["Pernambuco"]}), out],
|
|
223
|
+
ignore_index=True,
|
|
224
|
+
)
|
|
225
|
+
out.attrs = dict(df.attrs)
|
|
226
|
+
return out
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: condepe
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Access the open data of the Pernambuco Observatory (Condepe/Fidem) from Python
|
|
5
|
+
Project-URL: Homepage, https://strategicprojects.github.io/condepe-py/
|
|
6
|
+
Project-URL: Data, https://cf.pe.gov.br/observatorio
|
|
7
|
+
Project-URL: Repository, https://github.com/StrategicProjects/condepe-py
|
|
8
|
+
Project-URL: Issues, https://github.com/StrategicProjects/condepe-py/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/StrategicProjects/condepe-py/blob/main/NEWS.md
|
|
10
|
+
Author-email: André Leite <leite@castlab.org>, Marcos Wasiliew <marcos.wasiliew@gmail.com>, Hugo Vasconcelos <hugo.vasconcelos@ufpe.br>, Carlos Amorim <carlos.agaf@ufpe.br>, Diogo Bezerra <diogo.bezerra@ufpe.br>, Júlia Nascimento Barreto <juliabarreto@gd.seplag.pe.gov.br>
|
|
11
|
+
License: MIT License
|
|
12
|
+
|
|
13
|
+
Copyright (c) 2026 condepe authors
|
|
14
|
+
|
|
15
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
16
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
17
|
+
in the Software without restriction, including without limitation the rights
|
|
18
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
19
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
20
|
+
furnished to do so, subject to the following conditions:
|
|
21
|
+
|
|
22
|
+
The above copyright notice and this permission notice shall be included in all
|
|
23
|
+
copies or substantial portions of the Software.
|
|
24
|
+
|
|
25
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
26
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
27
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
28
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
29
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
30
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
31
|
+
SOFTWARE.
|
|
32
|
+
License-File: LICENSE
|
|
33
|
+
Keywords: Brazil,Condepe/Fidem,Pernambuco,municipalities,open-data,pandas,statistics
|
|
34
|
+
Classifier: Development Status :: 4 - Beta
|
|
35
|
+
Classifier: Intended Audience :: Science/Research
|
|
36
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
37
|
+
Classifier: Operating System :: OS Independent
|
|
38
|
+
Classifier: Programming Language :: Python :: 3
|
|
39
|
+
Classifier: Topic :: Scientific/Engineering
|
|
40
|
+
Requires-Python: >=3.10
|
|
41
|
+
Requires-Dist: pandas>=1.3
|
|
42
|
+
Requires-Dist: platformdirs>=3
|
|
43
|
+
Requires-Dist: pyarrow>=10
|
|
44
|
+
Requires-Dist: requests>=2.25
|
|
45
|
+
Provides-Extra: dev
|
|
46
|
+
Requires-Dist: build>=1.0; extra == 'dev'
|
|
47
|
+
Requires-Dist: pytest>=7; extra == 'dev'
|
|
48
|
+
Requires-Dist: responses>=0.23; extra == 'dev'
|
|
49
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
50
|
+
Provides-Extra: docs
|
|
51
|
+
Requires-Dist: matplotlib>=3.5; extra == 'docs'
|
|
52
|
+
Requires-Dist: mkdocs-material>=9.5; extra == 'docs'
|
|
53
|
+
Requires-Dist: mkdocstrings[python]>=0.26; extra == 'docs'
|
|
54
|
+
Provides-Extra: test
|
|
55
|
+
Requires-Dist: pytest>=7; extra == 'test'
|
|
56
|
+
Requires-Dist: responses>=0.23; extra == 'test'
|
|
57
|
+
Description-Content-Type: text/markdown
|
|
58
|
+
|
|
59
|
+
# condepe <img src="docs/assets/logo.svg" align="right" height="139" alt="condepe hex logo" />
|
|
60
|
+
|
|
61
|
+
Python access to the open data of the **Observatório de Pernambuco**, published by
|
|
62
|
+
[Condepe/Fidem](https://cf.pe.gov.br) (the Pernambuco State Agency for Planning and
|
|
63
|
+
Research): municipal series on population, economy, public finance, labour,
|
|
64
|
+
education, health, security, transport and environment for the 185
|
|
65
|
+
municipalities of Pernambuco, Brazil.
|
|
66
|
+
|
|
67
|
+
Every source is published in the same long format (one observation per row, 15
|
|
68
|
+
columns), so one set of functions covers all of them.
|
|
69
|
+
|
|
70
|
+
## Installation
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
pip install condepe
|
|
74
|
+
# development version:
|
|
75
|
+
pip install git+https://github.com/StrategicProjects/condepe-py
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## Usage
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
import condepe
|
|
82
|
+
|
|
83
|
+
# Sources available (one row each), with the catalogue version
|
|
84
|
+
cat = condepe.condepe_catalog()
|
|
85
|
+
cat.attrs["version"]
|
|
86
|
+
|
|
87
|
+
# Indicators of a source
|
|
88
|
+
condepe.condepe_indicators("sidra_populacao_estimada")
|
|
89
|
+
|
|
90
|
+
# A series: Recife (IBGE 2611606), 2020 onwards
|
|
91
|
+
condepe.condepe_series(
|
|
92
|
+
"sidra_populacao_estimada",
|
|
93
|
+
"Populacao residente estimada",
|
|
94
|
+
code="2611606",
|
|
95
|
+
years=range(2020, 2027),
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
# A whole source, and the municipality list
|
|
99
|
+
condepe.condepe_data("datasus_sinasc")
|
|
100
|
+
condepe.condepe_localities()
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Column names are kept in Portuguese, as published, so results stay traceable to
|
|
104
|
+
the official files: `cd_municipio_ibge`, `nm_municipio`, `ano`, `mes`,
|
|
105
|
+
`indicador`, `indicador_base`, `valor`, `unidade`, `natureza`, `id_fonte`,
|
|
106
|
+
`eixo`, `tema`, `assunto`, `fonte_original`, `arquivo_origem`. Monthly series
|
|
107
|
+
carry the period in `indicador` (`"… [MM/YYYY]"`) and `mes`; `indicador_base` is
|
|
108
|
+
the label without it.
|
|
109
|
+
|
|
110
|
+
## Where the data come from
|
|
111
|
+
|
|
112
|
+
- **Canonical:** <https://cf.pe.gov.br/observatorio/dados/atual/>
|
|
113
|
+
- **Mirror (fallback):** the Hugging Face dataset
|
|
114
|
+
[`mlkwy/observatorio-pernambuco`](https://huggingface.co/datasets/mlkwy/observatorio-pernambuco),
|
|
115
|
+
with the same relative paths.
|
|
116
|
+
|
|
117
|
+
Files are cached on disk per catalogue version (`condepe_cache_dir()`) and
|
|
118
|
+
checked against the sha256 published in the catalogue. The catalogue itself is
|
|
119
|
+
trusted for 6 hours. Set `CONDEPE_URL`, `CONDEPE_MIRROR_URL` or
|
|
120
|
+
`CONDEPE_CACHE_DIR` to override the defaults.
|
|
121
|
+
|
|
122
|
+
Documentation: <https://strategicprojects.github.io/condepe-py/>.
|
|
123
|
+
|
|
124
|
+
## Citation and license
|
|
125
|
+
|
|
126
|
+
Data: Condepe/Fidem, *Observatório de Pernambuco — Base de Dados do Estado*,
|
|
127
|
+
[CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). The citation with the
|
|
128
|
+
version in use is in `condepe_catalog().attrs["citation"]`.
|
|
129
|
+
|
|
130
|
+
Code: MIT. To read or write *private* buckets on `s3.pe.gov.br` with an
|
|
131
|
+
access key, see [`s3pe`](https://github.com/StrategicProjects/s3pe); `condepe`
|
|
132
|
+
needs no key, since the Observatory open data are public. An R package with the same functions is
|
|
133
|
+
[`condepe` for R](https://github.com/StrategicProjects/condepe-r).
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
condepe/__init__.py,sha256=h4l6WNPK8xVfeGTkC8sAg--Rsol-2Z7tBE02kpx7WH4,1466
|
|
2
|
+
condepe/_client.py,sha256=Du8oILzWB7CE70gv2BVLhWtZeAfezdN11iJB-mWR5hA,5956
|
|
3
|
+
condepe/_msg.py,sha256=C1jJiz_A72uWSw7Zf56Vf6mEkgc3kQxz6tQQjxsmEbA,648
|
|
4
|
+
condepe/data.py,sha256=IEBGM701am-FAEQfbm1n7guaW8-BNs16Qr73SBto8v0,7292
|
|
5
|
+
condepe-0.1.0.dist-info/METADATA,sha256=grWJGqyC1ew9uNXz0GPvUC2Rn1fgFRx_-y7su5b9Zko,5914
|
|
6
|
+
condepe-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
7
|
+
condepe-0.1.0.dist-info/licenses/LICENSE,sha256=I0Xb_g3ZGRlzoO6Ggc7XzutyHUVBtCa8SLtfQiltGdA,1072
|
|
8
|
+
condepe-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 condepe authors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|