misocialpy 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- misocialpy-0.4.0/.gitignore +25 -0
- misocialpy-0.4.0/LICENSE +21 -0
- misocialpy-0.4.0/NEWS.md +56 -0
- misocialpy-0.4.0/PKG-INFO +170 -0
- misocialpy-0.4.0/README.md +119 -0
- misocialpy-0.4.0/pyproject.toml +53 -0
- misocialpy-0.4.0/src/misocialpy/__init__.py +66 -0
- misocialpy-0.4.0/src/misocialpy/_cache.py +83 -0
- misocialpy-0.4.0/src/misocialpy/_client.py +162 -0
- misocialpy-0.4.0/src/misocialpy/_format.py +91 -0
- misocialpy-0.4.0/src/misocialpy/_msg.py +31 -0
- misocialpy-0.4.0/src/misocialpy/_response.py +170 -0
- misocialpy-0.4.0/src/misocialpy/data/dictionary.csv +133 -0
- misocialpy-0.4.0/src/misocialpy/data/themes.csv +8 -0
- misocialpy-0.4.0/src/misocialpy/data.py +297 -0
- misocialpy-0.4.0/src/misocialpy/fields.py +151 -0
- misocialpy-0.4.0/src/misocialpy/localities.py +312 -0
- misocialpy-0.4.0/src/misocialpy/periods.py +179 -0
- misocialpy-0.4.0/src/misocialpy/publication.py +191 -0
- misocialpy-0.4.0/src/misocialpy/validation.py +103 -0
- misocialpy-0.4.0/tests/conftest.py +97 -0
- misocialpy-0.4.0/tests/fixtures/bpc_suas_recife_202509.json +32 -0
- misocialpy-0.4.0/tests/fixtures/data_recife_jaboatao.json +48 -0
- misocialpy-0.4.0/tests/fixtures/field_check_missing.json +11 -0
- misocialpy-0.4.0/tests/fixtures/localities_202512.json +33441 -0
- misocialpy-0.4.0/tests/fixtures/parity_cases.json +654 -0
- misocialpy-0.4.0/tests/fixtures/periods_bpc_pe.json +312 -0
- misocialpy-0.4.0/tests/fixtures/periods_bpc_pe_nonzero.json +296 -0
- misocialpy-0.4.0/tests/test_api.py +68 -0
- misocialpy-0.4.0/tests/test_data.py +106 -0
- misocialpy-0.4.0/tests/test_final.py +192 -0
- misocialpy-0.4.0/tests/test_format.py +30 -0
- misocialpy-0.4.0/tests/test_localities.py +54 -0
- misocialpy-0.4.0/tests/test_paridade_casos.py +95 -0
- misocialpy-0.4.0/tests/test_parity.py +16 -0
- misocialpy-0.4.0/tests/test_periods.py +25 -0
- misocialpy-0.4.0/tests/test_profunda.py +390 -0
- misocialpy-0.4.0/tests/test_publication.py +63 -0
- misocialpy-0.4.0/tests/test_revisao.py +238 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
.DS_Store
|
|
2
|
+
.Rproj.user
|
|
3
|
+
CLAUDE.md
|
|
4
|
+
CLAUDE.local.md
|
|
5
|
+
.claude/
|
|
6
|
+
plano/
|
|
7
|
+
plano_misocial.html
|
|
8
|
+
REVISAO_*.md
|
|
9
|
+
ANALISE_*.md
|
|
10
|
+
RELATORIO_*.md
|
|
11
|
+
ESTADO_*.md
|
|
12
|
+
misocialr/docs/
|
|
13
|
+
.Rhistory
|
|
14
|
+
*.Rcheck/
|
|
15
|
+
misocialr_*.tar.gz
|
|
16
|
+
__pycache__/
|
|
17
|
+
*.pyc
|
|
18
|
+
.pytest_cache/
|
|
19
|
+
.ruff_cache/
|
|
20
|
+
dist/
|
|
21
|
+
build/
|
|
22
|
+
*.egg-info/
|
|
23
|
+
.venv/
|
|
24
|
+
tests/testthat/api.bak/
|
|
25
|
+
tests/testthat/_problems/
|
misocialpy-0.4.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 misocialpy authors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
misocialpy-0.4.0/NEWS.md
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# misocialpy 0.4.0
|
|
2
|
+
|
|
3
|
+
Same changes as misocialr 0.4.0: paging by `cursorMark`, `nature` and
|
|
4
|
+
`nature_basis` columns in `mds_fields()`, corrected label of the Fomento
|
|
5
|
+
Rural `..._ano_f` field and `set_cache_ttl(seconds)`.
|
|
6
|
+
|
|
7
|
+
# misocialpy 0.3.1
|
|
8
|
+
|
|
9
|
+
Same fixes as misocialr 0.3.1, plus the Python-specific ones:
|
|
10
|
+
|
|
11
|
+
* Results of `mds_periods()`, `mds_api_fields()` and `mds_localities()` are
|
|
12
|
+
copies: editing a result no longer changes the cache (F1).
|
|
13
|
+
* `mds_drop_unpublished()`: a fractional period (`202401.9`) and a missing
|
|
14
|
+
`field` (`pd.NA`) are errors; `today=None` is the only default (F3).
|
|
15
|
+
* `set_timeout()` requires a finite positive number and keeps the previous
|
|
16
|
+
value on error (F7).
|
|
17
|
+
* Extras `test` (pytest, responses) and `dev` (plus ruff and build) (F8).
|
|
18
|
+
* Numpy scalars and integer-valued numbers (`26.0`) are accepted, as in R.
|
|
19
|
+
* Three attempts in total, as in R (previously 4); `Retry-After` capped at
|
|
20
|
+
30 s; retries allowed on POST.
|
|
21
|
+
* `LICENSE` with the full MIT text (previously only the CRAN template).
|
|
22
|
+
* The `live` mark is registered in `conftest.py`: the suite behaves the
|
|
23
|
+
same outside the project (tested against the wheel installed in a clean
|
|
24
|
+
environment).
|
|
25
|
+
|
|
26
|
+
# misocialpy 0.3.0
|
|
27
|
+
|
|
28
|
+
Same fixes as misocialr 0.3.0: single response contract, no caching of incomplete
|
|
29
|
+
responses; `mds_localities()` paged and validated, with `period` and
|
|
30
|
+
`refresh`; `mds_drop_unpublished()` never deleting a non-zero value, with a
|
|
31
|
+
normalised period and `action="flag"`; identity and scalar value per
|
|
32
|
+
document; conflicts as `DataConflictError`; consistent paging; facets in
|
|
33
|
+
disjoint blocks; `refresh` and canonical cache keys; `HttpError`,
|
|
34
|
+
`ResponseError` and `DataConflictError`, all inheriting from `MdsError`
|
|
35
|
+
(and `ValidationError` also from `ValueError`); `storage_type`; local
|
|
36
|
+
validation with `validate=False`; warning for a municipality without
|
|
37
|
+
documents.
|
|
38
|
+
|
|
39
|
+
# misocialpy 0.2.1
|
|
40
|
+
|
|
41
|
+
Same fixes as misocialr 0.2.1: locality and period validated (empty, `None`, `NaN`,
|
|
42
|
+
non-integer, wrong check digit, invalid next to `"BR"`), text field as a
|
|
43
|
+
`ValidationError` (previously a `ValueError` without context), incomplete
|
|
44
|
+
or partial response as an `MdsError`, `mds_api_fields()` paged,
|
|
45
|
+
`mds_drop_unpublished()` with validation and `reference="brazil"`,
|
|
46
|
+
`set_timeout()` and `df.attrs["retrieved_at"]`. Minimum versions of pandas
|
|
47
|
+
and requests tested.
|
|
48
|
+
|
|
49
|
+
# misocialpy 0.2.0
|
|
50
|
+
|
|
51
|
+
Four new themes and shortcuts `mds_suas_rede()`, `mds_suas_financiamento()`,
|
|
52
|
+
`mds_pbf_condicionalidades()` and `mds_seguranca_alimentar()`.
|
|
53
|
+
|
|
54
|
+
# misocialpy 0.1.0
|
|
55
|
+
|
|
56
|
+
First version, a port of misocialr 0.1.0.
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: misocialpy
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: Access the Brazilian MDS Social Information Matrix (MI Social) API from Python
|
|
5
|
+
Project-URL: Homepage, https://github.com/StrategicProjects/misocial
|
|
6
|
+
Project-URL: Repository, https://github.com/StrategicProjects/misocial
|
|
7
|
+
Project-URL: Issues, https://github.com/StrategicProjects/misocial/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/StrategicProjects/misocial/blob/main/misocialpy/NEWS.md
|
|
9
|
+
Author-email: André Leite <leite@castlab.org>, Marcos Wasiliew <marcos.wasiliew@gmail.com>, Hugo Vasconcelos <hugo.vasconcelos@ufpe.br>, Carlos Amorim <carlos.agaf@ufpe.br>, Diogo Bezerra <diogo.bezerra@ufpe.br>
|
|
10
|
+
License: MIT License
|
|
11
|
+
|
|
12
|
+
Copyright (c) 2026 misocialpy authors
|
|
13
|
+
|
|
14
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
15
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
16
|
+
in the Software without restriction, including without limitation the rights
|
|
17
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
18
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
19
|
+
furnished to do so, subject to the following conditions:
|
|
20
|
+
|
|
21
|
+
The above copyright notice and this permission notice shall be included in all
|
|
22
|
+
copies or substantial portions of the Software.
|
|
23
|
+
|
|
24
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
25
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
26
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
27
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
28
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
29
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
30
|
+
SOFTWARE.
|
|
31
|
+
License-File: LICENSE
|
|
32
|
+
Keywords: BPC,Bolsa Familia,Brazil,Cadastro Unico,MDS,SUAS,open-data,pandas
|
|
33
|
+
Classifier: Development Status :: 4 - Beta
|
|
34
|
+
Classifier: Intended Audience :: Science/Research
|
|
35
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
36
|
+
Classifier: Operating System :: OS Independent
|
|
37
|
+
Classifier: Programming Language :: Python :: 3
|
|
38
|
+
Classifier: Topic :: Scientific/Engineering
|
|
39
|
+
Requires-Python: >=3.9
|
|
40
|
+
Requires-Dist: pandas>=1.3
|
|
41
|
+
Requires-Dist: requests>=2.25
|
|
42
|
+
Provides-Extra: dev
|
|
43
|
+
Requires-Dist: build>=1.0; extra == 'dev'
|
|
44
|
+
Requires-Dist: pytest>=7; extra == 'dev'
|
|
45
|
+
Requires-Dist: responses>=0.23; extra == 'dev'
|
|
46
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
47
|
+
Provides-Extra: test
|
|
48
|
+
Requires-Dist: pytest>=7; extra == 'test'
|
|
49
|
+
Requires-Dist: responses>=0.23; extra == 'test'
|
|
50
|
+
Description-Content-Type: text/markdown
|
|
51
|
+
|
|
52
|
+
# misocialpy
|
|
53
|
+
|
|
54
|
+
<!-- badges: start -->
|
|
55
|
+

|
|
56
|
+

|
|
57
|
+
[](https://github.com/StrategicProjects/misocial/actions/workflows/misocialpy.yaml)
|
|
58
|
+
<!-- badges: end -->
|
|
59
|
+
|
|
60
|
+
Pandas-friendly Python interface to the **MI Social** API (Matriz de
|
|
61
|
+
Informação Social) of the Brazilian Ministry of Development and Social
|
|
62
|
+
Assistance (MDS/SAGI):
|
|
63
|
+
<https://aplicacoes.mds.gov.br/sagi/servicos/misocial>.
|
|
64
|
+
|
|
65
|
+
A port of the R package
|
|
66
|
+
[`misocialr`](https://github.com/StrategicProjects/misocial/tree/main/misocialr),
|
|
67
|
+
with the same
|
|
68
|
+
functions, name by name, as `ibgepy` does with `ibger`. Both versions read
|
|
69
|
+
the same field dictionary (`src/misocialpy/data/dictionary.csv`, a copy of
|
|
70
|
+
`misocialr/inst/extdata/`) and are tested against the same recorded
|
|
71
|
+
responses.
|
|
72
|
+
|
|
73
|
+
| Theme | Function | Content |
|
|
74
|
+
| --- | --- | --- |
|
|
75
|
+
| `cadunico` | `mds_cadunico()` | Cadastro Único: registered families and people, by income bracket |
|
|
76
|
+
| `bolsa_familia` | `mds_bolsa_familia()` | Bolsa Família programme: beneficiary families, benefits and transfers |
|
|
77
|
+
| `bpc_suas` | `mds_bpc_suas()` | BPC benefit and the social assistance network (CRAS, CREAS) |
|
|
78
|
+
| `suas_rede` | `mds_suas_rede()` | SUAS network (CadSUAS): CRAS, CREAS, Centro POP, day centres, shelters and staff per unit |
|
|
79
|
+
| `suas_financiamento` | `mds_suas_financiamento()` | FNAS transfers and balances of the municipal funds, and IGD-M |
|
|
80
|
+
| `pbf_condicionalidades` | `mds_pbf_condicionalidades()` | Education and health conditionality monitoring (SICON) and sanctions |
|
|
81
|
+
| `seguranca_alimentar` | `mds_seguranca_alimentar()` | PAA, Fomento Rural and cisterns |
|
|
82
|
+
|
|
83
|
+
Other functions: `mds_data()`, `mds_themes()`, `mds_fields()`,
|
|
84
|
+
`mds_api_fields()`, `mds_periods()`, `mds_localities()`,
|
|
85
|
+
`mds_drop_unpublished()`, `ibge_code7()`, `mds_clear_cache()`,
|
|
86
|
+
`set_verbose()`.
|
|
87
|
+
|
|
88
|
+
## Installation
|
|
89
|
+
|
|
90
|
+
```sh
|
|
91
|
+
pip install "git+https://github.com/StrategicProjects/misocial#subdirectory=misocialpy"
|
|
92
|
+
|
|
93
|
+
# or, from a local clone:
|
|
94
|
+
pip install ./misocialpy # from the repository root
|
|
95
|
+
pip install . # from inside misocialpy/
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Requires Python >= 3.9, pandas >= 1.3 and requests >= 2.25 (tested with
|
|
99
|
+
pandas 1.3.5 and requests 2.25.1 on Python 3.9, and with pandas 3.0.6 on
|
|
100
|
+
Python 3.14).
|
|
101
|
+
|
|
102
|
+
## Usage
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
import misocialpy as m
|
|
106
|
+
|
|
107
|
+
m.mds_bolsa_familia(
|
|
108
|
+
periods=[202508, 202509],
|
|
109
|
+
localities=2611606, # Recife (6 or 7 digits)
|
|
110
|
+
fields=["qtd_familias_beneficiarias_bolsa_familia_i", "valor_repassado_bolsa_familia_f"],
|
|
111
|
+
)
|
|
112
|
+
# ibge_code locality_name period field unit value
|
|
113
|
+
# 0 2611606 RECIFE 202508 qtd_familias_beneficiarias_bolsa_familia_i Famílias 140350.0
|
|
114
|
+
# 1 2611606 RECIFE 202509 qtd_familias_beneficiarias_bolsa_familia_i Famílias 139452.0
|
|
115
|
+
# 2 2611606 RECIFE 202508 valor_repassado_bolsa_familia_f R$ 92124240.0
|
|
116
|
+
# 3 2611606 RECIFE 202509 valor_repassado_bolsa_familia_f R$ 93354456.0
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
(Selected columns; output from 2026-10-19.)
|
|
120
|
+
|
|
121
|
+
- `periods`: `None` (last 12 months), `-n`, years (`2024`) and/or `YYYYMM`.
|
|
122
|
+
- `localities`: `"BR"`, states (`"PE"` or `26`), municipalities with 6 or 7
|
|
123
|
+
digits.
|
|
124
|
+
|
|
125
|
+
The API answers **0** for months not yet published. `mds_periods()` shows
|
|
126
|
+
how many municipalities have a non-zero value in each month, and
|
|
127
|
+
`mds_drop_unpublished()` explicitly removes the trailing all-zero months;
|
|
128
|
+
`reference="brazil"` decides by the whole country and preserves legitimate
|
|
129
|
+
zeros of small selections.
|
|
130
|
+
|
|
131
|
+
Validation is the same as in the R package. Invalid input is a
|
|
132
|
+
`ValidationError`; an incomplete or malformed response (partial result,
|
|
133
|
+
missing facet, total that changes while paging, repeated document, no
|
|
134
|
+
municipality or several values) is a `ResponseError`; two values for the
|
|
135
|
+
same observation are a `DataConflictError`; a network or HTTP failure is an
|
|
136
|
+
`HttpError` (with `status`). All inherit from `MdsError`. None of this
|
|
137
|
+
enters the cache.
|
|
138
|
+
|
|
139
|
+
`mds_drop_unpublished()` never deletes a non-zero value; `action="flag"`
|
|
140
|
+
classifies the rows (`publication_status`, `publication_reason`,
|
|
141
|
+
`publication_reference`) without removing any.
|
|
142
|
+
|
|
143
|
+
- `misocialpy.set_timeout(300)`: maximum time per request (default 180 s);
|
|
144
|
+
`misocialpy.set_verbose(False)` hides the messages.
|
|
145
|
+
- `df.attrs["retrieved_at"]`: date and time (UTC) of the query.
|
|
146
|
+
- `refresh=True` in `mds_periods()`, `mds_localities()` and
|
|
147
|
+
`mds_api_fields()`; `mds_localities(period=)` picks the month of the
|
|
148
|
+
list. Results are copies: editing them does not change the cache.
|
|
149
|
+
- `misocialpy.set_cache_ttl(3600)`: cache validity in seconds (default: the
|
|
150
|
+
whole session).
|
|
151
|
+
- `mds_fields()` brings `nature` (`estoque`, `fluxo`, `fluxo anual`,
|
|
152
|
+
`acumulado`, `taxa`, `média`, `índice`) and `nature_basis`; see the
|
|
153
|
+
summing table in the R README.
|
|
154
|
+
- Paging by `cursorMark`.
|
|
155
|
+
- Queries by POST (avoids the HTTP 414 of long URLs); 3 attempts in total
|
|
156
|
+
on 429/5xx and on connection failure, with `Retry-After` capped at 30 s.
|
|
157
|
+
- Accepts numpy scalars (`np.int64(202401)`) and integer-valued numbers
|
|
158
|
+
(`26.0`), as in R. Years from 2000 to the year after the current one.
|
|
159
|
+
|
|
160
|
+
Development: `pip install -e ".[dev]"` (pytest, responses, ruff and build);
|
|
161
|
+
`pip install -e ".[test]"` for the tests only.
|
|
162
|
+
|
|
163
|
+
See the `misocialr` README for the notes on the fields.
|
|
164
|
+
|
|
165
|
+
## Tests
|
|
166
|
+
|
|
167
|
+
```sh
|
|
168
|
+
pytest # offline, with recorded fixtures
|
|
169
|
+
pytest -m live # against the real API
|
|
170
|
+
```
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# misocialpy
|
|
2
|
+
|
|
3
|
+
<!-- badges: start -->
|
|
4
|
+

|
|
5
|
+

|
|
6
|
+
[](https://github.com/StrategicProjects/misocial/actions/workflows/misocialpy.yaml)
|
|
7
|
+
<!-- badges: end -->
|
|
8
|
+
|
|
9
|
+
Pandas-friendly Python interface to the **MI Social** API (Matriz de
|
|
10
|
+
Informação Social) of the Brazilian Ministry of Development and Social
|
|
11
|
+
Assistance (MDS/SAGI):
|
|
12
|
+
<https://aplicacoes.mds.gov.br/sagi/servicos/misocial>.
|
|
13
|
+
|
|
14
|
+
A port of the R package
|
|
15
|
+
[`misocialr`](https://github.com/StrategicProjects/misocial/tree/main/misocialr),
|
|
16
|
+
with the same
|
|
17
|
+
functions, name by name, as `ibgepy` does with `ibger`. Both versions read
|
|
18
|
+
the same field dictionary (`src/misocialpy/data/dictionary.csv`, a copy of
|
|
19
|
+
`misocialr/inst/extdata/`) and are tested against the same recorded
|
|
20
|
+
responses.
|
|
21
|
+
|
|
22
|
+
| Theme | Function | Content |
|
|
23
|
+
| --- | --- | --- |
|
|
24
|
+
| `cadunico` | `mds_cadunico()` | Cadastro Único: registered families and people, by income bracket |
|
|
25
|
+
| `bolsa_familia` | `mds_bolsa_familia()` | Bolsa Família programme: beneficiary families, benefits and transfers |
|
|
26
|
+
| `bpc_suas` | `mds_bpc_suas()` | BPC benefit and the social assistance network (CRAS, CREAS) |
|
|
27
|
+
| `suas_rede` | `mds_suas_rede()` | SUAS network (CadSUAS): CRAS, CREAS, Centro POP, day centres, shelters and staff per unit |
|
|
28
|
+
| `suas_financiamento` | `mds_suas_financiamento()` | FNAS transfers and balances of the municipal funds, and IGD-M |
|
|
29
|
+
| `pbf_condicionalidades` | `mds_pbf_condicionalidades()` | Education and health conditionality monitoring (SICON) and sanctions |
|
|
30
|
+
| `seguranca_alimentar` | `mds_seguranca_alimentar()` | PAA, Fomento Rural and cisterns |
|
|
31
|
+
|
|
32
|
+
Other functions: `mds_data()`, `mds_themes()`, `mds_fields()`,
|
|
33
|
+
`mds_api_fields()`, `mds_periods()`, `mds_localities()`,
|
|
34
|
+
`mds_drop_unpublished()`, `ibge_code7()`, `mds_clear_cache()`,
|
|
35
|
+
`set_verbose()`.
|
|
36
|
+
|
|
37
|
+
## Installation
|
|
38
|
+
|
|
39
|
+
```sh
|
|
40
|
+
pip install "git+https://github.com/StrategicProjects/misocial#subdirectory=misocialpy"
|
|
41
|
+
|
|
42
|
+
# or, from a local clone:
|
|
43
|
+
pip install ./misocialpy # from the repository root
|
|
44
|
+
pip install . # from inside misocialpy/
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Requires Python >= 3.9, pandas >= 1.3 and requests >= 2.25 (tested with
|
|
48
|
+
pandas 1.3.5 and requests 2.25.1 on Python 3.9, and with pandas 3.0.6 on
|
|
49
|
+
Python 3.14).
|
|
50
|
+
|
|
51
|
+
## Usage
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
import misocialpy as m
|
|
55
|
+
|
|
56
|
+
m.mds_bolsa_familia(
|
|
57
|
+
periods=[202508, 202509],
|
|
58
|
+
localities=2611606, # Recife (6 or 7 digits)
|
|
59
|
+
fields=["qtd_familias_beneficiarias_bolsa_familia_i", "valor_repassado_bolsa_familia_f"],
|
|
60
|
+
)
|
|
61
|
+
# ibge_code locality_name period field unit value
|
|
62
|
+
# 0 2611606 RECIFE 202508 qtd_familias_beneficiarias_bolsa_familia_i Famílias 140350.0
|
|
63
|
+
# 1 2611606 RECIFE 202509 qtd_familias_beneficiarias_bolsa_familia_i Famílias 139452.0
|
|
64
|
+
# 2 2611606 RECIFE 202508 valor_repassado_bolsa_familia_f R$ 92124240.0
|
|
65
|
+
# 3 2611606 RECIFE 202509 valor_repassado_bolsa_familia_f R$ 93354456.0
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
(Selected columns; output from 2026-10-19.)
|
|
69
|
+
|
|
70
|
+
- `periods`: `None` (last 12 months), `-n`, years (`2024`) and/or `YYYYMM`.
|
|
71
|
+
- `localities`: `"BR"`, states (`"PE"` or `26`), municipalities with 6 or 7
|
|
72
|
+
digits.
|
|
73
|
+
|
|
74
|
+
The API answers **0** for months not yet published. `mds_periods()` shows
|
|
75
|
+
how many municipalities have a non-zero value in each month, and
|
|
76
|
+
`mds_drop_unpublished()` explicitly removes the trailing all-zero months;
|
|
77
|
+
`reference="brazil"` decides by the whole country and preserves legitimate
|
|
78
|
+
zeros of small selections.
|
|
79
|
+
|
|
80
|
+
Validation is the same as in the R package. Invalid input is a
|
|
81
|
+
`ValidationError`; an incomplete or malformed response (partial result,
|
|
82
|
+
missing facet, total that changes while paging, repeated document, no
|
|
83
|
+
municipality or several values) is a `ResponseError`; two values for the
|
|
84
|
+
same observation are a `DataConflictError`; a network or HTTP failure is an
|
|
85
|
+
`HttpError` (with `status`). All inherit from `MdsError`. None of this
|
|
86
|
+
enters the cache.
|
|
87
|
+
|
|
88
|
+
`mds_drop_unpublished()` never deletes a non-zero value; `action="flag"`
|
|
89
|
+
classifies the rows (`publication_status`, `publication_reason`,
|
|
90
|
+
`publication_reference`) without removing any.
|
|
91
|
+
|
|
92
|
+
- `misocialpy.set_timeout(300)`: maximum time per request (default 180 s);
|
|
93
|
+
`misocialpy.set_verbose(False)` hides the messages.
|
|
94
|
+
- `df.attrs["retrieved_at"]`: date and time (UTC) of the query.
|
|
95
|
+
- `refresh=True` in `mds_periods()`, `mds_localities()` and
|
|
96
|
+
`mds_api_fields()`; `mds_localities(period=)` picks the month of the
|
|
97
|
+
list. Results are copies: editing them does not change the cache.
|
|
98
|
+
- `misocialpy.set_cache_ttl(3600)`: cache validity in seconds (default: the
|
|
99
|
+
whole session).
|
|
100
|
+
- `mds_fields()` brings `nature` (`estoque`, `fluxo`, `fluxo anual`,
|
|
101
|
+
`acumulado`, `taxa`, `média`, `índice`) and `nature_basis`; see the
|
|
102
|
+
summing table in the R README.
|
|
103
|
+
- Paging by `cursorMark`.
|
|
104
|
+
- Queries by POST (avoids the HTTP 414 of long URLs); 3 attempts in total
|
|
105
|
+
on 429/5xx and on connection failure, with `Retry-After` capped at 30 s.
|
|
106
|
+
- Accepts numpy scalars (`np.int64(202401)`) and integer-valued numbers
|
|
107
|
+
(`26.0`), as in R. Years from 2000 to the year after the current one.
|
|
108
|
+
|
|
109
|
+
Development: `pip install -e ".[dev]"` (pytest, responses, ruff and build);
|
|
110
|
+
`pip install -e ".[test]"` for the tests only.
|
|
111
|
+
|
|
112
|
+
See the `misocialr` README for the notes on the fields.
|
|
113
|
+
|
|
114
|
+
## Tests
|
|
115
|
+
|
|
116
|
+
```sh
|
|
117
|
+
pytest # offline, with recorded fixtures
|
|
118
|
+
pytest -m live # against the real API
|
|
119
|
+
```
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "misocialpy"
|
|
7
|
+
version = "0.4.0"
|
|
8
|
+
description = "Access the Brazilian MDS Social Information Matrix (MI Social) API from Python"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { file = "LICENSE" }
|
|
11
|
+
requires-python = ">=3.9"
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "André Leite", email = "leite@castlab.org" },
|
|
14
|
+
{ name = "Marcos Wasiliew", email = "marcos.wasiliew@gmail.com" },
|
|
15
|
+
{ name = "Hugo Vasconcelos", email = "hugo.vasconcelos@ufpe.br" },
|
|
16
|
+
{ name = "Carlos Amorim", email = "carlos.agaf@ufpe.br" },
|
|
17
|
+
{ name = "Diogo Bezerra", email = "diogo.bezerra@ufpe.br" },
|
|
18
|
+
]
|
|
19
|
+
keywords = ["MDS", "Cadastro Unico", "Bolsa Familia", "BPC", "SUAS", "Brazil", "open-data", "pandas"]
|
|
20
|
+
classifiers = [
|
|
21
|
+
"Development Status :: 4 - Beta",
|
|
22
|
+
"Intended Audience :: Science/Research",
|
|
23
|
+
"License :: OSI Approved :: MIT License",
|
|
24
|
+
"Programming Language :: Python :: 3",
|
|
25
|
+
"Topic :: Scientific/Engineering",
|
|
26
|
+
"Operating System :: OS Independent",
|
|
27
|
+
]
|
|
28
|
+
dependencies = [
|
|
29
|
+
"requests>=2.25",
|
|
30
|
+
"pandas>=1.3",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[project.urls]
|
|
34
|
+
Homepage = "https://github.com/StrategicProjects/misocial"
|
|
35
|
+
Repository = "https://github.com/StrategicProjects/misocial"
|
|
36
|
+
Issues = "https://github.com/StrategicProjects/misocial/issues"
|
|
37
|
+
Changelog = "https://github.com/StrategicProjects/misocial/blob/main/misocialpy/NEWS.md"
|
|
38
|
+
|
|
39
|
+
[project.optional-dependencies]
|
|
40
|
+
test = ["pytest>=7", "responses>=0.23"]
|
|
41
|
+
dev = ["pytest>=7", "responses>=0.23", "ruff>=0.6", "build>=1.0"]
|
|
42
|
+
|
|
43
|
+
[tool.hatch.build.targets.wheel]
|
|
44
|
+
packages = ["src/misocialpy"]
|
|
45
|
+
|
|
46
|
+
[tool.pytest.ini_options]
|
|
47
|
+
testpaths = ["tests"]
|
|
48
|
+
markers = ["live: hits the real MI Social API (run with -m live)"]
|
|
49
|
+
addopts = "-m 'not live'"
|
|
50
|
+
|
|
51
|
+
[tool.ruff]
|
|
52
|
+
line-length = 100
|
|
53
|
+
target-version = "py39"
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""misocialpy — access the MDS MI Social API from Python.
|
|
2
|
+
|
|
3
|
+
A pandas-friendly port of the R package `misocialr`. Monthly municipal series
|
|
4
|
+
of the Cadastro Unico, the Bolsa Familia programme and its conditionalities,
|
|
5
|
+
the BPC, the social assistance network and its funding, and food security
|
|
6
|
+
programmes, from the Social Information Matrix of the
|
|
7
|
+
Brazilian Ministry of Development and Social Assistance (MDS/SAGI).
|
|
8
|
+
|
|
9
|
+
Quick start
|
|
10
|
+
-----------
|
|
11
|
+
>>> import misocialpy
|
|
12
|
+
>>> misocialpy.mds_bpc_suas(periods=2025, localities="PE") # doctest: +SKIP
|
|
13
|
+
>>> misocialpy.mds_periods("bpc_ben_i", localities="PE") # doctest: +SKIP
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from ._cache import clear_cache as mds_clear_cache
|
|
19
|
+
from ._cache import set_cache_ttl
|
|
20
|
+
from ._client import DataConflictError, HttpError, MdsError, ResponseError, set_timeout
|
|
21
|
+
from ._msg import set_verbose
|
|
22
|
+
from .data import (
|
|
23
|
+
mds_bolsa_familia,
|
|
24
|
+
mds_bpc_suas,
|
|
25
|
+
mds_cadunico,
|
|
26
|
+
mds_data,
|
|
27
|
+
mds_pbf_condicionalidades,
|
|
28
|
+
mds_seguranca_alimentar,
|
|
29
|
+
mds_suas_financiamento,
|
|
30
|
+
mds_suas_rede,
|
|
31
|
+
)
|
|
32
|
+
from .fields import mds_api_fields, mds_fields, mds_themes
|
|
33
|
+
from .localities import ibge_code7, mds_localities
|
|
34
|
+
from .periods import mds_periods
|
|
35
|
+
from .publication import mds_drop_unpublished
|
|
36
|
+
from .validation import ValidationError
|
|
37
|
+
|
|
38
|
+
__version__ = "0.4.0"
|
|
39
|
+
|
|
40
|
+
__all__ = [
|
|
41
|
+
"DataConflictError",
|
|
42
|
+
"HttpError",
|
|
43
|
+
"MdsError",
|
|
44
|
+
"ResponseError",
|
|
45
|
+
"ValidationError",
|
|
46
|
+
"__version__",
|
|
47
|
+
"ibge_code7",
|
|
48
|
+
"mds_api_fields",
|
|
49
|
+
"mds_bolsa_familia",
|
|
50
|
+
"mds_bpc_suas",
|
|
51
|
+
"mds_cadunico",
|
|
52
|
+
"mds_clear_cache",
|
|
53
|
+
"mds_data",
|
|
54
|
+
"mds_drop_unpublished",
|
|
55
|
+
"mds_fields",
|
|
56
|
+
"mds_localities",
|
|
57
|
+
"mds_pbf_condicionalidades",
|
|
58
|
+
"mds_periods",
|
|
59
|
+
"mds_seguranca_alimentar",
|
|
60
|
+
"mds_suas_financiamento",
|
|
61
|
+
"mds_suas_rede",
|
|
62
|
+
"mds_themes",
|
|
63
|
+
"set_cache_ttl",
|
|
64
|
+
"set_timeout",
|
|
65
|
+
"set_verbose",
|
|
66
|
+
]
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""In-memory, per-session cache (equivalent to the R package's environment).
|
|
2
|
+
|
|
3
|
+
Holds localities, field lists, period facets and fields already confirmed to
|
|
4
|
+
exist.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import math
|
|
10
|
+
import time
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
_CACHE: dict[str, Any] = {}
|
|
14
|
+
_STORED_AT: dict[str, float] = {}
|
|
15
|
+
_TTL = math.inf
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def set_cache_ttl(seconds: float = math.inf) -> None:
|
|
19
|
+
"""Set how long, in seconds, a cached result is used (default: the whole
|
|
20
|
+
session). Must be a non-negative number; on error the previous value is
|
|
21
|
+
kept."""
|
|
22
|
+
global _TTL
|
|
23
|
+
if (
|
|
24
|
+
not isinstance(seconds, (int, float))
|
|
25
|
+
or isinstance(seconds, bool)
|
|
26
|
+
or math.isnan(seconds)
|
|
27
|
+
or seconds < 0
|
|
28
|
+
):
|
|
29
|
+
raise ValueError("`seconds` must be a non-negative number.")
|
|
30
|
+
_TTL = float(seconds)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _copy(value: Any) -> Any:
|
|
34
|
+
"""Independent copy of a cached table, with its attrs.
|
|
35
|
+
|
|
36
|
+
Pandas objects are mutable: handing out the stored instance would let an
|
|
37
|
+
edit of a result change the metadata used by later calls (periods feed
|
|
38
|
+
the inference of publication).
|
|
39
|
+
"""
|
|
40
|
+
if hasattr(value, "copy") and hasattr(value, "attrs"):
|
|
41
|
+
out = value.copy(deep=True)
|
|
42
|
+
out.attrs = dict(value.attrs)
|
|
43
|
+
return out
|
|
44
|
+
return value
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def cache_get(key: str, refresh: bool = False) -> Any:
|
|
48
|
+
"""Read a copy of a cached value, unless a refresh is requested."""
|
|
49
|
+
if refresh or key not in _CACHE:
|
|
50
|
+
return None
|
|
51
|
+
if time.monotonic() - _STORED_AT.get(key, 0.0) > _TTL:
|
|
52
|
+
return None
|
|
53
|
+
return _copy(_CACHE[key])
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def cache_set(key: str, value: Any) -> Any:
|
|
57
|
+
"""Store a copy of a value and return another (only complete, validated
|
|
58
|
+
responses reach here)."""
|
|
59
|
+
from datetime import datetime, timezone
|
|
60
|
+
|
|
61
|
+
if hasattr(value, "attrs"):
|
|
62
|
+
value.attrs["retrieved_at"] = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
63
|
+
_CACHE[key] = _copy(value)
|
|
64
|
+
_STORED_AT[key] = time.monotonic()
|
|
65
|
+
return _copy(value)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def locality_key(loc: dict) -> str:
|
|
69
|
+
"""Canonical key: ['PE', 'PB'] and ['PB', 'PE'] share the cached result."""
|
|
70
|
+
return f"uf={','.join(sorted(loc['states']))};mun={','.join(sorted(loc['municipalities']))}"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def clear_cache() -> None:
|
|
74
|
+
"""Remove all cached localities, field lists, periods and field checks.
|
|
75
|
+
|
|
76
|
+
Forces fresh API calls on subsequent requests. Public alias:
|
|
77
|
+
:func:`misocialpy.mds_clear_cache`.
|
|
78
|
+
"""
|
|
79
|
+
_CACHE.clear()
|
|
80
|
+
_STORED_AT.clear()
|
|
81
|
+
from . import _msg
|
|
82
|
+
|
|
83
|
+
_msg.success("MI Social cache cleared.")
|