opendata-rfb 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opendata_rfb-1.0.0/LICENSE +21 -0
- opendata_rfb-1.0.0/PKG-INFO +263 -0
- opendata_rfb-1.0.0/README.md +227 -0
- opendata_rfb-1.0.0/pyproject.toml +86 -0
- opendata_rfb-1.0.0/src/filings_rfb/__init__.py +27 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/__init__.py +1 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/config/__init__.py +1 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/config/contracts/__init__.py +71 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/config/contracts/cnpj_companies.py +23 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/config/contracts/cnpj_establishments.py +49 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/config/contracts/cnpj_lookups.py +37 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/config/contracts/cnpj_partners.py +29 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/config/contracts/cnpj_simples.py +23 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/config/contracts/example_source.py +25 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/config/ports/__init__.py +13 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/config/ports/example_port.py +63 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/__init__.py +1 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/br_identifiers.py +288 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/compact_dates.py +48 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/daily_cache.py +248 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/dtypes.py +319 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/http_downloader.py +267 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/parquet_writer.py +139 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/provenance.py +251 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/raw_workspace.py +73 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/regime_adapter.py +76 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/regime_registry.py +202 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/regime_window.py +150 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/retry/__init__.py +23 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/retry/_schedule.py +77 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/retry/backoff.py +212 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/retry/log_emitter.py +55 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/retry/policy.py +105 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/sidecar_metadata.py +154 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/spec_gap_registry.py +98 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/stream_downloader.py +692 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/tabular_reader.py +948 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/text.py +79 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/typing/__init__.py +30 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/typing/abc_type_checker.py +16 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/typing/decorators.py +34 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/typing/policy.py +93 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/typing/protocol_type_checker.py +20 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/typing/type_checker.py +87 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/typing/validate.py +75 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/webdav_client.py +202 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/xml_reader.py +497 -0
- opendata_rfb-1.0.0/src/filings_rfb/_internal/utils/zip_extractor.py +326 -0
- opendata_rfb-1.0.0/src/filings_rfb/ingestion/__init__.py +1 -0
- opendata_rfb-1.0.0/src/filings_rfb/ingestion/cnpj/__init__.py +1 -0
- opendata_rfb-1.0.0/src/filings_rfb/ingestion/cnpj/_multipart.py +317 -0
- opendata_rfb-1.0.0/src/filings_rfb/ingestion/cnpj/_share.py +155 -0
- opendata_rfb-1.0.0/src/filings_rfb/ingestion/cnpj/companies.py +129 -0
- opendata_rfb-1.0.0/src/filings_rfb/ingestion/cnpj/establishments.py +128 -0
- opendata_rfb-1.0.0/src/filings_rfb/ingestion/cnpj/lookups.py +214 -0
- opendata_rfb-1.0.0/src/filings_rfb/ingestion/cnpj/partners.py +107 -0
- opendata_rfb-1.0.0/src/filings_rfb/ingestion/cnpj/simples.py +125 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 guilhermegor
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: opendata-rfb
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Typed pandas readers for the Receita Federal do Brasil (RFB) CNPJ open data
|
|
5
|
+
License: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Keywords: filings-rfb,python,library
|
|
8
|
+
Author: guilhermegor
|
|
9
|
+
Requires-Python: >=3.10,<4.0
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
22
|
+
Provides-Extra: parquet
|
|
23
|
+
Requires-Dist: beartype (>=0.22)
|
|
24
|
+
Requires-Dist: defusedxml (>=0.7)
|
|
25
|
+
Requires-Dist: numpy (>=1.22)
|
|
26
|
+
Requires-Dist: openpyxl (>=3.1.0)
|
|
27
|
+
Requires-Dist: pandas (>=2.1.0)
|
|
28
|
+
Requires-Dist: pyarrow (>=15.0) ; extra == "parquet"
|
|
29
|
+
Project-URL: Bug Reports, https://github.com/guilhermegor/filings-rfb/issues
|
|
30
|
+
Project-URL: Documentation, https://guilhermegor.github.io/filings-rfb/
|
|
31
|
+
Project-URL: Homepage, https://guilhermegor.github.io/filings-rfb/
|
|
32
|
+
Project-URL: Repository, https://github.com/guilhermegor/filings-rfb
|
|
33
|
+
Project-URL: Source, https://github.com/guilhermegor/filings-rfb
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
|
|
36
|
+
<!-- readme-archetype: library -->
|
|
37
|
+
# filings-rfb
|
|
38
|
+
|
|
39
|
+
<p align="center"><img src="https://raw.githubusercontent.com/guilhermegor/filings-rfb/main/assets/logo.png" width="200" alt="Receita Federal emblem"></p>
|
|
40
|
+
|
|
41
|
+
[](https://www.repostatus.org/#active)
|
|
42
|
+

|
|
43
|
+
[](https://pypi.org/project/filings-rfb/)
|
|
44
|
+
[](https://guilhermegor.github.io/filings-rfb/)
|
|
45
|
+
[](https://github.com/astral-sh/ruff)
|
|
46
|
+
[](https://mypy-lang.org/)
|
|
47
|
+

|
|
48
|
+

|
|
49
|
+

|
|
50
|
+

|
|
51
|
+
|
|
52
|
+
<!-- slug: overview -->
|
|
53
|
+
## Overview
|
|
54
|
+
|
|
55
|
+
`filings-rfb` reads the Receita Federal do Brasil (RFB) **CNPJ open data** (the company registry)
|
|
56
|
+
into typed pandas DataFrames. It covers every dataset the RFB publishes:
|
|
57
|
+
|
|
58
|
+
| Dataset | Reader | Content |
|
|
59
|
+
|---------|--------|---------|
|
|
60
|
+
| Empresas | `read_companies` | Legal entities (basic CNPJ, name, legal nature, share capital) |
|
|
61
|
+
| Estabelecimentos | `read_establishments` | Establishments, with the full 14-character CNPJ and its validity |
|
|
62
|
+
| Socios | `read_partners` | Partners and representatives (CPFs stay masked) |
|
|
63
|
+
| Simples | `read_simples` | Simples Nacional and MEI options |
|
|
64
|
+
| Cnaes, Motivos, Municipios, Naturezas, Paises, Qualificacoes | `read_lookup` | Code lookup tables |
|
|
65
|
+
|
|
66
|
+
**Data source and cadence.** The data comes from the RFB's public share
|
|
67
|
+
(`arquivos.receitafederal.gov.br`), which publishes one folder per reference month (`YYYY-MM`),
|
|
68
|
+
refreshed monthly. A reader takes the month as its argument; a month the share does not publish
|
|
69
|
+
raises an error rather than falling back to another one.
|
|
70
|
+
|
|
71
|
+
⚠️ **Size.** The archives are large and are never held in memory: each reader streams every ZIP
|
|
72
|
+
to disk and yields typed chunks. `Estabelecimentos` is the heaviest (ten parts, up to 2.3 GB
|
|
73
|
+
each compressed) and `Simples.zip` inflates to ~3.2 GB. Plan for disk space and a long first
|
|
74
|
+
run; pass `path_workdir=` to keep the ZIPs so a rerun resumes instead of downloading again.
|
|
75
|
+
|
|
76
|
+
<!-- slug: documentation -->
|
|
77
|
+
## Documentation
|
|
78
|
+
|
|
79
|
+
The full documentation is published at <https://guilhermegor.github.io/filings-rfb/>:
|
|
80
|
+
|
|
81
|
+
- [Usage](https://guilhermegor.github.io/filings-rfb/latest/usage/): install, readers, basics
|
|
82
|
+
- [Examples](https://guilhermegor.github.io/filings-rfb/latest/examples/): task-oriented recipes
|
|
83
|
+
- [API Reference](https://guilhermegor.github.io/filings-rfb/latest/api/): every public function
|
|
84
|
+
- [FAQ](https://guilhermegor.github.io/filings-rfb/latest/faq/): common questions and troubleshooting
|
|
85
|
+
- [Changelog](https://guilhermegor.github.io/filings-rfb/latest/changelog/): release history
|
|
86
|
+
|
|
87
|
+
To serve it locally: `bash bin/poe_exec.sh docs_serve` (runs the project venv's `poe`, no activation needed).
|
|
88
|
+
|
|
89
|
+
<!-- slug: getting-started -->
|
|
90
|
+
## Getting Started
|
|
91
|
+
|
|
92
|
+
Python 3.10+ is the only prerequisite.
|
|
93
|
+
|
|
94
|
+
<!-- slug: install -->
|
|
95
|
+
### Install
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
pip install filings-rfb
|
|
99
|
+
pip install 'filings-rfb[parquet]' # optional: Parquet export (pulls in pyarrow)
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Or with Poetry:
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
poetry add filings-rfb
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
To work on the library itself, see [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
109
|
+
|
|
110
|
+
<!-- slug: usage -->
|
|
111
|
+
### Usage
|
|
112
|
+
|
|
113
|
+
Every dataset reader (`read_companies`, `read_establishments`, `read_partners`,
|
|
114
|
+
`read_simples`) takes a reference month (`YYYY-MM`) and yields chunks of `int_chunk_rows` rows
|
|
115
|
+
(default 500,000), so memory stays flat. `read_lookup` is the exception: it takes
|
|
116
|
+
`str_table, path_zip, str_url, str_etag` and returns a single DataFrame, since each lookup table
|
|
117
|
+
is small. Every row carries provenance columns (`url`,
|
|
118
|
+
`updated_at`, `source_key`, `package_version`, `ingestion_run_id`, `content_hash`,
|
|
119
|
+
`source_etag`).
|
|
120
|
+
|
|
121
|
+
**Empresas.** `capital_social` is an exact `Decimal`, never a float.
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
from filings_rfb import read_companies
|
|
125
|
+
|
|
126
|
+
for df_chunk in read_companies("2026-10"):
|
|
127
|
+
...
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
**Estabelecimentos.** Adds the full `cnpj_completo` and its validity flag `cnpj_valido`
|
|
131
|
+
(alphanumeric-aware).
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
from filings_rfb import read_establishments
|
|
135
|
+
|
|
136
|
+
for df_chunk in read_establishments("2026-10"):
|
|
137
|
+
...
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
**Socios.** Masked CPFs (`***240659**`) are kept verbatim, never unmasked; `***000000**` as
|
|
141
|
+
representative document becomes null.
|
|
142
|
+
|
|
143
|
+
```python
|
|
144
|
+
from filings_rfb import read_partners
|
|
145
|
+
|
|
146
|
+
for df_chunk in read_partners("2026-10"):
|
|
147
|
+
...
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
**Simples.** Booleans and real dates, with the RFB's `00000000` read as null.
|
|
151
|
+
|
|
152
|
+
```python
|
|
153
|
+
from filings_rfb import read_simples
|
|
154
|
+
|
|
155
|
+
for df_chunk in read_simples("2026-10"):
|
|
156
|
+
...
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
**Lookups.** `read_lookup` reads one downloaded lookup ZIP (one of `CNPJ_LOOKUP_NAMES`).
|
|
160
|
+
|
|
161
|
+
```python
|
|
162
|
+
from pathlib import Path
|
|
163
|
+
|
|
164
|
+
from filings_rfb import read_lookup
|
|
165
|
+
from filings_rfb._internal.utils.stream_downloader import download_file_streaming
|
|
166
|
+
from filings_rfb.ingestion.cnpj._share import file_url, latest_reference_month
|
|
167
|
+
|
|
168
|
+
str_url = file_url(latest_reference_month(), "Cnaes.zip")
|
|
169
|
+
path_zip, str_etag = download_file_streaming(str_url, Path("data/raw/Cnaes.zip"))
|
|
170
|
+
df_cnaes = read_lookup("Cnaes", path_zip, str_url, str_etag)
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
**Parquet.** `export_companies_parquet("2026-10", path)` writes `Empresas` to one Parquet file
|
|
174
|
+
for lazy querying (DuckDB, Polars `scan_parquet`); it needs the `parquet` extra.
|
|
175
|
+
|
|
176
|
+
⚠️ Do not read these files with Polars directly: it silently misparses the `.zip` and corrupts
|
|
177
|
+
latin-1 text.
|
|
178
|
+
|
|
179
|
+
<!-- slug: project-structure -->
|
|
180
|
+
## Project Structure
|
|
181
|
+
|
|
182
|
+
```text
|
|
183
|
+
filings-rfb/
|
|
184
|
+
├── .github/ workflows, CODEOWNERS, PR template
|
|
185
|
+
├── assets/ logo
|
|
186
|
+
├── bin/ gate scripts and helpers
|
|
187
|
+
├── docs/ published documentation (mkdocs)
|
|
188
|
+
├── src/filings_rfb/
|
|
189
|
+
│ ├── __init__.py the public API (__all__)
|
|
190
|
+
│ ├── ingestion/cnpj/ one reader per RFB dataset
|
|
191
|
+
│ └── _internal/ private: contracts, utils, ports
|
|
192
|
+
├── tests/
|
|
193
|
+
│ ├── unit/
|
|
194
|
+
│ └── integration/
|
|
195
|
+
├── pyproject.toml
|
|
196
|
+
└── poe_tasks.toml task runner entry points
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
<!-- slug: development -->
|
|
200
|
+
## Development
|
|
201
|
+
|
|
202
|
+
```bash
|
|
203
|
+
git clone https://github.com/guilhermegor/filings-rfb.git
|
|
204
|
+
cd filings-rfb
|
|
205
|
+
bash bin/venv.sh # bootstrap: Poetry venv and all deps (poe included)
|
|
206
|
+
bash bin/precommit.sh # install the git hooks
|
|
207
|
+
bash bin/poe_exec.sh unit_tests # unit suite
|
|
208
|
+
bash bin/poe_exec.sh integration_tests # integration suite
|
|
209
|
+
bash bin/poe_exec.sh lint # ruff, codespell and the structure gates
|
|
210
|
+
bash bin/poe_exec.sh docs_serve # serve the docs locally
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
<!-- slug: public-api -->
|
|
214
|
+
## Public API
|
|
215
|
+
|
|
216
|
+
The public surface is whatever the package `__init__.py` lists in `__all__`; everything
|
|
217
|
+
under `_internal/` is private.
|
|
218
|
+
|
|
219
|
+
- `read_companies(str_month, int_chunk_rows=500_000, path_workdir=None)`: `Empresas` chunks.
|
|
220
|
+
- `read_establishments(...)`: `Estabelecimentos` chunks, same signature.
|
|
221
|
+
- `read_partners(...)`: `Socios` chunks, same signature.
|
|
222
|
+
- `read_simples(...)`: `Simples` chunks, same signature.
|
|
223
|
+
- `export_companies_parquet(str_month, path_dest, int_chunk_rows=500_000, path_workdir=None)`:
|
|
224
|
+
`Empresas` to one Parquet file (needs the `parquet` extra).
|
|
225
|
+
- `read_lookup(str_table, path_zip, str_url, str_etag)`: one of the six lookup tables
|
|
226
|
+
(`CNPJ_LOOKUP_NAMES`) as a typed DataFrame.
|
|
227
|
+
- `CNPJ_LOOKUP_NAMES`, `__version__`.
|
|
228
|
+
|
|
229
|
+
<!-- slug: publishing -->
|
|
230
|
+
## Publishing
|
|
231
|
+
|
|
232
|
+
Releases go out from the release workflows under `.github/workflows/` through PyPI trusted
|
|
233
|
+
publishing (OIDC); the git tag is the version.
|
|
234
|
+
|
|
235
|
+
<!-- slug: contributing -->
|
|
236
|
+
## Contributing
|
|
237
|
+
|
|
238
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for the branch, commit and review rules.
|
|
239
|
+
|
|
240
|
+
<!-- slug: license -->
|
|
241
|
+
## License
|
|
242
|
+
|
|
243
|
+
This project is licensed under MIT; see [LICENSE](LICENSE).
|
|
244
|
+
|
|
245
|
+
<!-- slug: authors -->
|
|
246
|
+
## Authors
|
|
247
|
+
|
|
248
|
+
- guilhermegor — [GitHub](https://github.com/guilhermegor) | [LinkedIn](https://www.linkedin.com/in/guilhermegor)
|
|
249
|
+
|
|
250
|
+
<!-- slug: useful-links -->
|
|
251
|
+
## Useful Links
|
|
252
|
+
|
|
253
|
+
- [Documentation](https://guilhermegor.github.io/filings-rfb/)
|
|
254
|
+
- [PyPI](https://pypi.org/project/filings-rfb/)
|
|
255
|
+
- [GitHub Repository](https://github.com/guilhermegor/filings-rfb)
|
|
256
|
+
- [Issue Tracker](https://github.com/guilhermegor/filings-rfb/issues)
|
|
257
|
+
|
|
258
|
+
---
|
|
259
|
+
|
|
260
|
+
<sub>Generated from the lib-minimal template via [BlueprintX](https://github.com/guilhermegor/blueprintx).
|
|
261
|
+
Unofficial, community library. Not affiliated with or endorsed by the Receita Federal do Brasil;
|
|
262
|
+
the emblem is theirs.</sub>
|
|
263
|
+
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
<!-- readme-archetype: library -->
|
|
2
|
+
# filings-rfb
|
|
3
|
+
|
|
4
|
+
<p align="center"><img src="https://raw.githubusercontent.com/guilhermegor/filings-rfb/main/assets/logo.png" width="200" alt="Receita Federal emblem"></p>
|
|
5
|
+
|
|
6
|
+
[](https://www.repostatus.org/#active)
|
|
7
|
+

|
|
8
|
+
[](https://pypi.org/project/filings-rfb/)
|
|
9
|
+
[](https://guilhermegor.github.io/filings-rfb/)
|
|
10
|
+
[](https://github.com/astral-sh/ruff)
|
|
11
|
+
[](https://mypy-lang.org/)
|
|
12
|
+

|
|
13
|
+

|
|
14
|
+

|
|
15
|
+

|
|
16
|
+
|
|
17
|
+
<!-- slug: overview -->
|
|
18
|
+
## Overview
|
|
19
|
+
|
|
20
|
+
`filings-rfb` reads the Receita Federal do Brasil (RFB) **CNPJ open data** (the company registry)
|
|
21
|
+
into typed pandas DataFrames. It covers every dataset the RFB publishes:
|
|
22
|
+
|
|
23
|
+
| Dataset | Reader | Content |
|
|
24
|
+
|---------|--------|---------|
|
|
25
|
+
| Empresas | `read_companies` | Legal entities (basic CNPJ, name, legal nature, share capital) |
|
|
26
|
+
| Estabelecimentos | `read_establishments` | Establishments, with the full 14-character CNPJ and its validity |
|
|
27
|
+
| Socios | `read_partners` | Partners and representatives (CPFs stay masked) |
|
|
28
|
+
| Simples | `read_simples` | Simples Nacional and MEI options |
|
|
29
|
+
| Cnaes, Motivos, Municipios, Naturezas, Paises, Qualificacoes | `read_lookup` | Code lookup tables |
|
|
30
|
+
|
|
31
|
+
**Data source and cadence.** The data comes from the RFB's public share
|
|
32
|
+
(`arquivos.receitafederal.gov.br`), which publishes one folder per reference month (`YYYY-MM`),
|
|
33
|
+
refreshed monthly. A reader takes the month as its argument; a month the share does not publish
|
|
34
|
+
raises an error rather than falling back to another one.
|
|
35
|
+
|
|
36
|
+
⚠️ **Size.** The archives are large and are never held in memory: each reader streams every ZIP
|
|
37
|
+
to disk and yields typed chunks. `Estabelecimentos` is the heaviest (ten parts, up to 2.3 GB
|
|
38
|
+
each compressed) and `Simples.zip` inflates to ~3.2 GB. Plan for disk space and a long first
|
|
39
|
+
run; pass `path_workdir=` to keep the ZIPs so a rerun resumes instead of downloading again.
|
|
40
|
+
|
|
41
|
+
<!-- slug: documentation -->
|
|
42
|
+
## Documentation
|
|
43
|
+
|
|
44
|
+
The full documentation is published at <https://guilhermegor.github.io/filings-rfb/>:
|
|
45
|
+
|
|
46
|
+
- [Usage](https://guilhermegor.github.io/filings-rfb/latest/usage/): install, readers, basics
|
|
47
|
+
- [Examples](https://guilhermegor.github.io/filings-rfb/latest/examples/): task-oriented recipes
|
|
48
|
+
- [API Reference](https://guilhermegor.github.io/filings-rfb/latest/api/): every public function
|
|
49
|
+
- [FAQ](https://guilhermegor.github.io/filings-rfb/latest/faq/): common questions and troubleshooting
|
|
50
|
+
- [Changelog](https://guilhermegor.github.io/filings-rfb/latest/changelog/): release history
|
|
51
|
+
|
|
52
|
+
To serve it locally: `bash bin/poe_exec.sh docs_serve` (runs the project venv's `poe`, no activation needed).
|
|
53
|
+
|
|
54
|
+
<!-- slug: getting-started -->
|
|
55
|
+
## Getting Started
|
|
56
|
+
|
|
57
|
+
Python 3.10+ is the only prerequisite.
|
|
58
|
+
|
|
59
|
+
<!-- slug: install -->
|
|
60
|
+
### Install
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install filings-rfb
|
|
64
|
+
pip install 'filings-rfb[parquet]' # optional: Parquet export (pulls in pyarrow)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Or with Poetry:
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
poetry add filings-rfb
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
To work on the library itself, see [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
74
|
+
|
|
75
|
+
<!-- slug: usage -->
|
|
76
|
+
### Usage
|
|
77
|
+
|
|
78
|
+
Every dataset reader (`read_companies`, `read_establishments`, `read_partners`,
|
|
79
|
+
`read_simples`) takes a reference month (`YYYY-MM`) and yields chunks of `int_chunk_rows` rows
|
|
80
|
+
(default 500,000), so memory stays flat. `read_lookup` is the exception: it takes
|
|
81
|
+
`str_table, path_zip, str_url, str_etag` and returns a single DataFrame, since each lookup table
|
|
82
|
+
is small. Every row carries provenance columns (`url`,
|
|
83
|
+
`updated_at`, `source_key`, `package_version`, `ingestion_run_id`, `content_hash`,
|
|
84
|
+
`source_etag`).
|
|
85
|
+
|
|
86
|
+
**Empresas.** `capital_social` is an exact `Decimal`, never a float.
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
from filings_rfb import read_companies
|
|
90
|
+
|
|
91
|
+
for df_chunk in read_companies("2026-10"):
|
|
92
|
+
...
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
**Estabelecimentos.** Adds the full `cnpj_completo` and its validity flag `cnpj_valido`
|
|
96
|
+
(alphanumeric-aware).
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
from filings_rfb import read_establishments
|
|
100
|
+
|
|
101
|
+
for df_chunk in read_establishments("2026-10"):
|
|
102
|
+
...
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
**Socios.** Masked CPFs (`***240659**`) are kept verbatim, never unmasked; `***000000**` as
|
|
106
|
+
representative document becomes null.
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
from filings_rfb import read_partners
|
|
110
|
+
|
|
111
|
+
for df_chunk in read_partners("2026-10"):
|
|
112
|
+
...
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
**Simples.** Booleans and real dates, with the RFB's `00000000` read as null.
|
|
116
|
+
|
|
117
|
+
```python
|
|
118
|
+
from filings_rfb import read_simples
|
|
119
|
+
|
|
120
|
+
for df_chunk in read_simples("2026-10"):
|
|
121
|
+
...
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
**Lookups.** `read_lookup` reads one downloaded lookup ZIP (one of `CNPJ_LOOKUP_NAMES`).
|
|
125
|
+
|
|
126
|
+
```python
|
|
127
|
+
from pathlib import Path
|
|
128
|
+
|
|
129
|
+
from filings_rfb import read_lookup
|
|
130
|
+
from filings_rfb._internal.utils.stream_downloader import download_file_streaming
|
|
131
|
+
from filings_rfb.ingestion.cnpj._share import file_url, latest_reference_month
|
|
132
|
+
|
|
133
|
+
str_url = file_url(latest_reference_month(), "Cnaes.zip")
|
|
134
|
+
path_zip, str_etag = download_file_streaming(str_url, Path("data/raw/Cnaes.zip"))
|
|
135
|
+
df_cnaes = read_lookup("Cnaes", path_zip, str_url, str_etag)
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
**Parquet.** `export_companies_parquet("2026-10", path)` writes `Empresas` to one Parquet file
|
|
139
|
+
for lazy querying (DuckDB, Polars `scan_parquet`); it needs the `parquet` extra.
|
|
140
|
+
|
|
141
|
+
⚠️ Do not read these files with Polars directly: it silently misparses the `.zip` and corrupts
|
|
142
|
+
latin-1 text.
|
|
143
|
+
|
|
144
|
+
<!-- slug: project-structure -->
|
|
145
|
+
## Project Structure
|
|
146
|
+
|
|
147
|
+
```text
|
|
148
|
+
filings-rfb/
|
|
149
|
+
├── .github/ workflows, CODEOWNERS, PR template
|
|
150
|
+
├── assets/ logo
|
|
151
|
+
├── bin/ gate scripts and helpers
|
|
152
|
+
├── docs/ published documentation (mkdocs)
|
|
153
|
+
├── src/filings_rfb/
|
|
154
|
+
│ ├── __init__.py the public API (__all__)
|
|
155
|
+
│ ├── ingestion/cnpj/ one reader per RFB dataset
|
|
156
|
+
│ └── _internal/ private: contracts, utils, ports
|
|
157
|
+
├── tests/
|
|
158
|
+
│ ├── unit/
|
|
159
|
+
│ └── integration/
|
|
160
|
+
├── pyproject.toml
|
|
161
|
+
└── poe_tasks.toml task runner entry points
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
<!-- slug: development -->
|
|
165
|
+
## Development
|
|
166
|
+
|
|
167
|
+
```bash
|
|
168
|
+
git clone https://github.com/guilhermegor/filings-rfb.git
|
|
169
|
+
cd filings-rfb
|
|
170
|
+
bash bin/venv.sh # bootstrap: Poetry venv and all deps (poe included)
|
|
171
|
+
bash bin/precommit.sh # install the git hooks
|
|
172
|
+
bash bin/poe_exec.sh unit_tests # unit suite
|
|
173
|
+
bash bin/poe_exec.sh integration_tests # integration suite
|
|
174
|
+
bash bin/poe_exec.sh lint # ruff, codespell and the structure gates
|
|
175
|
+
bash bin/poe_exec.sh docs_serve # serve the docs locally
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
<!-- slug: public-api -->
|
|
179
|
+
## Public API
|
|
180
|
+
|
|
181
|
+
The public surface is whatever the package `__init__.py` lists in `__all__`; everything
|
|
182
|
+
under `_internal/` is private.
|
|
183
|
+
|
|
184
|
+
- `read_companies(str_month, int_chunk_rows=500_000, path_workdir=None)`: `Empresas` chunks.
|
|
185
|
+
- `read_establishments(...)`: `Estabelecimentos` chunks, same signature.
|
|
186
|
+
- `read_partners(...)`: `Socios` chunks, same signature.
|
|
187
|
+
- `read_simples(...)`: `Simples` chunks, same signature.
|
|
188
|
+
- `export_companies_parquet(str_month, path_dest, int_chunk_rows=500_000, path_workdir=None)`:
|
|
189
|
+
`Empresas` to one Parquet file (needs the `parquet` extra).
|
|
190
|
+
- `read_lookup(str_table, path_zip, str_url, str_etag)`: one of the six lookup tables
|
|
191
|
+
(`CNPJ_LOOKUP_NAMES`) as a typed DataFrame.
|
|
192
|
+
- `CNPJ_LOOKUP_NAMES`, `__version__`.
|
|
193
|
+
|
|
194
|
+
<!-- slug: publishing -->
|
|
195
|
+
## Publishing
|
|
196
|
+
|
|
197
|
+
Releases go out from the release workflows under `.github/workflows/` through PyPI trusted
|
|
198
|
+
publishing (OIDC); the git tag is the version.
|
|
199
|
+
|
|
200
|
+
<!-- slug: contributing -->
|
|
201
|
+
## Contributing
|
|
202
|
+
|
|
203
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for the branch, commit and review rules.
|
|
204
|
+
|
|
205
|
+
<!-- slug: license -->
|
|
206
|
+
## License
|
|
207
|
+
|
|
208
|
+
This project is licensed under MIT; see [LICENSE](LICENSE).
|
|
209
|
+
|
|
210
|
+
<!-- slug: authors -->
|
|
211
|
+
## Authors
|
|
212
|
+
|
|
213
|
+
- guilhermegor — [GitHub](https://github.com/guilhermegor) | [LinkedIn](https://www.linkedin.com/in/guilhermegor)
|
|
214
|
+
|
|
215
|
+
<!-- slug: useful-links -->
|
|
216
|
+
## Useful Links
|
|
217
|
+
|
|
218
|
+
- [Documentation](https://guilhermegor.github.io/filings-rfb/)
|
|
219
|
+
- [PyPI](https://pypi.org/project/filings-rfb/)
|
|
220
|
+
- [GitHub Repository](https://github.com/guilhermegor/filings-rfb)
|
|
221
|
+
- [Issue Tracker](https://github.com/guilhermegor/filings-rfb/issues)
|
|
222
|
+
|
|
223
|
+
---
|
|
224
|
+
|
|
225
|
+
<sub>Generated from the lib-minimal template via [BlueprintX](https://github.com/guilhermegor/blueprintx).
|
|
226
|
+
Unofficial, community library. Not affiliated with or endorsed by the Receita Federal do Brasil;
|
|
227
|
+
the emblem is theirs.</sub>
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
[tool.poetry]
|
|
2
|
+
name = "opendata-rfb"
|
|
3
|
+
version = "1.0.0"
|
|
4
|
+
description = "Typed pandas readers for the Receita Federal do Brasil (RFB) CNPJ open data"
|
|
5
|
+
authors = ["guilhermegor"]
|
|
6
|
+
readme = "README.md"
|
|
7
|
+
license = "MIT"
|
|
8
|
+
homepage = "https://guilhermegor.github.io/filings-rfb/"
|
|
9
|
+
documentation = "https://guilhermegor.github.io/filings-rfb/"
|
|
10
|
+
repository = "https://github.com/guilhermegor/filings-rfb"
|
|
11
|
+
keywords = ["filings-rfb", "python", "library"]
|
|
12
|
+
packages = [{ include = "filings_rfb", from = "src" }]
|
|
13
|
+
exclude = ["src/**/CLAUDE.md"]
|
|
14
|
+
package-mode = true
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Operating System :: OS Independent",
|
|
18
|
+
"License :: OSI Approved :: MIT License",
|
|
19
|
+
"Development Status :: 3 - Alpha",
|
|
20
|
+
"Intended Audience :: Developers",
|
|
21
|
+
"Topic :: Software Development :: Libraries",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[tool.poetry.urls]
|
|
25
|
+
"Bug Reports" = "https://github.com/guilhermegor/filings-rfb/issues"
|
|
26
|
+
"Source" = "https://github.com/guilhermegor/filings-rfb"
|
|
27
|
+
"Documentation" = "https://guilhermegor.github.io/filings-rfb/"
|
|
28
|
+
|
|
29
|
+
[tool.poetry.dependencies]
|
|
30
|
+
python = ">=3.10,<4.0"
|
|
31
|
+
beartype = ">=0.22"
|
|
32
|
+
pandas = ">=2.1.0"
|
|
33
|
+
numpy = ">=1.22"
|
|
34
|
+
openpyxl = ">=3.1.0"
|
|
35
|
+
defusedxml = ">=0.7"
|
|
36
|
+
pyarrow = { version = ">=15.0", optional = true }
|
|
37
|
+
|
|
38
|
+
[tool.poetry.extras]
|
|
39
|
+
parquet = ["pyarrow"]
|
|
40
|
+
|
|
41
|
+
[tool.poetry.group.dev.dependencies]
|
|
42
|
+
poethepoet = ">=0.35"
|
|
43
|
+
pre-commit = ">=4.1.0"
|
|
44
|
+
pretty-errors = ">=1.2.25"
|
|
45
|
+
commitizen = ">=4.4.1"
|
|
46
|
+
build = ">=1.2.2"
|
|
47
|
+
twine = ">=6.1.0"
|
|
48
|
+
ruff = ">=0.11.13,<0.12"
|
|
49
|
+
codespell = ">=2.4.1"
|
|
50
|
+
pydocstyle = ">=6.3.0"
|
|
51
|
+
pytest = ">=8.0.0"
|
|
52
|
+
pytest-cov = ">=6.0.0"
|
|
53
|
+
pytest-randomly = ">=3.15.0"
|
|
54
|
+
coverage = ">=7.6.0"
|
|
55
|
+
genbadge = {extras = ["coverage"], version = ">=1.1.0"}
|
|
56
|
+
mypy = ">=1.8.0"
|
|
57
|
+
sqlfluff = ">=3.0.0"
|
|
58
|
+
yamllint = ">=1.35.0"
|
|
59
|
+
shfmt-py = ">=3.7.0"
|
|
60
|
+
shellcheck-py = ">=0.10.0"
|
|
61
|
+
deptry = ">=0.20"
|
|
62
|
+
tomli = { version = ">=2.0.1", python = "<3.11" }
|
|
63
|
+
pytest-mock = "^3.17.0"
|
|
64
|
+
pyarrow = ">=15.0"
|
|
65
|
+
|
|
66
|
+
[tool.poetry.group.docs.dependencies]
|
|
67
|
+
mkdocs = ">=1.6.0"
|
|
68
|
+
mkdocs-material = ">=9.5.0"
|
|
69
|
+
pymdown-extensions = ">=10.0"
|
|
70
|
+
mike = ">=2.1.0"
|
|
71
|
+
|
|
72
|
+
[tool.deptry.per_rule_ignores]
|
|
73
|
+
DEP002 = ["openpyxl"]
|
|
74
|
+
|
|
75
|
+
[tool.poetry-dynamic-versioning]
|
|
76
|
+
# Published version is derived from the latest git tag matching `v$version` (e.g. v0.2.0). On
|
|
77
|
+
# an exact tag the build is clean (0.2.0); off-tag builds get a PEP 440 dev version. The CI
|
|
78
|
+
# release workflows pass POETRY_DYNAMIC_VERSIONING_BYPASS to build a specific version. Needs
|
|
79
|
+
# `python -m build` (a PEP 517 frontend), not `poetry build` (which ignores the backend).
|
|
80
|
+
enable = false
|
|
81
|
+
vcs = "git"
|
|
82
|
+
style = "pep440"
|
|
83
|
+
|
|
84
|
+
[build-system]
|
|
85
|
+
requires = ["poetry-core>=1.7.0", "poetry-dynamic-versioning>=1.4.0,<2.0.0"]
|
|
86
|
+
build-backend = "poetry_dynamic_versioning.backend"
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""filings-rfb package."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
|
|
5
|
+
from .ingestion.cnpj.companies import export_companies_parquet, read_companies
|
|
6
|
+
from .ingestion.cnpj.establishments import read_establishments
|
|
7
|
+
from .ingestion.cnpj.lookups import CNPJ_LOOKUP_NAMES, read_lookup
|
|
8
|
+
from .ingestion.cnpj.partners import read_partners
|
|
9
|
+
from .ingestion.cnpj.simples import read_simples
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
try:
|
|
13
|
+
__version__ = version("opendata-rfb")
|
|
14
|
+
except PackageNotFoundError: # pragma: no cover - source tree without an installed dist
|
|
15
|
+
__version__ = "0.0.0"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"CNPJ_LOOKUP_NAMES",
|
|
20
|
+
"__version__",
|
|
21
|
+
"export_companies_parquet",
|
|
22
|
+
"read_companies",
|
|
23
|
+
"read_establishments",
|
|
24
|
+
"read_lookup",
|
|
25
|
+
"read_partners",
|
|
26
|
+
"read_simples",
|
|
27
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Private internals of the filings_rfb package (not a public API)."""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Private configuration internals (data contracts) of filings_rfb."""
|