perudata 0.1.1__tar.gz

perudata-0.1.1/.github/probe_new_year.py

@@ -0,0 +1,62 @@
+"""CI probe: is there an ENAHO proyecto code newer than the ones we map?
+
+Scans a window above the highest known code for a live Modulo34 zip. On a hit
+it opens a GitHub issue (once — skips codes already reported) so a maintainer
+can content-verify the internal `año` and add one line to enaho.YEAR_CODE.
+"""
+from __future__ import annotations
+
+import json
+import subprocess
+
+from perudata import enaho
+
+WINDOW = 120
+
+
+def existing_issue_codes() -> set[int]:
+    try:
+        out = subprocess.run(
+            ["gh", "issue", "list", "--label", "new-inei-year", "--state", "all",
+             "--json", "title"], capture_output=True, text=True, check=True).stdout
+        codes = set()
+        for it in json.loads(out or "[]"):
+            for tok in it["title"].split():
+                if tok.isdigit():
+                    codes.add(int(tok))
+        return codes
+    except Exception:
+        return set()
+
+
+def main() -> None:
+    top = max(enaho.YEAR_CODE.values())
+    print(f"probing codes {top + 1}..{top + WINDOW} (newest mapped: "
+          f"{enaho.latest_year()} = {top})")
+    hits = enaho.discover(0, lo=top + 1, hi=top + WINDOW)
+    if not hits:
+        print("no new codes on the INEI server")
+        return
+    known = existing_issue_codes()
+    fresh = [c for c in hits if c not in known]
+    if not fresh:
+        print(f"codes {hits} already reported")
+        return
+    body = (
+        f"The INEI server answers 200 for Modulo34 at proyecto code(s) **{fresh}** — "
+        f"likely ENAHO {enaho.latest_year() + 1}.\n\n"
+        "Next steps:\n"
+        "1. download it and read the internal `año` variable (codes are NOT chronological)\n"
+        "2. add the verified line to `enaho.YEAR_CODE`\n"
+        "3. run `perudata validate` for the new year against INEI's published poverty\n"
+    )
+    subprocess.run(
+        ["gh", "issue", "create",
+         "--title", f"New ENAHO proyecto code(s) live: {' '.join(map(str, fresh))}",
+         "--label", "new-inei-year", "--body", body],
+        check=True)
+    print(f"issue created for {fresh}")
+
+
+if __name__ == "__main__":
+    main()

perudata-0.1.1/.github/workflows/watch-new-year.yml

@@ -0,0 +1,31 @@
+name: Watch INEI for a new ENAHO year
+
+# INEI publishes each new ENAHO year as a fresh proyecto code with no
+# announcement API. This probes the code range above the newest mapped year
+# every Monday and opens an issue the day a new Modulo34 appears.
+on:
+  schedule:
+    - cron: "0 12 * * 1"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  probe:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install -e .
+      - name: Ensure label exists
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: gh label create new-inei-year --force --color 0e8a16 --description "INEI published a new survey year" || true
+      - name: Probe unmapped proyecto codes
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: python .github/probe_new_year.py

perudata-0.1.1/.gitignore

@@ -0,0 +1,9 @@
+__pycache__/
+*.pyc
+*.egg-info/
+dist/
+build/
+.venv/
+peru_raw/
+*.dta
+*.sav

perudata-0.1.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Carlos Chávez Padilla
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

perudata-0.1.1/PKG-INFO

@@ -0,0 +1,170 @@
+Metadata-Version: 2.4
+Name: perudata
+Version: 0.1.1
+Summary: Peru's official INEI microdata (ENAHO, ENDES, EPEN, EEA, ENAHO Panel): download, load and validate against official statistics
+Project-URL: Homepage, https://github.com/cesarchavezp29/perudata
+Project-URL: Issues, https://github.com/cesarchavezp29/perudata/issues
+Author-email: Carlos Chávez Padilla <cesarchavezpadilla@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: dhs,economics,enaho,endes,inei,microdata,peru,poverty,survey
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Sociology
+Requires-Python: >=3.9
+Requires-Dist: pandas>=1.5
+Requires-Dist: pyreadstat>=1.2
+Description-Content-Type: text/markdown
+
+# perudata
+
+**Peru's official INEI microdata, one import away.**
+
+```python
+from perudata import enaho, validate
+
+df = enaho.load(2024, "34")        # ENAHO sumaria: income, spending, poverty
+validate.poverty(years=[2024])     # reproduces INEI's official 27.6% from raw data
+```
+
+Five national surveys, one consistent API. Every file is verified on download
+by actually opening it, and a built-in validation gate reproduces INEI's
+official poverty series **to 0.0 percentage points, 2004–2025** before you
+build anything on top.
+
+| Survey | What it is | Coverage | Format |
+|---|---|---|---|
+| `enaho` | ENAHO — national household survey (income, poverty, employment, education, health, governance) | 2004–2025, 29 modules | Stata |
+| `panel` | ENAHO Panel — same households re-interviewed (true poverty dynamics, FE) | 10 releases, 2007–2023 | Stata |
+| `endes` | ENDES — Peru's DHS (fertility, child health, anemia, domestic violence) | 1996–2024 | SPSS |
+| `epen` | EPE/EPEN — permanent employment surveys (Lima monthly since 2001, national) | 279 verified datasets | CSV |
+| `eea` | EEA — annual economic survey of firms (sales, assets, employment, CIIU) | 2001–2024, 604 datasets | CSV |
+
+## Install
+
+```bash
+pip install perudata            # once published; for now:
+pip install git+https://github.com/cesarchavezp29/perudata
+```
+
+Requires Python ≥ 3.9. Dependencies: `pandas`, `pyreadstat`.
+
+## Quickstart
+
+```python
+from perudata import enaho, endes, epen, eea, panel, validate
+
+# --- ENAHO (annual cross-section) ------------------------------------------
+enaho.years()                        # [2004, ..., 2025]
+enaho.modules()                      # DataFrame of the 29 modules
+enaho.download([2023, 2024], ["01", "34", "85"])
+df = enaho.load(2024, "34")          # auto-downloads if missing
+
+# national poverty, the official INEI way (person = factor07 x mieperho):
+w = df["factor07"] * df["mieperho"]
+poverty = 100 * w[df["pobreza"].isin([1, 2])].sum() / w.sum()   # -> 27.6
+
+# --- validation gate ---------------------------------------------------------
+validate.poverty()                   # full 2004-2025 table vs official INEI
+
+# --- ENAHO Panel (longitudinal) ----------------------------------------------
+panel.releases()                              # [2011, 2015, ..., 2022, 2023] — 10 releases
+df, meta = panel.load_long(2023, "sumaria")   # latest: 2019-2023, tidy long
+meta["waves"]                                 # [2019, 2020, 2021, 2022, 2023]
+# any release works the same, e.g. the earliest 5-year panel:
+df, meta = panel.load_long(2011, "sumaria")   # 2007-2011
+
+# --- ENDES (DHS) --------------------------------------------------------------
+endes.download(2024, ["peso_talla_anemia"])
+kids = endes.load(2024, "peso_talla_anemia")
+
+# --- EPEN (employment) ---------------------------------------------------------
+epen.search("lima")                  # find datasets in the verified catalog
+df = epen.load(997)                  # download + read by catalog code
+
+# --- EEA (firms) ----------------------------------------------------------------
+eea.modules(2024)                    # sector modules of EEA 2024
+df = eea.load(eea.modules(2024)["csv_code"].iloc[0])
+```
+
+Or from the command line:
+
+```bash
+perudata enaho --years 2024 --modules 01 34
+perudata validate --years 2024
+perudata epen --search "dpto 2024"
+```
+
+## Where files go
+
+Everything lands under `./peru_raw` (override per call with `out=` or globally
+with the `PERUDATA_DIR` environment variable), named so you always know what a
+file is:
+
+```
+peru_raw/
+  enaho/sumaria/enaho-2024-34.dta
+  enaho_panel/2011_302/panel-2011-sumaria.dta
+  endes/2024_968/1629_hogar/968-Modulo1629/*.sav
+  epen/804_15_lima.../*.csv
+  eea/987-Modulo1968/a2022_s04_fF2/*.csv
+  */_manifest.csv          # what was downloaded, when, rows x cols
+```
+
+Downloads are **idempotent**: a file that exists and verified is skipped, so
+you can re-run a script after adding years and only the new ones are fetched.
+
+## Why this exists
+
+INEI publishes superb microdata, but using it programmatically means knowing
+a pile of undocumented conventions, all of which this package encodes:
+
+- Each survey-year is a numbered *proyecto* (`1031-Modulo34.zip`), and the
+  codes are **not chronological** — code 279 is 2010 while 280–285 are
+  2004–2009. Every ENAHO code here was verified by reading the internal `año`
+  variable of its file.
+- ENAHO 2004–2011 ships **Stata 7 (v110)** files that pandas cannot read —
+  `load()` falls back to pyreadstat transparently.
+- Sumaria zips carry `-12`/`-12g` alternates that silently lack the poverty
+  variables. The canonical file is selected for you.
+- ENDES exists **only in SPSS** for the full 1996–2024 series, and its module
+  numbers were renumbered in 2020 (64–74 → 1629–1641).
+- The ENAHO Panel changed module numbering in 2018 (01/34 → 1474–1479) and old
+  releases are WIDE files; `panel.load_long()` returns a tidy long panel
+  either way.
+- EPEN and EEA are CSV-only with no year→code formula at all — the package
+  ships verified catalogs (279 and 604 datasets) discovered by probing the
+  server and opening every file.
+
+## Validation
+
+`validate.poverty()` recomputes national monetary poverty from the raw sumaria
+(person-weighted, `factor07 × mieperho`, `pobreza ∈ {1,2}`) and compares it to
+the official series from INEI's *Evolución de la Pobreza Monetaria* reports:
+
+```
+year  poverty_pct  official_poverty  pov_diff
+2004         58.7              58.7       0.0
+...
+2024         27.6              27.6       0.0
+2025         25.7              25.7       0.0
+```
+
+22/22 years within 0.1pp (0.0pp in every year, extreme poverty included where
+published).
+
+## Notes
+
+- Data © INEI (Instituto Nacional de Estadística e Informática del Perú),
+  published as open microdata. This package only automates download and
+  reading — cite INEI and the survey when you publish.
+- Value labels are not applied on load; you get the raw codes exactly as INEI
+  documents them (the PDFs inside each zip are kept for EPEN/EEA).
+- New ENAHO year out? `enaho.discover(2026)` scans the server for its code.
+
+## License
+
+MIT — see [LICENSE](LICENSE). Built by [Carlos Chávez Padilla](https://github.com/cesarchavezp29).

perudata-0.1.1/README.md

@@ -0,0 +1,149 @@
+# perudata
+
+**Peru's official INEI microdata, one import away.**
+
+```python
+from perudata import enaho, validate
+
+df = enaho.load(2024, "34")        # ENAHO sumaria: income, spending, poverty
+validate.poverty(years=[2024])     # reproduces INEI's official 27.6% from raw data
+```
+
+Five national surveys, one consistent API. Every file is verified on download
+by actually opening it, and a built-in validation gate reproduces INEI's
+official poverty series **to 0.0 percentage points, 2004–2025** before you
+build anything on top.
+
+| Survey | What it is | Coverage | Format |
+|---|---|---|---|
+| `enaho` | ENAHO — national household survey (income, poverty, employment, education, health, governance) | 2004–2025, 29 modules | Stata |
+| `panel` | ENAHO Panel — same households re-interviewed (true poverty dynamics, FE) | 10 releases, 2007–2023 | Stata |
+| `endes` | ENDES — Peru's DHS (fertility, child health, anemia, domestic violence) | 1996–2024 | SPSS |
+| `epen` | EPE/EPEN — permanent employment surveys (Lima monthly since 2001, national) | 279 verified datasets | CSV |
+| `eea` | EEA — annual economic survey of firms (sales, assets, employment, CIIU) | 2001–2024, 604 datasets | CSV |
+
+## Install
+
+```bash
+pip install perudata            # once published; for now:
+pip install git+https://github.com/cesarchavezp29/perudata
+```
+
+Requires Python ≥ 3.9. Dependencies: `pandas`, `pyreadstat`.
+
+## Quickstart
+
+```python
+from perudata import enaho, endes, epen, eea, panel, validate
+
+# --- ENAHO (annual cross-section) ------------------------------------------
+enaho.years()                        # [2004, ..., 2025]
+enaho.modules()                      # DataFrame of the 29 modules
+enaho.download([2023, 2024], ["01", "34", "85"])
+df = enaho.load(2024, "34")          # auto-downloads if missing
+
+# national poverty, the official INEI way (person = factor07 x mieperho):
+w = df["factor07"] * df["mieperho"]
+poverty = 100 * w[df["pobreza"].isin([1, 2])].sum() / w.sum()   # -> 27.6
+
+# --- validation gate ---------------------------------------------------------
+validate.poverty()                   # full 2004-2025 table vs official INEI
+
+# --- ENAHO Panel (longitudinal) ----------------------------------------------
+panel.releases()                              # [2011, 2015, ..., 2022, 2023] — 10 releases
+df, meta = panel.load_long(2023, "sumaria")   # latest: 2019-2023, tidy long
+meta["waves"]                                 # [2019, 2020, 2021, 2022, 2023]
+# any release works the same, e.g. the earliest 5-year panel:
+df, meta = panel.load_long(2011, "sumaria")   # 2007-2011
+
+# --- ENDES (DHS) --------------------------------------------------------------
+endes.download(2024, ["peso_talla_anemia"])
+kids = endes.load(2024, "peso_talla_anemia")
+
+# --- EPEN (employment) ---------------------------------------------------------
+epen.search("lima")                  # find datasets in the verified catalog
+df = epen.load(997)                  # download + read by catalog code
+
+# --- EEA (firms) ----------------------------------------------------------------
+eea.modules(2024)                    # sector modules of EEA 2024
+df = eea.load(eea.modules(2024)["csv_code"].iloc[0])
+```
+
+Or from the command line:
+
+```bash
+perudata enaho --years 2024 --modules 01 34
+perudata validate --years 2024
+perudata epen --search "dpto 2024"
+```
+
+## Where files go
+
+Everything lands under `./peru_raw` (override per call with `out=` or globally
+with the `PERUDATA_DIR` environment variable), named so you always know what a
+file is:
+
+```
+peru_raw/
+  enaho/sumaria/enaho-2024-34.dta
+  enaho_panel/2011_302/panel-2011-sumaria.dta
+  endes/2024_968/1629_hogar/968-Modulo1629/*.sav
+  epen/804_15_lima.../*.csv
+  eea/987-Modulo1968/a2022_s04_fF2/*.csv
+  */_manifest.csv          # what was downloaded, when, rows x cols
+```
+
+Downloads are **idempotent**: a file that exists and verified is skipped, so
+you can re-run a script after adding years and only the new ones are fetched.
+
+## Why this exists
+
+INEI publishes superb microdata, but using it programmatically means knowing
+a pile of undocumented conventions, all of which this package encodes:
+
+- Each survey-year is a numbered *proyecto* (`1031-Modulo34.zip`), and the
+  codes are **not chronological** — code 279 is 2010 while 280–285 are
+  2004–2009. Every ENAHO code here was verified by reading the internal `año`
+  variable of its file.
+- ENAHO 2004–2011 ships **Stata 7 (v110)** files that pandas cannot read —
+  `load()` falls back to pyreadstat transparently.
+- Sumaria zips carry `-12`/`-12g` alternates that silently lack the poverty
+  variables. The canonical file is selected for you.
+- ENDES exists **only in SPSS** for the full 1996–2024 series, and its module
+  numbers were renumbered in 2020 (64–74 → 1629–1641).
+- The ENAHO Panel changed module numbering in 2018 (01/34 → 1474–1479) and old
+  releases are WIDE files; `panel.load_long()` returns a tidy long panel
+  either way.
+- EPEN and EEA are CSV-only with no year→code formula at all — the package
+  ships verified catalogs (279 and 604 datasets) discovered by probing the
+  server and opening every file.
+
+## Validation
+
+`validate.poverty()` recomputes national monetary poverty from the raw sumaria
+(person-weighted, `factor07 × mieperho`, `pobreza ∈ {1,2}`) and compares it to
+the official series from INEI's *Evolución de la Pobreza Monetaria* reports:
+
+```
+year  poverty_pct  official_poverty  pov_diff
+2004         58.7              58.7       0.0
+...
+2024         27.6              27.6       0.0
+2025         25.7              25.7       0.0
+```
+
+22/22 years within 0.1pp (0.0pp in every year, extreme poverty included where
+published).
+
+## Notes
+
+- Data © INEI (Instituto Nacional de Estadística e Informática del Perú),
+  published as open microdata. This package only automates download and
+  reading — cite INEI and the survey when you publish.
+- Value labels are not applied on load; you get the raw codes exactly as INEI
+  documents them (the PDFs inside each zip are kept for EPEN/EEA).
+- New ENAHO year out? `enaho.discover(2026)` scans the server for its code.
+
+## License
+
+MIT — see [LICENSE](LICENSE). Built by [Carlos Chávez Padilla](https://github.com/cesarchavezp29).

perudata-0.1.1/examples/poverty_replication.py

@@ -0,0 +1,14 @@
+"""
+Reproduce INEI's official national poverty series from raw ENAHO microdata.
+
+Downloads the sumaria (module 34) for the requested years into ./peru_raw and
+prints computed vs official poverty. Expect 0.0pp differences.
+
+    python examples/poverty_replication.py 2022 2023 2024
+"""
+import sys
+
+from perudata import validate
+
+years = [int(y) for y in sys.argv[1:]] or [2023, 2024, 2025]
+validate.poverty(years=years)

perudata-0.1.1/pyproject.toml

@@ -0,0 +1,35 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "perudata"
+version = "0.1.1"
+description = "Peru's official INEI microdata (ENAHO, ENDES, EPEN, EEA, ENAHO Panel): download, load and validate against official statistics"
+readme = "README.md"
+license = { text = "MIT" }
+authors = [{ name = "Carlos Chávez Padilla", email = "cesarchavezpadilla@gmail.com" }]
+requires-python = ">=3.9"
+dependencies = [
+    "pandas>=1.5",
+    "pyreadstat>=1.2",
+]
+keywords = ["peru", "inei", "enaho", "endes", "dhs", "microdata", "poverty", "survey", "economics"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Sociology",
+]
+
+[project.urls]
+Homepage = "https://github.com/cesarchavezp29/perudata"
+Issues = "https://github.com/cesarchavezp29/perudata/issues"
+
+[project.scripts]
+perudata = "perudata.cli:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/perudata"]

perudata-0.1.1/src/perudata/__init__.py

@@ -0,0 +1,19 @@
+"""
+perudata — Peru's official INEI microdata, one import away.
+
+    from perudata import enaho, endes, epen, eea, panel, validate
+
+    df = enaho.load(2024, "34")        # ENAHO sumaria: income, spending, poverty
+    validate.poverty(years=[2024])     # reproduces INEI's official 27.6%
+
+Five surveys, one consistent API (download / load / catalog), every file
+verified on download by opening it, and a validation gate that reproduces the
+official poverty series to 0.0pp before you build anything on top.
+
+Data lands in ./peru_raw by default — override per call with out= or globally
+with the PERUDATA_DIR environment variable.
+"""
+from . import eea, enaho, endes, epen, panel, validate  # noqa: F401
+
+__version__ = "0.1.1"
+__all__ = ["enaho", "endes", "epen", "eea", "panel", "validate"]