LSMS_Library 0.2.10.dev0__py3-none-any.whl → 0.7.1__py3-none-any.whl

lsms_library/ISSUES.md

@@ -0,0 +1,9 @@
+# Open Issues
+
+Items that need attention. Resolved items are removed; use `git log` for history.
+Actionable items should also be filed on GitHub (`gh issue list --repo ligon/LSMS_Library`).
+## 2026-04-12 14:02:27Z Nepal – sample
+
+- Waves: 1995-96, 2003-04, 2010-11
+- Error: `PathMissingError: The path '/home/ligon/Projects/LSMS_Library/lsms_library/countries/Nepal/1995-96/Data/../Data/c1_nlss_public.dta' does not exist in the target repository '/home/ligon/Projects/LSMS_Library/lsms_library/countries/Nepal/1995-96/Data/../Data/c1_nlss_public.dta' neither as a DVC output nor as a Git-tracked file.`
+

lsms_library/Makefile

@@ -4,21 +4,42 @@
 
 .SILENT: help
 
+empty :=
+space := $(empty) $(empty)
+
+PYTHON ?= python3
+DVC ?= $(PYTHON) -m dvc
+format ?= parquet
+
+slug = $(shell printf '%s' "$(1)" | tr '[:upper:]' '[:lower:]' | tr ' -' '__')
+DEFAULT_WAVE_RAW := $(if $(strip $(wave)),$(firstword $(wave)),all_waves)
+DEFAULT_WAVE := $(subst $(space),_,$(strip $(DEFAULT_WAVE_RAW)))
+OUT_FILE ?= countries/build/$(country)/$(DEFAULT_WAVE)/$(table).$(format)
+stage_name = $(call slug,$(country))_$(call slug,$(DEFAULT_WAVE_RAW))_$(call slug,$(table))
+
+
 COUNTRIES := $(shell ls .. | grep -e ^[A-Z] | grep / | sed -e 's/\//,/g')
 
 help:
 	echo "USAGE:"
-	echo "Either <test> or <build> datasets or estimate <demands>"
+	echo "Either <test> or <build> datasets, <materialize>/<persist> tables, <register> stages, or estimate <demands>"
 	echo "for a particular country using, e.g.,"
 	echo "	make test country=Malawi"
 	echo "or"
 	echo "	make build country=Malawi"
 	echo "or"
-	echo "	make demands country=Malawi"
+	echo "	make demands country=Malawi market=Region"
+	echo "or"
+	echo "	make materialize country=Malawi wave=2019-20 table=household_roster"
+	echo "or"
+	echo "	make persist country=Malawi wave=2019-20 table=household_roster"
+	echo "or"
+	echo "	make register country=Uganda wave=2023-24 table=household_roster"
 	echo ""
 	echo "where country is one of:"
 	echo $(COUNTRIES)
 
+
 build:
 	echo $(country)
 	+make -C ../$(country)/_
@@ -26,8 +47,26 @@ build:
 test:
 	./harmony.py $(country)
 
-demands: build test
-	./demands.py $(country)
+materialize:
+	$(PYTHON) -m lsms_library.cli materialize \
+		$(if $(strip $(wave)),$(foreach w,$(wave),--wave $(w) ),$(if $(strip $(all_waves)),--all-waves ,))\
+		--country $(country) \
+		--table $(table) \
+		--format $(format) \
+		--out $(OUT_FILE)
+
+register:
+	$(PYTHON) -m lsms_library.cli register \
+		--country $(country) \
+		--wave $(wave) \
+		--table $(table) \
+		--format $(format)
+
+persist:
+	cd countries && $(DVC) repro materialize@$(stage_name)
+
+demands:
+	$(PYTHON) -m lsms_library.demands $(country) $(if $(market),--market $(market),)
 
 clean:
 	+make -C ../$(country)/_ clean

lsms_library/__init__.py

@@ -1,13 +1,211 @@
+"""LSMS Library: a harmonized interface to Living Standards Measurement Study data.
+
+The library provides a uniform API across 40+ LSMS-ISA and related
+household surveys from the World Bank Microdata Library.  Its design
+principle is to harmonize the *interface*, not the data: surveys differ
+in structure, and the library preserves survey-specific detail while
+offering a common way to read it.
+
+Typical usage
+-------------
+Instantiate a :class:`Country` and call its table methods as attributes.
+Table names (``food_acquired``, ``household_roster``, ``shocks``, ...)
+come from the country's ``data_scheme.yml``.
+
+.. code-block:: python
+
+    import lsms_library as ll
+
+    uga = ll.Country('Uganda')
+    uga.waves              # ['2005-06', '2009-10', ...]
+    uga.data_scheme        # ['household_roster', 'food_acquired', ...]
+
+    roster = uga.household_roster()    # harmonized DataFrame across waves
+    food   = uga.food_expenditures()   # derived from food_acquired
+    chars  = uga.household_characteristics()  # derived from household_roster
+
+:class:`Feature` assembles the same table across every country that
+declares it, with a ``country`` index level prepended:
+
+.. code-block:: python
+
+    df = ll.Feature('household_roster')()               # every country
+    df = ll.Feature('shocks')(['Ethiopia', 'Niger'])    # subset
+
+Returned DataFrames are indexed by some subset of ``(country, t, v, i,
+pid, j)`` — wave (``t``), cluster (``v``), household (``i``), person
+(``pid``), item (``j``) — depending on the table.
+
+Public API
+----------
+- :class:`Country` — top-level per-country interface. Exposes ``waves``,
+  ``data_scheme``, and one method per table in the scheme. Derived
+  tables (``household_characteristics``, ``food_expenditures``,
+  ``food_prices``, ``food_quantities``) are dispatched via
+  ``__getattr__`` without needing to be in ``data_scheme.yml``.
+- :class:`Wave` — single-wave view of a country, used internally by
+  :class:`Country` but occasionally handy for debugging.
+- :class:`Feature` — cross-country aggregator for a given table name.
+- ``tools`` (alias for :mod:`lsms_library.local_tools`) — ``get_dataframe``
+  for reading ``.dta``/``.csv``/``.parquet`` with local → DVC → WB NADA
+  fallback, ``to_parquet`` for writing caches under ``data_root()``, and
+  the ``df_data_grabber`` helper used by wave-level scripts.
+- :func:`authenticate` — interactive DVC/S3 credential unlock. Usually
+  runs automatically on import when a World Bank Microdata API key is
+  available.
+- :mod:`lsms_library.transformations` — derivation helpers for
+  roster-to-characteristics and food-to-expenditures.
+
+Data access and caching
+-----------------------
+Reads use a fallback chain: local file on disk → DVC filesystem → World
+Bank NADA download. Parquet caches are written under ``data_root()``,
+which defaults to ``~/.local/share/lsms_library/{Country}/`` and can be
+overridden via ``LSMS_DATA_DIR`` or the ``data_dir`` key in
+``~/.config/lsms_library/config.yml``.
+
+As of v0.7.0, :meth:`Country` methods check the parquet cache before
+consulting DVC, giving 10–17× cross-process speedups. The cache has no
+automatic staleness check — set ``LSMS_NO_CACHE=1`` or run
+``lsms-library cache clear --country {Country}`` after editing source
+configs or scripts.
+
+Environment variables
+---------------------
+- ``MICRODATA_API_KEY`` — World Bank Microdata Library API key (also
+  readable from ``~/.config/lsms_library/config.yml``). Required for
+  direct downloads; also unlocks the S3 read cache at import time.
+- ``LSMS_DATA_DIR`` — override the parquet cache root.
+- ``LSMS_NO_CACHE`` — disable the v0.7.0 top-of-function cache read.
+- ``LSMS_BUILD_BACKEND=make`` — force rebuild from source, bypassing
+  both the cache and the DVC stage layer.
+- ``LSMS_SKIP_AUTH`` — suppress the import-time authentication attempt.
+
+Further reading
+---------------
+- ``CLAUDE.md`` at the repository root for contributor-facing design
+  notes, the canonical schema (``lsms_library/data_info.yml``), and the
+  list of cross-cutting gotchas.
+- ``.claude/skills/add-feature/`` for adding a new table to a country,
+  with sub-skills for ``sample``, ``shocks``, ``assets``,
+  ``panel-ids``, ``food-acquired``, and ``pp-ph`` (post-planting /
+  post-harvest countries).
+- ``CONTRIBUTING.org`` for the new-wave workflow.
+
+Data terms of use
+-----------------
+Underlying microdata must be obtained from the World Bank Microdata
+Library (https://microdata.worldbank.org/) under their terms of use.
+The library's S3 bucket is a read cache over the authoritative WB NADA
+downloads, unlocked automatically when a valid API key is present.
+"""
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _get_version
+
+try:
+    __version__ = _get_version("LSMS_Library")
+except PackageNotFoundError:
+    __version__ = "0.0.0"
+del _get_version
+
+from pathlib import Path
+import os
+import warnings
+
+import logging
+from sys import stderr
 from . import country
-from . country import Country, Wave
+from .country import Country, Wave
+from .feature import Feature
 from . import local_tools as tools
-from .categorical_mapping import ai_agent
 from . import transformations
 from .dvc_permissions import authenticate
+try:
+    from dvc.ui import ui as dvc_ui
+    from dvc.logger import LoggerHandler, setup as dvc_log_setup
+    from functools import wraps
+    import sys
 
-from pathlib import Path
-gpg_path = Path(__file__).resolve().parent / 'countries' / '.dvc'
-creds_file = gpg_path / 's3_creds'
-if not creds_file.exists():
-    authenticate()
+    if hasattr(dvc_ui, "rich_console"):
+        try:
+            dvc_ui.rich_console.file = stderr
+        except (AttributeError, ImportError):
+            # Defensive against DVC rich-console API drift; programmer bugs
+            # (TypeError, NameError) propagate.
+            pass
+    if hasattr(dvc_ui, "error_console"):
+        try:
+            dvc_ui.error_console.file = stderr
+        except (AttributeError, ImportError):
+            # Defensive against DVC rich-console API drift.
+            pass
+    original_init = LoggerHandler.__init__
+
+    @wraps(original_init)
+    def _patched_logger_handler_init(self, stream, *args, **kwargs):
+        if stream is sys.stdout:
+            stream = stderr
+        original_init(self, stream, *args, **kwargs)
+
+    LoggerHandler.__init__ = _patched_logger_handler_init  # type: ignore[assignment]
+
+    dvc_log_setup()
+    for logger_name in ("dvc", "dvc_objects", "dvc_data", ""):
+        logger_obj = logging.getLogger(logger_name)
+        for handler in logger_obj.handlers:
+            if isinstance(handler, LoggerHandler) and getattr(handler, "stream", None) is not stderr:
+                handler.stream = stderr
+except (ImportError, AttributeError):
+    # DVC optional or its UI module changed shape; proceed without
+    # UI tweaks.  Programmer bugs surface unchanged.
+    pass
+
+from .config import s3_creds_path as _s3_creds_path
+creds_file = _s3_creds_path()
+
+SKIP_AUTH = os.getenv("LSMS_SKIP_AUTH", "").lower() in {"1", "true", "yes"}
 
+if not SKIP_AUTH and not creds_file.exists():
+    # Try non-interactive S3 unlock first.  This only decrypts the local
+    # s3_reader_creds.gpg — no network call required.  WB API key validation
+    # (which requires a network round-trip to microdata.worldbank.org) is
+    # deferred until the first actual data access via permissions() in
+    # data_access.py.  This keeps ``import lsms_library`` fast.
+    try:
+        from .data_access import _auto_unlock_s3 as _unlock
+        _unlocked = _unlock()
+        if not _unlocked:
+            # Local GPG decryption failed or no .gpg file — fall back to
+            # interactive passphrase prompt.
+            raise RuntimeError("auto-unlock did not produce S3 credentials")
+    except (ImportError, RuntimeError, OSError):
+        # Auto-unlock skipped (missing dep), failed (no .gpg / bad
+        # passphrase / gpg binary missing).  Fall through to interactive
+        # authenticate(); programmer bugs propagate.
+        try:
+            authenticate()
+        except (RuntimeError, OSError) as exc:
+            # Auth flow failure (network, subprocess, missing creds).
+            # Warn but don't crash import.
+            warnings.warn(
+                f"Automatic DVC authentication failed: {exc}. "
+                "Set LSMS_SKIP_AUTH=1 to suppress, or set "
+                "MICRODATA_API_KEY for non-interactive access."
+            )
+elif not SKIP_AUTH and creds_file.exists():
+    # User-config creds are already in place, so the auto-unlock branch
+    # above is short-circuited.  We still need to populate the legacy
+    # in-tree ``lsms_library/countries/.dvc/s3_creds`` path so that
+    # legacy wave scripts using ``dvc.api.open()`` (which don't pass a
+    # ``credentialpath`` override) can find credentials.  Without this,
+    # a fresh clone with valid user-config creds still hits
+    # ``NoCredentialsError`` on Uganda's earnings, enterprise_income,
+    # food_acquired etc. via the legacy DVC API path.
+    try:
+        from .data_access import _sync_legacy_dvc_creds as _sync
+        _sync()
+    except (ImportError, OSError):
+        # Sync failure is non-fatal --- legacy `dvc.api.open()` callers
+        # will surface a clear NoCredentialsError if it matters.
+        pass

lsms_library/calendars.py

@@ -0,0 +1,220 @@
+"""Calendar conversion helpers.
+
+This module currently exposes one calendar -- Ethiopian -- because it's
+the only non-Gregorian calendar that surfaces in raw LSMS data we care
+about.  Other calendars (Hijri, Coptic) can be added on demand.
+
+Ethiopian calendar quick reference
+==================================
+
+* The Ethiopian calendar is roughly 7-8 years behind the Gregorian:
+  Eth Year 1, 1, 1 = 29 Aug 8 CE Julian (= 27 Aug 8 CE Gregorian).
+* It has 13 months -- 12 of 30 days each, plus a 13th month called
+  Pagume of 5 days (6 in Ethiopian leap years).
+* Ethiopian leap years are every 4 years where ``year % 4 == 3``.
+* The Ethiopian new year (Meskerem 1) falls on Gregorian September 11
+  in normal years, September 12 in years where the *preceding*
+  Ethiopian year was a leap year (i.e., the prior Pagume had 6 days).
+
+Month name reference (Amharic transliterations seen in raw data):
+
+    1.  Meskerem  (~Sep 11 - Oct 10 Greg)
+    2.  Tikimt    (~Oct 11 - Nov  9)
+    3.  Hidar     (~Nov 10 - Dec  9)
+    4.  Tahsas    (~Dec 10 - Jan  8)
+    5.  Tir       (~Jan  9 - Feb  7)
+    6.  Yekatit   (~Feb  8 - Mar  9)
+    7.  Megabit   (~Mar 10 - Apr  8)
+    8.  Miyazya   (~Apr  9 - May  8)
+    9.  Ginbot    (~May  9 - Jun  7)
+   10.  Sene      (~Jun  8 - Jul  7)
+   11.  Hamle     (~Jul  8 - Aug  6)
+   12.  Nehase    (~Aug  7 - Sep  5)
+   13.  Pagume    (~Sep  6 - Sep 10, 5 days; 6 in leap years)
+
+The conversion uses Julian Day Numbers as the pivot.  The forward
+direction (Eth -> JDN) is closed-form arithmetic; the JDN -> Greg step
+is Edward G. Richards' standard formula (Mapping Time, 2013, 25.18.5).
+
+References:
+* Ethiopian Calendar -- Wikipedia
+* https://github.com/Senamiku/ethiopian-date-converter
+* Richards, E. G. (2013), "Calendars", in S. E. Urban & P. K. Seidelmann
+  (eds.) *Explanatory Supplement to the Astronomical Almanac*, 3rd ed.
+"""
+from __future__ import annotations
+
+from datetime import date
+
+
+# Julian Day Number of Eth 1/1/1 = 29 Aug 8 CE Julian = 27 Aug 8 CE Greg
+_ETH_EPOCH_JDN = 1724221
+
+
+# Canonical Ethiopian month order (1-indexed; 0th slot kept for clarity).
+ETHIOPIAN_MONTHS: list[str] = [
+    "",            # 0 unused
+    "Meskerem", "Tikimt", "Hidar", "Tahsas", "Tir",
+    "Yekatit", "Megabit", "Miyazya", "Ginbot", "Sene",
+    "Hamle", "Nehase", "Pagume",
+]
+
+
+# Lowercase variant -> month integer.  Includes common transliteration
+# spellings and short forms encountered in LSMS-Ethiopia raw data
+# (`hh_s1q04g_2` in 2013-14 GHS Wave 2).
+_ETH_MONTH_LOOKUP: dict[str, int] = {
+    # Canonical
+    "meskerem": 1, "tikimt": 2, "hidar": 3, "tahsas": 4, "tir": 5,
+    "yekatit": 6, "megabit": 7, "miyazya": 8, "ginbot": 9, "sene": 10,
+    "hamle": 11, "nehase": 12, "pagume": 13,
+    # Common alternate spellings
+    "meskarem": 1, "meskerm": 1, "tekemt": 2, "tikemt": 2,
+    "tahesas": 4, "tahisas": 4, "ter": 5, "yekatet": 6,
+    "meyazia": 8, "miyaza": 8, "miazia": 8, "ginbo": 9,
+    "sine": 10, "senie": 10, "nehasse": 12, "pagumen": 13, "puagme": 13,
+}
+
+
+def parse_ethiopian_month(name) -> int | None:
+    """Map an Ethiopian month name (Amharic transliteration) to 1..13.
+
+    Returns ``None`` for unrecognised input or NaN/missing.
+    """
+    if name is None:
+        return None
+    if not isinstance(name, str):
+        # Numeric input that already encodes the month number directly.
+        try:
+            v = int(float(name))
+        except (TypeError, ValueError):
+            return None
+        return v if 1 <= v <= 13 else None
+    s = name.strip().lower()
+    if not s:
+        return None
+    return _ETH_MONTH_LOOKUP.get(s)
+
+
+def is_ethiopian_leap_year(year: int) -> bool:
+    """Ethiopian leap years: ``year % 4 == 0``.
+
+    Per Wikipedia ("Ethiopian calendar"), Eth 8, 12, ..., 1996, 2000,
+    2004, 2008, 2012 are leap.  Pagume (the 13th month) has 6 days in
+    those years and 5 days otherwise.
+
+    Note: this differs from the Gregorian leap rule -- Eth 2100 will be
+    leap (% 4 == 0) while Greg 2100 is not (century-no-400-rule), so
+    the calendars drift by one day in 2100+ -- but for the LSMS-Ethiopia
+    data range (1900-2099) the rules align.
+    """
+    return year % 4 == 0
+
+
+def ethiopian_to_jdn(year: int, month: int, day: int) -> int:
+    """Convert an Ethiopian (year, month, day) to Julian Day Number.
+
+    Validates that the inputs form a real Ethiopian date; raises
+    ``ValueError`` otherwise.
+    """
+    if not (1 <= month <= 13):
+        raise ValueError(f"Ethiopian month must be 1..13, got {month}")
+    if month == 13:
+        max_day = 6 if is_ethiopian_leap_year(year) else 5
+    else:
+        max_day = 30
+    if not (1 <= day <= max_day):
+        raise ValueError(
+            f"Ethiopian day {day} out of range for "
+            f"month {month} (max={max_day}, year={year})"
+        )
+    # Leap-day count: number of Eth leap years in [1, year-1] (the
+    # current year's leap day, if any, has not yet occurred at month 1
+    # day 1 — Pagume 6 falls at the *end* of the year).
+    return (
+        _ETH_EPOCH_JDN
+        + 365 * (year - 1)
+        + ((year - 1) // 4)
+        + 30 * (month - 1)
+        + day - 1
+    )
+
+
+def jdn_to_gregorian(jdn: int) -> date:
+    """Convert a Julian Day Number to a Gregorian ``datetime.date``.
+
+    Uses Edward G. Richards' formula (Mapping Time, 2013), valid for
+    any JDN representing a Gregorian date after the calendar's start.
+    """
+    a = jdn + 32044
+    b = (4 * a + 3) // 146097
+    c = a - (146097 * b) // 4
+    d = (4 * c + 3) // 1461
+    e = c - (1461 * d) // 4
+    m = (5 * e + 2) // 153
+    g_day = e - (153 * m + 2) // 5 + 1
+    g_month = m + 3 - 12 * (m // 10)
+    g_year = 100 * b + d - 4800 + m // 10
+    return date(g_year, g_month, g_day)
+
+
+def ethiopian_to_gregorian(year: int, month: int, day: int) -> date:
+    """Convert an Ethiopian (year, month, day) to a Gregorian ``date``.
+
+    >>> ethiopian_to_gregorian(2000, 1, 1)
+    datetime.date(2007, 9, 11)
+    >>> ethiopian_to_gregorian(2007, 13, 6)   # Pagume 6 in a leap year
+    datetime.date(2015, 9, 11)
+    >>> ethiopian_to_gregorian(2008, 1, 1)    # day after the leap Pagume
+    datetime.date(2015, 9, 12)
+    """
+    return jdn_to_gregorian(ethiopian_to_jdn(year, month, day))
+
+
+def disambiguate_two_digit_eth_year(
+    yy: int, *, interview_eth_year: int, reported_age: int | None = None,
+) -> int | None:
+    """Resolve a 2-digit Ethiopian year to its 4-digit form.
+
+    LSMS-Ethiopia 2013-14 GHS Wave 2 records year of birth as a 2-digit
+    integer (e.g. ``89`` for Eth 1989, ``1`` for Eth 2001).  The
+    century is implicit; this helper picks the candidate that yields a
+    plausible age in [0, 120], using ``reported_age`` as a tiebreaker
+    when both centuries are plausible.
+
+    Returns ``None`` if neither candidate yields a plausible age, or if
+    the input is already 4-digit and out of range.
+    """
+    if yy is None:
+        return None
+    try:
+        yy = int(yy)
+    except (TypeError, ValueError):
+        return None
+    # 2-digit input: try both centuries.  Already-4-digit input passes
+    # through but is still subject to the plausibility check below.
+    if yy < 100:
+        candidates = [1900 + yy, 2000 + yy]
+    else:
+        candidates = [yy]
+    plausible: list[int] = []
+    for c in candidates:
+        a = interview_eth_year - c
+        if 0 <= a <= 120:
+            plausible.append(c)
+    if not plausible:
+        return None
+    if len(plausible) == 1:
+        return plausible[0]
+    # Both centuries yield a plausible age (typical for yy in 0..13 with
+    # interview around Eth 2006).  Pick the candidate closest to the
+    # reported age when available; otherwise prefer the more recent.
+    if reported_age is not None:
+        try:
+            ra = int(reported_age)
+        except (TypeError, ValueError):
+            ra = None
+        if ra is not None and 0 <= ra <= 120:
+            return min(plausible, key=lambda c: abs((interview_eth_year - c) - ra))
+    # Default: pick the more recent (largest) century-candidate.
+    return max(plausible)

lsms_library/categorical_mapping/ai_agent.py

@@ -1,4 +1,17 @@
-# AI Agent
+"""GPT-based helper for suggesting categorical harmonization mappings.
+
+Experimental tooling that sends a DataFrame of raw survey labels to an
+OpenAI chat model and asks it to propose canonical "Preferred Label"
+matches. Used as a drafting aid for maintainers filling out the
+``#+NAME: harmonize_food`` and ``#+NAME: unit`` tables in a country's
+``categorical_mapping.org``. Output is always reviewed by a human before
+being committed — the library itself never calls this module at runtime.
+
+Requires the ``openai`` package and an API key (set via the ``api_key``
+module-level variable or looked up from ``authinfo``). Not part of the
+public API and not loaded by :mod:`lsms_library.__init__` unless
+explicitly imported.
+"""
 import pandas as pd
 from ligonlibrary import authinfo
 import openai

lsms_library/categorical_mapping/categorical_mapping_helper.py

@@ -1,3 +1,22 @@
+"""String-similarity helpers for drafting categorical mapping tables.
+
+Lightweight NLP utilities used when bootstrapping a country's
+``categorical_mapping.org`` from raw survey labels. The main operations
+are:
+
+- :func:`regularize_string` — strip accents, lowercase, and spell-check
+  via TextBlob so that "Sécheresse" and "secheresse" land in the same
+  bucket before similarity scoring.
+- :func:`preprocess` / :func:`get_label_vector` — turn a list of raw
+  labels into feature vectors keyed on cleaned tokens.
+- :func:`get_cosine_similarity` — score two label vectors so the
+  maintainer can propose the highest-ranked canonical label for each
+  raw variant.
+
+Like :mod:`.ai_agent`, this is a drafting aid for maintainers — the
+library does not call these functions at runtime, and the output is
+always reviewed by a human before being committed.
+"""
 import pandas as pd
 import numpy as np
 import re