LSMS_Library 0.2.7__py3-none-any.whl → 0.2.10.dev0__py3-none-any.whl

lsms_library/countries/Uganda/_/unitlabels.csv

@@ -0,0 +1,290 @@
+Code,Preferred Label
+0,---
+1,Kg
+2,Gram
+3,Litre
+4,Akendo (Small)
+5,Metre
+6,Square metre
+7,Yard
+8,Millilitre
+9,Sack (120 kgs)
+10,Sack (100 kgs)
+11,Sack (80 kgs)
+12,Sack (50 kgs)
+13,Sack (unspecified)
+14,Jerrican (20 lts)
+15,Jerrican (10 lts)
+16,Jerrican (5 lts)
+17,Jerrican (3 lts)
+18,Jerrican (2 lts)
+19,Jerrican (1 lt)
+20,Tin (Debe) (20 lts)
+21,Tin (5 lts)
+22,Plastic Basin (15 lts)
+23,Bottle(750ml)
+24,Bottle(500ml)
+25,Bottle(350ml)
+26,Bottle(300ml)
+27,Bottle(250ml)
+28,Bottle(150ml)
+29,Kimbo/Cowboy/Blueband Tin (2kg)
+30,Kimbo/Cowboy/Blueband Tin (1kg)
+31,Kimbo/Cowboy/Blueband Tin (0.5kg)
+32,Cup/Mug(0.5lt)
+33,Glass(0.25lt)
+34,Ladle(100g)
+35,Table spoon
+36,Tea spoon
+37,Basket (20 kg)
+38,Basket (10 kg)
+39,Basket (5 kg)
+40,Basket (2 kg)
+41,Loaf (1kg)
+42,Loaf (500g)
+43,Buns (200 g)
+44,Buns (100 g)
+45,Buns (50 g)
+46,Bathing soap (Tablet)
+47,Washing soap (Bar)
+48,Washing soap (Tablet)
+49,Packet (2 kg)
+50,Packet (1 kg)
+51,Packet (500 g)
+52,Packet (250 g)
+53,Packet (100 g)
+54,Packet(125g)
+55,Fish Whole (Small)
+56,Fish Whole (Medium)
+57,Fish Whole  (Large)
+58,Fish Cut piece (up to 1kg)
+59,Fish Cut piece (1 - 2 kg)
+60,Fish Cut piece(Above 2  kg)
+61,Tray of 30 eggs
+62,Ream
+63,Crate
+64,Heap (unspecified)
+65,Dozen
+66,Bundle (Unspecified)
+67,Bunch (Big)
+68,Bunch (Medium)
+69,Bunch (Small)
+70,Bunch (Big)
+71,Bunch (Medium)
+72,Bunch (Small)
+73,Cluster (Unspecified)
+74,Gourd (1-5lts)
+75,Gourd (5-10lts)
+76,Gourd (Above 10 lts)
+77,Jug (2 lts)
+78,Jug (1.5 lts)
+79,Jug (1 lt)
+80,Tot (50 ml)
+81,Tot (sachet)
+82,Tot (Unspecified)
+83,Tobacco leaf (Number)
+84,Pair
+85,Number of Units (General)
+86,Acre
+87,Piece (Big)
+88,Piece (Medium)
+89,Piece (Small)
+90,Heap (Big)
+91,Heap (Medium)
+92,Heap (Small)
+93,Cluster (Big)
+94,Cluster (Medium)
+95,Cluster (Small)
+96,Bundle (Big)
+97,Bundle (Medium)
+98,Bundle (Small)
+99,Others specify
+100,Fish Whole (Large)
+101,Fish Whole (Medium)
+102,Fish Whole (Small)
+103,Plastic Basin (5 ltrs)
+104,Glass (500ml)
+105,Glass (125ml)
+106,Jug (2.5ltrs)
+107,Nice cup (100g) - Large
+108,Nice cup (60g) - Medium
+109,Nice cup (50g) - Small
+110,Metallic tumbler (100g) - Big
+111,Metallic tumbler (50g) - Small
+112,Plastic tumbler (50g) - Big
+113,Plastic tumbler (30g) - Small
+114,Plastic plate (60g) - Large
+115,Plastic plate (30g) - Small
+116,Metallic plate (100g) - Large
+117,Metallic plate (80g) - Small
+118,Plastic bowl (40g)
+119,Nomi Tin (1kg)
+120,Nomi Tin (500g)
+121,Nomi Tin (250g)
+122,Nido Tin (400g)
+123,Akendo (Big)
+124,Akendo (Medium)
+125,Akendo (Small)
+126,Jerrican (500 ml)
+127,Container (Big)
+128,Container (Medium)
+129,Container (Small)
+130,Container (Smallest)
+131,Bottle (500g)
+132,Bottle (350g)
+133,Sadolin Tin (3ltrs)
+134,Loaf (1kg)
+135,Loaf (500g)
+137,Bottle (1ltr)
+138,Bottle (1.5ltr)
+139,Bottle (2ltr)
+140,Sacket (10g)
+141,Sacket (15g)
+142,Sacket (20g)
+143,Sacket (25g)
+144,Sacket (30g)
+145,Sacket (50g)
+146,Sacket (100g)
+147,Sacket (150g)
+104002,---
+104003,---
+104087,---
+104088,---
+104089,---
+105091,---
+105092,---
+107087,---
+107088,---
+107089,---
+107090,---
+107091,---
+107092,---
+109087,---
+109088,---
+109089,---
+109090,---
+109092,---
+111087,---
+111088,---
+111089,---
+111090,---
+113087,---
+113088,---
+113089,---
+114087,---
+114089,---
+121087,---
+121089,---
+122055,---
+122056,---
+122057,---
+123055,---
+123056,---
+123057,---
+123058,---
+123059,---
+123060,---
+123090,---
+123091,---
+123092,---
+124088,---
+125050,---
+125051,---
+125052,---
+128004,---
+128005,---
+128051,---
+128052,---
+128053,---
+129001,---
+129002,---
+129003,---
+129004,---
+130090,---
+130091,---
+130092,---
+131093,---
+131094,---
+131095,---
+132087,---
+132088,---
+132089,---
+133087,---
+133088,---
+133089,---
+133090,---
+133092,---
+135087,---
+135088,---
+135089,---
+135090,---
+135091,---
+135092,---
+135096,---
+135097,---
+135098,---
+136087,---
+136088,---
+136089,---
+136090,---
+136091,---
+136092,---
+137087,---
+137088,---
+137089,---
+138096,---
+138097,---
+138098,---
+151024,---
+151026,---
+152024,---
+153001,---
+154001,---
+155054,---
+155099,---
+157001,---
+158024,---
+158026,---
+159024,---
+160001,---
+161001,---
+164087,---
+164088,---
+164089,---
+165087,---
+165088,---
+165089,---
+166087,---
+166088,---
+166089,---
+167087,---
+167088,---
+167089,---
+167090,---
+167091,---
+167092,---
+168087,---
+168088,---
+168089,---
+168090,---
+168091,---
+168092,---
+169087,---
+169088,---
+169089,---
+170087,---
+170088,---
+170089,---
+171087,---
+171088,---
+171089,---
+807087,---
+807089,---
+136,---
+156087,---
+139093,---
+142092,---
+156096,---
+156090,---
+181087,---
+142090,---

lsms_library/country.py

@@ -43,9 +43,8 @@ class Wave:
         
     @property
     def file_path(self):
-        var = files("lsms_library") / "countries" / self.folder
-        return var
-    
+        return files("lsms_library") / "countries" / self.folder
+
     @property
     def resources(self):
         """Load the data_info.yml that describes table structure, merges, etc."""
@@ -212,7 +211,7 @@ class Wave:
             df_edit_function = mapping_info.pop('df_edit')
             dfs = []
             for file, mappings in mapping_info.items():
-                df = df_data_grabber(f'lsms_library/countries/{self.folder}/Data/{file}', mappings['idxvars'], **mappings['myvars'], convert_categoricals=convert_cat)
+                df = df_data_grabber(f'./{self.folder}/Data/{file}', mappings['idxvars'], **mappings['myvars'], convert_categoricals=convert_cat)
                 df = check_adding_t(df)
                 # Oddity with large number for missing code
                 na = df.select_dtypes(exclude=['object', 'datetime64[ns]']).max().max()
@@ -266,7 +265,7 @@ class Wave:
 
             cwd_path = self.file_path.parent / "_"
             relative_parquet_path = parquet_fn.relative_to(cwd_path.parent)  # Convert to relative path
-            subprocess.run(["make", '../' + str(relative_parquet_path)], cwd=cwd_path, check=True)
+            subprocess.run(["make", "-s", '../' + str(relative_parquet_path)], cwd=cwd_path, check=True)
             print(f"Makefile executed successfully for {self.name}. Rechecking for parquet file...",file=stderr)
 
             if not parquet_fn.exists():
@@ -331,7 +330,7 @@ class Country:
     #                 'nonfood_expenditures.parquet', 'enterprise_income.parquet', 'assets.parquet',
     #                 'earnings.parquet', 'housing.parquet', 'income.parquet', 'fct.parquet', 'nutrition.parquet']
 
-    def __init__(self,country_name, preload_panel_ids = True):
+    def __init__(self, country_name, preload_panel_ids=True, verbose=False):
         self.name = country_name
         self._panel_ids_cache = None
         self._updated_ids_cache = None
@@ -510,7 +509,7 @@ class Country:
                 relative_path = target_path.relative_to(cwd_path.parent)
                 make_target = '../' + str(relative_path)
 
-            subprocess.run(["make", make_target], cwd=cwd_path, check=True)
+            subprocess.run(["make", "-s", make_target], cwd=cwd_path, check=True)
             print(f"Makefile executed successfully for {self.name}. Rechecking for {target_path.name}...",file=stderr)
 
             if not target_path.exists():

lsms_library/dvc_permissions.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+import git
+from pathlib import Path
+import getpass
+import pkgutil
+import gnupg
+
+def is_git_repo(path='.'):
+    """
+    Check if the specified path is a Git repository.
+
+    Parameters:
+    path (str): The path to check. Default is the current directory.
+
+    Returns:
+    bool: True if the path is a valid Git repository, False otherwise.
+    """
+    try:
+        _ = git.Repo(path).git_dir
+        return True
+    except git.exc.InvalidGitRepositoryError:
+        return False
+
+def authenticate(gpg_key_file='s3_reader_creds.gpg'):
+    """
+    Decrypt the specified GPG file and store the decrypted credentials securely.
+
+    This function prompts the user for a passphrase to decrypt the provided GPG
+    file.
+
+    Parameters:
+    gpg_key_file (str): The name of the GPG file containing the encrypted credentials.
+                        Default is 's3_reader_creds.gpg'.
+
+    Raises:
+    ValueError: If decryption fails due to an incorrect passphrase or other issues.
+    """
+    # Construct the path to the encrypted file relative to this function’s location
+    gpg_path = Path(__file__).resolve().parent / 'countries' / '.dvc'
+    encrypted_file = gpg_path / gpg_key_file
+
+    # Load the encrypted data
+    encrypted_data = encrypted_file.read_bytes()
+
+    # Initialize GPG
+    gpg = gnupg.GPG()
+
+    # Prompt the user for the passphrase securely
+    passphrase = getpass.getpass(prompt='Enter passphrase for decryption: ')
+
+    # Decrypt the data
+    decrypted_data = gpg.decrypt(encrypted_data, passphrase=passphrase)
+
+    if decrypted_data.ok:
+        # Write the decrypted data to a temporary file securely
+        creds_file = gpg_path / 's3_creds'
+
+        if creds_file.exists():
+            user_input = input(f"The file {creds_file} already exists. Overwrite? (yes/no): ").strip().lower()
+            if user_input not in ['yes', 'y']:
+                print("Operation aborted. Credentials were not written.")
+                return
+
+        with open(creds_file, 'w') as f:
+            f.write(str(decrypted_data))
+        print(f"Decryption successful; credentials written to {creds_file}")
+    else:
+        raise ValueError("Decryption failed: check the passphrase and try again")

lsms_library/local_tools.py

@@ -696,11 +696,11 @@ def panel_ids(Waves):
             if isinstance(wave_info[1], list): #tanzania
                 df = wave_info[2](df, wave_info[1])
             else:
-                df[wave_info[1]] = df[wave_info[1]].apply(format_id)
-                df[wave_info[2]] = df[wave_info[2]].apply(format_id)
+                df.loc[:,wave_info[1]] = df[wave_info[1]].apply(format_id)
+                df.loc[:,wave_info[2]] = df[wave_info[2]].apply(format_id)
                 # If a transformation function is provided (tuple length 4), apply it to the old_id column
                 if len(wave_info) == 4:
-                    df[wave_info[2]] = df[wave_info[2]].apply(wave_info[3])
+                    df.loc[:,wave_info[2]] = df[wave_info[2]].apply(wave_info[3])
                 df['t'] = wave_year
                 df = df.rename(columns={wave_info[1]: 'i', wave_info[2]: 'previous_i'})
                 df = df.set_index(['t', 'i'])[['previous_i']]

lsms_library-0.2.10.dev0.dist-info/METADATA

@@ -0,0 +1,290 @@
+Metadata-Version: 2.3
+Name: LSMS_Library
+Version: 0.2.10.dev0
+Summary: Abstraction layer for Living Standards Measurement Survey data
+License: CC-BY-4.0
+Author: Ethan Ligon
+Author-email: ligon@berkeley.edu
+Requires-Python: >=3.11, <4.0
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: CFEDemands (>=0.7.0)
+Requires-Dist: PyYAML
+Requires-Dist: dvc[s3] (>=3.61.0,<4.0.0)
+Requires-Dist: ligonlibrary (>=0.1.1,<0.2.0)
+Requires-Dist: lsms (>=0.4.7,<0.5.0)
+Requires-Dist: openai (>=1.99.5,<2.0.0)
+Requires-Dist: pandas (>=2.2,<3.0)
+Requires-Dist: pyarrow (>=21.0.0,<22.0.0)
+Requires-Dist: pyreadstat (>=1.3.0,<2.0.0)
+Requires-Dist: python-gnupg
+Requires-Dist: scikit-learn (>=1.7.0,<2.0.0)
+Requires-Dist: xarray (>=2025.7.1,<2026.0.0)
+Description-Content-Type: text/plain
+
+#+TITLE: LSMS_Library
+#+AUTHOR: Ethan Ligon
+#+OPTIONS: toc:nil
+
+[[https://doi.org/10.5281/zenodo.17258079][https://zenodo.org/badge/796958546.svg]]
+
+A Python library providing a uniform interface to Living Standards Measurement Study (LSMS) household surveys from multiple countries and years, without the data loss typical of traditional harmonization approaches.
+
+* The Problem
+
+LSMS datasets are invaluable for studying poverty, consumption, and household welfare across developing countries. However, each country's survey uses different:
+- Variable names and encodings
+- Food classification systems
+- Questionnaire structures
+- File formats and organization
+
+Researchers typically spend weeks learning each new dataset's idiosyncrasies or use pre-harmonized datasets that sacrifice detail and comparability. Cross-country or longitudinal analyses become prohibitively time-consuming.
+
+* The Solution
+
+LSMS_Library provides an *abstraction layer* that gives you a consistent interface to work with any supported LSMS dataset. Instead of harmonizing the data itself (which loses information), we harmonize the /way you access/ the data.
+
+This means you can:
+- Write analysis code once and apply it to multiple countries/years
+- Switch between datasets without rewriting your code
+- Preserve the full detail and structure of the original surveys
+- Extend support to new surveys by writing simple YAML configuration files
+
+* Quick Start
+
+#+begin_src python
+import lsms_library as ll
+
+# Load a country's LSMS data
+uga = ll.Country('Uganda')
+
+# See available survey waves
+print(uga.waves)
+# ['2005-06', '2009-10', '2010-11', '2011-12', '2013-14', '2015-16', '2018-19', '2019-20']
+
+# See available standardized data types
+print(uga.data_scheme)
+# ['people_last7days', 'food_acquired', 'food_expenditures', 'food_prices', 
+#  'household_characteristics', 'income', 'nutrition', ...]
+
+# Access standardized food expenditure data across all waves
+food_exp = uga.food_expenditures()
+# Returns a multi-indexed DataFrame with household, time, region, and food item
+#+end_src
+
+* Key Features
+
+- *Uniform Interface*: Access variables using consistent names across countries (e.g., =food_expenditures()=, =household_characteristics()=)
+- *Multi-Wave Panel Support*: For countries with panel surveys, household IDs are automatically standardized across waves, enabling longitudinal analysis without manual matching
+- *Zero Data Loss*: Original data structure and detail preserved; access raw data through the same interface
+- *Standardized Data Schemes*: Common data types (=food_prices=, =nutrition=, =income=, etc.) mapped across all countries
+- *DVC Integration*: Stream data from remote storage without filling your disk
+- *Extensible*: Add new surveys by creating YAML configuration files (no Python required)
+- *Multiple Countries*: Supports LSMS surveys from Nigeria, Tanzania, Uganda, Ethiopia, Malawi, and more
+
+* Installation
+
+#+begin_src bash
+pip install LSMS_Library
+#+end_src
+
+** Data Access
+
+The library uses DVC (Data Version Control) to manage data stored in remote S3 buckets. To access the data, you'll need credentials:
+
+- *Read access*: Contact ligon@berkeley.edu for read-only credentials to access the data
+- *Write access (contributors)*: To contribute new datasets, contact ligon@berkeley.edu for write credentials. You'll need to establish [[https://docs.github.com/en/authentication/managing-commit-signature-verification/generating-a-new-gpg-key][GPG/PGP credentials]] for secure access.
+
+Once you have credentials, the library will handle data streaming automatically.
+
+* Usage Examples
+
+** Working with Food Consumption Data
+
+#+begin_src python
+import lsms_library as ll
+
+# Load country data
+uga = ll.Country('Uganda')
+tza = ll.Country('Tanzania')
+
+# Access food expenditure data with consistent structure
+uga_food = uga.food_expenditures()
+tza_food = tza.food_expenditures()
+
+# Both return DataFrames with the same multi-index structure:
+# Index: (household_id, time, region, food_item)
+# Even though the original surveys have completely different formats
+
+# Access other standardized data types
+prices = uga.food_prices()
+nutrition = uga.nutrition()
+income = uga.income()
+#+end_src
+
+** Cross-Country Comparison
+
+#+begin_src python
+import lsms_library as ll
+import pandas as pd
+
+# Load multiple countries
+countries = {
+    'Uganda': ll.Country('Uganda'),
+    'Tanzania': ll.Country('Tanzania'),
+    'Nigeria': ll.Country('Nigeria')
+}
+
+# Collect food expenditure data from all countries
+expenditure_data = {}
+for name, country in countries.items():
+    df = country.food_expenditures()
+    df['country'] = name
+    expenditure_data[name] = df
+
+# Combine into a single DataFrame for analysis
+combined = pd.concat(expenditure_data.values(), ignore_index=False)
+
+# Now you can analyze across countries with consistent variable names
+# e.g., compare rice prices, consumption patterns, etc.
+#+end_src
+
+** Panel Data Analysis
+
+For countries with panel surveys, household IDs are already harmonized across waves:
+
+#+begin_src python
+import lsms_library as ll
+
+# Load a country with panel data
+uga = ll.Country('Uganda')
+
+# Get food expenditures across all waves
+food_exp = uga.food_expenditures()
+
+# The multi-index includes time (wave), so you can track households over time
+# Index levels: (household_id, time, region, food_item)
+
+# Example: Track a specific household across waves
+household_id = '00c9353d8ebe42faabf5919b81d7fae7'
+household_over_time = food_exp.xs(household_id, level='i')
+
+# Or analyze changes between specific waves
+wave_2015 = food_exp.xs('2015-16', level='t')
+wave_2019 = food_exp.xs('2019-20', level='t')
+
+# Check panel structure and attrition patterns
+panel_structure = ll.local_tools.panel_attrition(
+    uga.household_characteristics(), 
+    uga.waves
+)
+# Returns a matrix showing number of households appearing in each wave pair:
+#         2005-06 2009-10 2010-11 2011-12 2013-14 2015-16 2018-19 2019-20
+# 2005-06    3122    2606    2386    2363    1566    1470    1358    1290
+# 2009-10     NaN    2974    2617    2581    1685    1578    1454    1379
+# ...
+# Diagonal shows total households per wave; off-diagonal shows panel overlap
+#+end_src
+
+** Exploring Available Data
+
+#+begin_src python
+import lsms_library as ll
+
+uga = ll.Country('Uganda')
+
+# See all available survey waves
+print(uga.waves)
+# ['2005-06', '2009-10', '2010-11', '2011-12', '2013-14', '2015-16', '2018-19', '2019-20']
+
+# See all standardized data types available
+print(uga.data_scheme)
+# ['people_last7days', 'cluster_features', 'shocks', 'earnings',
+#  'food_acquired', 'nutrition', 'household_characteristics',
+#  'food_quantities', 'food_expenditures', 'food_prices',
+#  'panel_ids', 'income', 'enterprise_income', 'other_features']
+
+# Access any standardized data type using the same pattern
+household_chars = uga.household_characteristics()
+shocks = uga.shocks()
+earnings = uga.earnings()
+#+end_src
+
+* Available Datasets
+
+The library currently supports LSMS surveys from:
+- *Ethiopia*: Multiple waves from the LSMS-ISA program
+- *Malawi*: Multiple waves including panel data
+- *Nigeria*: GHS-Panel surveys
+- *Tanzania*: NPS surveys
+- *Uganda*: UNPS surveys
+- And more...
+
+For a complete list of available surveys, see the country directories in the repository.
+
+* Adding New Surveys
+
+Adding a new LSMS survey requires no Python programming—just create YAML configuration files that map the survey's variables to the standardized interface. See [[file:CONTRIBUTING.org][CONTRIBUTING.org]] for detailed instructions.
+
+Brief overview:
+1. Create directory structure: =Country/Year/Documentation= and =Country/Year/Data=
+2. Add source data using DVC
+3. Create YAML files mapping variables to standard names
+4. Submit a pull request
+
+* Documentation
+
+- *Food Classification*: Food items are standardized for spelling and format within each country. Note that food categories differ significantly across countries (e.g., what constitutes "Beans" in Uganda may not match Tanzania's classification), so cross-country food comparisons should be done carefully.
+- *Variable Mappings*: YAML files in each survey directory show how local variables map to standard names
+- *Panel IDs*: For countries with panel surveys, household identifiers are harmonized automatically across waves
+- *API Reference*: [Coming soon]
+
+* Contributing
+
+We welcome contributions! Whether you're:
+- Adding new survey datasets
+- Improving variable mappings
+- Fixing bugs
+- Improving documentation
+
+See [[file:CONTRIBUTING.org][CONTRIBUTING.org]] for detailed guidelines on adding new datasets using DVC.
+
+* Citation
+
+If you use LSMS_Library in your research, please cite:
+
+#+begin_src bibtex
+@software{ligon25:lsms_library,
+  author =    {Ethan Ligon},
+  title =     {{\tt LSMS_Library}: Abstraction layer for working with Living Standards Measurement Surveys},
+  year =      2025,
+  doi = {10.5281/zenodo.17258079},
+  url = {https://pypi.org/project/lsms_library/}
+}
+#+end_src
+
+* License
+
+See the [[file:LICENSE][LICENSE]] file in the repository for details.
+
+* Contact
+
+For questions, issues, or collaboration:
+- *Data Access*: Email ligon@berkeley.edu for read or write credentials
+- *GitHub Issues*: Report bugs or request features at the repository
+- *Contributing*: Contact ligon@berkeley.edu to discuss contributions (GPG/PGP credentials required for write access)
+
+* Acknowledgments
+
+This project builds on data collection efforts by:
+- The World Bank's Living Standards Measurement Study (LSMS) team
+- National statistical offices in participating countries
+- The LSMS-ISA initiative
+
+---
+
+*Note*: This library is under active development. APIs may change as we refine the abstraction layer based on user feedback.
+