instate 1.0.0__py3-none-any.whl

instate/__init__.py

@@ -0,0 +1,35 @@
+"""
+instate: Predict state and language from Indian lastnames.
+
+This package provides functions to:
+1. Look up state distributions from 2017 Indian electoral rolls
+2. Predict states and languages using neural networks
+
+Main functions:
+- get_state_distribution: Get P(state|lastname) from electoral rolls
+- get_state_languages: Map states to official languages
+- predict_state: Neural prediction of most likely states
+- predict_language: Neural prediction of most likely languages
+"""
+
+from .electoral import (
+    get_state_distribution,
+    get_state_languages,
+    list_available_states,
+)
+from .predict import predict_language, predict_state
+
+__all__ = [
+    "get_state_distribution",
+    "get_state_languages",
+    "predict_state",
+    "predict_language",
+    "list_available_states",
+]
+
+try:
+    from importlib.metadata import version
+
+    __version__ = version("instate")
+except ImportError:
+    __version__ = "1.0.0"

instate/_utils.py

@@ -0,0 +1,357 @@
+"""
+Internal utilities for instate package.
+
+This module contains helper functions for data loading, name cleaning, and caching.
+These are internal utilities and not part of the public API.
+"""
+
+from __future__ import annotations
+
+import os
+
+import pandas as pd
+import requests
+import torch
+import torch.nn as nn
+from tqdm import tqdm
+
+# Cache for loaded data
+_CACHE = {}
+
+# URLs for downloading data
+ELECTORAL_DATA_URLS = {
+    "v1": "https://github.com/appeler/instate/raw/main/data/instate_unique_ln_state_prop_v1.csv.gz",
+}
+
+MODEL_URLS = {
+    "gru": "https://dataverse.harvard.edu/api/v1/access/datafile/6981460",
+}
+
+
+def prepare_name_dataframe(
+    names: pd.DataFrame | list[str], name_column: str | None = None
+) -> pd.DataFrame:
+    """Convert input to DataFrame with standardized name column.
+
+    Args:
+        names: DataFrame or list of names
+        name_column: Column name if DataFrame provided
+
+    Returns:
+        DataFrame with names in first column
+    """
+    if isinstance(names, list):
+        return pd.DataFrame({"name": names})
+
+    df = names.copy()
+
+    # If no column specified, try to find one
+    if name_column is None:
+        # Look for common name columns
+        possible_cols = [
+            c
+            for c in df.columns
+            if any(n in c.lower() for n in ["name", "lastname", "surname"])
+        ]
+        if not possible_cols:
+            # Just use first column
+            name_column = df.columns[0]
+        else:
+            name_column = possible_cols[0]
+
+    # Ensure the name column is first
+    if name_column != df.columns[0]:
+        cols = [name_column] + [c for c in df.columns if c != name_column]
+        df = df[cols]
+
+    return df
+
+
+def clean_name(name: str) -> str:
+    """Clean and standardize a single name.
+
+    - Convert to lowercase
+    - Strip whitespace
+    - Remove non-alphabetic characters
+
+    Args:
+        name: Input name string
+
+    Returns:
+        Cleaned name
+    """
+    if not isinstance(name, str):
+        return ""
+
+    # Basic cleaning
+    cleaned = name.strip().lower()
+
+    # Keep only alphabetic characters
+    cleaned = "".join(c for c in cleaned if c.isalpha())
+
+    return cleaned
+
+
+def clean_names_in_df(df: pd.DataFrame, name_column: str) -> pd.DataFrame:
+    """Clean names in a DataFrame column.
+
+    Args:
+        df: Input DataFrame
+        name_column: Column containing names
+
+    Returns:
+        DataFrame with added __cleaned_name column and filtered rows
+    """
+    result = df.copy()
+
+    # Handle empty DataFrame
+    if len(result) == 0:
+        result["__cleaned_name"] = pd.Series([], dtype=str)
+        return result
+
+    # Clean names
+    result["__cleaned_name"] = result[name_column].apply(clean_name)
+
+    # Filter out invalid names
+    result = result[result["__cleaned_name"].str.len() > 2]
+
+    # Drop duplicates based on cleaned name
+    result = result.drop_duplicates(subset=["__cleaned_name"], keep="first")
+
+    return result
+
+
+def get_app_file_path(filename: str) -> str:
+    """Get path for cached application data.
+
+    Args:
+        filename: Name of file
+
+    Returns:
+        Full path to file in app data directory
+    """
+    user_dir = os.path.expanduser("~")
+    app_data_dir = os.path.join(user_dir, ".instate")
+
+    if not os.path.exists(app_data_dir):
+        os.makedirs(app_data_dir)
+
+    return os.path.join(app_data_dir, filename)
+
+
+def download_file(url: str, target: str) -> bool:
+    """Download file with progress bar.
+
+    Args:
+        url: URL to download from
+        target: Target file path
+
+    Returns:
+        True if successful, False otherwise
+    """
+    try:
+        r = requests.get(url, stream=True)
+
+        if r.status_code == 200:
+            chunk_size = 64 * 1024
+            total_size = int(r.headers.get("content-length", 0)) / chunk_size + 1
+
+            with open(target, "wb") as f:
+                for data in tqdm(
+                    r.iter_content(chunk_size),
+                    total=round(total_size, 1),
+                    unit_scale=chunk_size / 1024,
+                    unit="KB",
+                    desc="Downloading",
+                ):
+                    f.write(data)
+            return True
+        else:
+            print(f"Download failed with status code: {r.status_code}")
+            return False
+    except Exception as e:
+        print(f"Download error: {e}")
+        return False
+
+
+def load_electoral_data(dataset: str = "v1") -> pd.DataFrame:
+    """Load electoral rolls data, downloading if needed.
+
+    Args:
+        dataset: Dataset version to load
+
+    Returns:
+        DataFrame with electoral rolls data
+    """
+    global _CACHE
+
+    cache_key = f"electoral_{dataset}"
+    if cache_key in _CACHE:
+        return _CACHE[cache_key]
+
+    # Check if data exists locally
+    filename = f"instate_unique_ln_state_prop_{dataset}.csv.gz"
+    data_path = get_app_file_path(filename)
+
+    if not os.path.exists(data_path):
+        # First try to find the file in the package data directory
+        package_data_dir = os.path.join(os.path.dirname(__file__), "data")
+        local_path = os.path.join(package_data_dir, filename)
+
+        if os.path.exists(local_path):
+            print("Copying electoral rolls data from package...")
+            import shutil
+
+            shutil.copy2(local_path, data_path)
+        else:
+            print(f"Downloading electoral rolls data ({dataset})...")
+            if not download_file(ELECTORAL_DATA_URLS[dataset], data_path):
+                raise RuntimeError("Failed to download electoral data")
+
+    # Load data
+    df = pd.read_csv(data_path)
+    df.rename(columns={"last_name": "__last_name"}, inplace=True)
+
+    # Cache it
+    _CACHE[cache_key] = df
+
+    return df
+
+
+def load_gru_model():
+    """Load GRU model for state prediction.
+
+    Returns:
+        Loaded PyTorch model
+    """
+    global _CACHE
+
+    if "gru_model" in _CACHE:
+        return _CACHE["gru_model"]
+
+    # Check if model exists
+    model_path = get_app_file_path("instate_gru.pth")
+
+    if not os.path.exists(model_path):
+        print("Downloading GRU model...")
+        if not download_file(MODEL_URLS["gru"], model_path):
+            raise RuntimeError("Failed to download GRU model")
+
+    # Load model
+    from .constants import GRU_HIDDEN_SIZE, GRU_N_LETTERS, GT_KEYS
+    from .nnets import GRU_net
+
+    device = torch.device("cpu")
+    model = GRU_net(GRU_N_LETTERS, GRU_HIDDEN_SIZE, len(GT_KEYS))
+    model.load_state_dict(torch.load(model_path, map_location=device))
+    model.eval()
+
+    _CACHE["gru_model"] = model
+
+    return model
+
+
+def load_lstm_model():
+    """Load LSTM model for language prediction.
+
+    Returns:
+        Loaded PyTorch model and supporting data
+    """
+    global _CACHE
+
+    if "lstm_model" in _CACHE:
+        return _CACHE["lstm_model"], _CACHE["lstm_data"]
+
+    # Import constants instead of loading from files
+    from .constants import (
+        CHAR_TO_IDX,
+        IDX_TO_LANG,
+        NUM_LANGUAGES,
+        VOCAB_SIZE,
+    )
+    # LanguagePredictor is now defined in this file
+
+    # Model configuration
+    embedding_dim = 50
+    hidden_dim = 128
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    model = LanguagePredictor(VOCAB_SIZE, embedding_dim, hidden_dim, NUM_LANGUAGES)
+    model.to(device)
+
+    # Load weights
+    data_dir = os.path.dirname(__file__)
+    model_file = os.path.join(data_dir, "data", "state_lang_labels.pt")
+    if torch.cuda.is_available():
+        model.load_state_dict(torch.load(model_file))
+    else:
+        model.load_state_dict(torch.load(model_file, map_location=torch.device("cpu")))
+
+    model.eval()
+
+    # Cache everything
+    lstm_data = {"char2idx": CHAR_TO_IDX, "idx2lang": IDX_TO_LANG, "device": device}
+
+    _CACHE["lstm_model"] = model
+    _CACHE["lstm_data"] = lstm_data
+
+    return model, lstm_data
+
+
+def load_language_lookup_data():
+    """Load data for KNN language lookup.
+
+    Returns:
+        DataFrame with lastname to language mapping
+    """
+    import tarfile
+
+    global _CACHE
+
+    if "lang_lookup" in _CACHE:
+        return _CACHE["lang_lookup"]
+
+    data_file_name = "lastname_langs_india"
+    data_path = get_app_file_path(data_file_name)
+
+    if not os.path.exists(data_path + ".csv"):
+        data_dir = os.path.dirname(__file__)
+        gz_path = os.path.join(data_dir, "data", f"{data_file_name}.csv.tar.gz")
+        print("Extracting language lookup data...")
+        with tarfile.open(gz_path, "r:gz") as tar:
+            tar.extract(f"{data_file_name}.csv", data_path, filter="data")
+
+    df = pd.read_csv(f"{data_path}/{data_file_name}.csv")
+    _CACHE["lang_lookup"] = df
+
+    return df
+
+
+class LanguagePredictor(nn.Module):
+    """LSTM model for predicting languages from names.
+
+    This model uses character embeddings and LSTM to predict the top 3 most
+    likely languages for a given name.
+    """
+
+    def __init__(
+        self, num_chars, embedding_dim=64, lstm_hidden_dim=128, num_languages=37
+    ):
+        super().__init__()
+        self.embedding = nn.Embedding(num_chars, embedding_dim)
+        self.lstm = nn.LSTM(embedding_dim, lstm_hidden_dim, batch_first=True)
+        self.fc1 = nn.Linear(lstm_hidden_dim, num_languages)
+        self.fc2 = nn.Linear(lstm_hidden_dim, num_languages)
+        self.fc3 = nn.Linear(lstm_hidden_dim, num_languages)
+
+    def forward(self, x, lengths):
+        x = self.embedding(x)
+        x = nn.utils.rnn.pack_padded_sequence(
+            x, lengths, batch_first=True, enforce_sorted=False
+        )
+        _, (h_n, _) = self.lstm(x)
+        h_n = h_n.squeeze(0)
+        out1 = self.fc1(h_n)
+        out2 = self.fc2(h_n)
+        out3 = self.fc3(h_n)
+        return out1, out2, out3

instate/constants.py

@@ -0,0 +1,127 @@
+"""
+Constants for the instate package.
+
+Contains static data that was previously stored in small files.
+"""
+
+from __future__ import annotations
+
+# Language list (originally from langs.txt)
+LANGUAGES = [
+    "sindhi",
+    "nepali",
+    "kannada",
+    "marathi",
+    "mizo",
+    "adi",
+    "garo",
+    "tagin",
+    "assamese",
+    "hindi",
+    "odia",
+    "french",
+    "punjabi",
+    "naga languages",
+    "english",
+    "chenchu",
+    "urdu",
+    "bengali",
+    "maithili",
+    "dogri",
+    "kokborok",
+    "santali",
+    "kashmiri",
+    "gujarati",
+    "apatani",
+    "tulu",
+    "konkani",
+    "telugu",
+    "malayalam",
+    "tamil",
+    "meitei",
+    "khasi",
+    "gondi",
+    "bodo",
+    "nishi",
+    "chakma",
+    "pahari and kumauni",
+]
+
+# Character to index mapping (originally from char2idx.json)
+CHAR_TO_IDX = {
+    "<PAD>": 0,
+    "n": 1,
+    "g": 2,
+    "i": 3,
+    "m": 4,
+    "c": 5,
+    "w": 6,
+    "u": 7,
+    "e": 8,
+    "v": 9,
+    "d": 10,
+    "a": 11,
+    "l": 12,
+    "t": 13,
+    "s": 14,
+    "q": 15,
+    "b": 16,
+    "f": 17,
+    "o": 18,
+    "z": 19,
+    "p": 20,
+    "r": 21,
+    "k": 22,
+    "h": 23,
+    "y": 24,
+    "x": 25,
+    "j": 26,
+}
+
+# Derived mappings for convenience
+LANG_TO_IDX = {lang: idx for idx, lang in enumerate(LANGUAGES)}
+IDX_TO_LANG = {idx: lang for lang, idx in LANG_TO_IDX.items()}
+
+# Model dimensions
+VOCAB_SIZE = len(CHAR_TO_IDX)
+NUM_LANGUAGES = len(LANGUAGES)
+
+# GRU model constants for state prediction
+GT_KEYS = [
+    "Andaman and Nicobar Islands",
+    "Andhra Pradesh",
+    "Arunachal Pradesh",
+    "Assam",
+    "Bihar",
+    "Chandigarh",
+    "Dadra and Nagar Haveli",
+    "Daman and Diu",
+    "Delhi",
+    "Goa",
+    "Gujarat",
+    "Haryana",
+    "Jharkhand",
+    "Jammu and Kashmir and Ladakh",
+    "Karnataka",
+    "Kerala",
+    "Maharashtra",
+    "Manipur",
+    "Meghalaya",
+    "Mizoram",
+    "Madhya Pradesh",
+    "Nagaland",
+    "Odisha",
+    "Puducherry",
+    "Punjab",
+    "Rajasthan",
+    "Sikkim",
+    "Telangana",
+    "Tripura",
+    "Uttar Pradesh",
+    "Uttarakhand",
+]
+
+# GRU model configuration
+GRU_HIDDEN_SIZE = 2048
+GRU_ALL_LETTERS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.,;"
+GRU_N_LETTERS = len(GRU_ALL_LETTERS)

instate/data/state_to_languages.csv

@@ -0,0 +1,36 @@
+state,official_languages,addl_official_languages,most_spoken_lang,second_most_spoken_lang,third_most_spoken_lang,fourth_most_spoken_lang,fifth_most_spoken_lang
+Andaman and Nicobar Islands,"Hindi, English",,Bengali,Hindi,Tamil,Telugu,Malayalam
+Andhra Pradesh,Telugu,Urdu,Telugu,Urdu,Tamil,Chenchu,Kannada
+Arunachal Pradesh,English,,Nishi,Adi,Apatani,Tagin,Hindi
+Assam,Assamese,"Bengali, Bodo",Assamese,Bengali,Bodo,Hindi,Nepali
+Bihar,Hindi,Urdu,Hindi (including Bihari languages),Maithili,Urdu,Bengali,Santali
+Chandigarh,English,,Hindi,Punjabi,Urdu,Nepali,Bengali
+Chhattisgarh,Hindi,Chhattisgarhi,Hindi,Odia,Bengali,Telugu,Marathi
+Dadra and Nagar Haveli and Daman and Diu,"Hindi, English",Gujarati,Gujarati,Hindi,Marathi,Konkani,Bengali
+Delhi,"Hindi, English","Urdu, Punjabi",Hindi,Punjabi,Urdu,Bengali,Maithili
+Goa,Konkani,Marathi,Konkani,Marathi,Hindi,Kannada,Urdu
+Gujarat,Gujarati,,Gujarati,Hindi,Marathi,Sindhi,Urdu
+Haryana,Hindi,Punjabi,Hindi,Punjabi,Urdu,Bengali,Maithili
+Himachal Pradesh,Hindi,Sanskrit,Hindi (including Pahari languages),Punjabi,Nepali,Kashmiri,Dogri
+Jammu and Kashmir and Ladakh,"Kashmiri, Dogri, Hindi, Urdu, English",,Kashmiri,Dogri,Hindi,Punjabi,Urdu
+Jharkhand,Hindi,"Angika, Bengali, Bhojpuri, Bhumij, Ho, Kharia, Khortha, Kurmali, Kurukh, Magahi, Maithili, Mundari, Nagpuri, Odia, Santali, Urdu",Hindi (including Bihari languages),Santali,Bengali,Urdu,Odia
+Karnataka,Kannada,,Kannada,Telugu,Tamil,Marathi,Kodava
+Kerala,Malayalam,English,Malayalam,Tamil,Tulu,Kannada,Konkani
+Lakshadweep,"English, Hindi",,Malayalam,Dhivehi,Tamil,Hindi,Telugu
+Madhya Pradesh,Hindi,,Hindi,Marathi,Urdu,Sindhi,Gujarati
+Maharashtra,Marathi,,Marathi,Hindi,Urdu,Gujarati,Telugu
+Manipur,Meitei,English,Meitei,Nepali,Hindi,Bengali,English
+Meghalaya,English,Khasi,Khasi,Garo,Bengali,Nepali,Hindi
+Mizoram,"English, Hindi, Mizo",,Mizo,English,Hindi,Meitei,Chakma
+Nagaland,English,,Naga languages,English,Hindi,Assamese,Meitei
+Odisha,Odia,,Odia,Hindi,Santali,Urdu,Telugu
+Puducherry,"Tamil, English, French",,Tamil,Telugu,Malayalam,French,English
+Punjab,Punjabi,,Punjabi,Hindi,Urdu,Bengali,English
+Rajasthan,Hindi,English,Hindi (including Rajasthani languages),Punjabi,Gujarati,Sindhi,Urdu
+Sikkim,"English, Nepali","Bhutia, Gurung, Lepcha, Limbu, Manggar, Mukhia, Newari, Rai, Sherpa, Tamang",Nepali,Hindi,Bengali,Urdu,Punjabi
+Tamil Nadu,Tamil,English,Tamil,Telugu,Kannada,Urdu,Malayalam
+Telangana,Telugu,Urdu,Telugu,Urdu,Gondi,Kannada,Marathi
+Tripura,"Bengali, English, Kokborok",,Bengali,Kokborok,Chakma,English,Meitei
+Uttar Pradesh,Hindi,Urdu,Hindi (including UP local dialects),Urdu,Punjabi,Nepali,Bengali
+Uttarakhand,Hindi,Sanskrit,Hindi,Pahari and Kumauni,Urdu,Punjabi,Bengali
+West Bengal,"Bengali, English",,Bengali,Hindi,Santali,Urdu,Nepali

instate/electoral.py

@@ -0,0 +1,141 @@
+"""
+Electoral rolls based name-to-state lookup.
+
+Functions for looking up state distributions from 2017 Indian electoral rolls data.
+"""
+
+from __future__ import annotations
+
+import os
+
+import pandas as pd
+
+
+def get_state_distribution(
+    names: pd.DataFrame | list[str], name_column: str | None = None
+) -> pd.DataFrame:
+    """Get P(state|lastname) from 2017 Indian electoral rolls.
+
+    This returns the empirical distribution of a lastname across Indian states
+    based on the electoral rolls data. This is the Bayes optimal estimate
+    given the observed frequencies.
+
+    Args:
+        names: DataFrame containing names or list of name strings.
+            Names are automatically cleaned (lowercase, stripped).
+        name_column: If names is a DataFrame, the column containing names.
+            If None and DataFrame has 'name' or 'lastname', uses that.
+
+    Returns:
+        DataFrame with original data plus 31 state probability columns.
+        State columns are named by state (e.g., 'delhi', 'punjab').
+        Values are proportions (0-1) representing P(state|lastname).
+
+    Examples:
+        >>> names = ["dhingra", "sood", "gowda"]
+        >>> result = get_state_distribution(names)
+        >>> result[["name", "delhi", "punjab", "karnataka"]]
+
+        >>> df = pd.DataFrame({"lastname": ["dhingra", "sood"]})
+        >>> result = get_state_distribution(df, "lastname")
+        >>> result.columns[:5].tolist()
+    """
+    from ._utils import clean_names_in_df, load_electoral_data, prepare_name_dataframe
+
+    # Convert to DataFrame if needed
+    df = prepare_name_dataframe(names, name_column)
+
+    # Clean names for matching
+    df = clean_names_in_df(df, df.columns[0])
+
+    # Load electoral rolls data
+    electoral_data = load_electoral_data()
+
+    # Merge to get state distributions
+    # Electoral data has __last_name as key
+    result = pd.merge(
+        df, electoral_data, left_on="__cleaned_name", right_on="__last_name", how="left"
+    )
+
+    # Drop temporary columns
+    result = result.drop(columns=["__cleaned_name", "__last_name"], errors="ignore")
+
+    return result
+
+
+def get_state_languages(
+    states: pd.DataFrame | list[str], state_column: str | None = None
+) -> pd.DataFrame:
+    """Map Indian states to their official languages.
+
+    Based on census data, returns the official language(s) for each state.
+
+    Args:
+        states: DataFrame containing states or list of state names.
+        state_column: If states is a DataFrame, the column containing state names.
+
+    Returns:
+        DataFrame with state and official_languages columns.
+        If input was DataFrame, adds official_languages column.
+
+    Examples:
+        >>> states = ["Delhi", "Punjab", "Karnataka"]
+        >>> result = get_state_languages(states)
+        >>> result[["state", "official_languages"]]
+
+        >>> df = pd.DataFrame({"state_name": ["Delhi", "Punjab"]})
+        >>> result = get_state_languages(df, "state_name")
+    """
+    # Prepare DataFrame
+    if isinstance(states, list):
+        df = pd.DataFrame({"state": states})
+        state_col = "state"
+    else:
+        df = states.copy()
+        if state_column is None:
+            # Try to find state column
+            possible_cols = [c for c in df.columns if "state" in c.lower()]
+            if not possible_cols:
+                raise ValueError("state_column must be specified for DataFrame input")
+            state_col = possible_cols[0]
+        else:
+            state_col = state_column
+
+    # Load state-language mapping
+    data_dir = os.path.dirname(__file__)
+    state_lang_path = os.path.join(data_dir, "data", "state_to_languages.csv")
+    state_lang_map = pd.read_csv(state_lang_path)
+
+    # Merge to add languages
+    result = df.merge(state_lang_map, left_on=state_col, right_on="state", how="left")
+
+    # Clean up duplicate state column if needed
+    if state_col != "state" and "state_y" not in result.columns:
+        result = result.drop(columns=["state"], errors="ignore")
+
+    return result
+
+
+def list_available_states() -> list[str]:
+    """List all states available in the electoral rolls dataset.
+
+    Returns:
+        List of state names available in the data.
+
+    Examples:
+        >>> states = list_available_states()
+        >>> len(states)
+        31
+        >>> "Delhi" in states
+        True
+    """
+    from ._utils import load_electoral_data
+
+    electoral_data = load_electoral_data()
+    # Get column names, excluding non-state columns
+    state_cols = [
+        col
+        for col in electoral_data.columns
+        if not col.startswith("__") and col != "last_name"
+    ]
+    return sorted(state_cols)

instate/nnets.py

@@ -0,0 +1,46 @@
+import torch
+import torch.nn as nn
+
+from .constants import GRU_ALL_LETTERS, GRU_HIDDEN_SIZE, GRU_N_LETTERS
+
+# For backward compatibility with existing code
+n_hidden = GRU_HIDDEN_SIZE
+all_letters = GRU_ALL_LETTERS
+n_letters = GRU_N_LETTERS
+
+
+def infer(net, name: str):
+    net.eval()
+    name_ohe = name_rep(name)
+    hidden = net.init_hidden()
+
+    for i in range(name_ohe.size()[0]):
+        output, hidden = net(name_ohe[i], hidden)
+
+    return output
+
+
+def name_rep(name: str):
+    rep = torch.zeros(len(name), 1, n_letters)
+    for index, letter in enumerate(name):
+        pos = all_letters.find(letter)
+        rep[index][0][pos] = 1
+    return rep
+
+
+class GRU_net(nn.Module):
+    def __init__(self, input_size, hidden_size, output_size):
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.gru_cell = nn.GRU(input_size, hidden_size)
+        self.h2o = nn.Linear(hidden_size, output_size)
+        self.softmax = nn.LogSoftmax(dim=2)
+
+    def forward(self, input_, hidden):
+        out, hidden = self.gru_cell(input_.view(1, 1, -1), hidden)
+        output = self.h2o(hidden)
+        output = self.softmax(output)
+        return output.view(1, -1), hidden
+
+    def init_hidden(self):
+        return torch.zeros(1, 1, self.hidden_size)

instate/predict.py

@@ -0,0 +1,235 @@
+"""
+Neural network predictions for names not in electoral rolls.
+
+Functions for predicting states and languages using trained models.
+"""
+
+from __future__ import annotations
+
+from functools import partial
+
+import pandas as pd
+import torch
+from Levenshtein import distance
+
+
+def predict_state(
+    names: pd.DataFrame | list[str],
+    name_column: str | None = None,
+    top_k: int = 3,
+    model: str = "gru",
+) -> pd.DataFrame:
+    """Predict most likely Indian states for given names using neural network.
+
+    Uses a trained GRU model to predict which Indian states a person with
+    the given lastname is most likely to be from. This is useful for names
+    not found in the electoral rolls data.
+
+    Args:
+        names: DataFrame containing names or list of name strings.
+            Names are automatically cleaned (lowercase, stripped).
+        name_column: If names is a DataFrame, the column containing names.
+        top_k: Number of top states to return (default: 3).
+        model: Model to use for prediction. Currently only "gru" supported.
+
+    Returns:
+        DataFrame with name and predicted_states columns.
+        predicted_states contains a list of top_k state names.
+
+    Examples:
+        >>> names = ["dhingra", "sood", "gowda"]
+        >>> result = predict_state(names, top_k=3)
+        >>> result["predicted_states"][0]
+        ['Delhi', 'Punjab', 'Haryana']
+
+        >>> df = pd.DataFrame({"lastname": ["sharma", "patel"]})
+        >>> result = predict_state(df, "lastname", top_k=2)
+        >>> len(result["predicted_states"][0])
+        2
+    """
+    from ._utils import clean_name, load_gru_model, prepare_name_dataframe
+    from .constants import GT_KEYS
+    from .nnets import infer
+
+    if model != "gru":
+        raise ValueError(f"Model '{model}' not supported. Use 'gru'.")
+
+    # Prepare DataFrame
+    df = prepare_name_dataframe(names, name_column)
+    name_col = df.columns[0]
+
+    # Load model
+    gru_model = load_gru_model()
+
+    # Predict for each name
+    predictions = []
+    for name in df[name_col]:
+        cleaned = clean_name(name)
+        if not cleaned or len(cleaned) < 3:
+            predictions.append([])
+            continue
+
+        # Run inference
+        output = infer(gru_model, cleaned)
+        _, indices = output.topk(top_k)
+        idx_list = indices.numpy().flatten().tolist()
+        pred_states = [GT_KEYS[i] for i in idx_list]
+        predictions.append(pred_states)
+
+    # Add predictions to DataFrame
+    result = df.copy()
+    result["predicted_states"] = predictions
+
+    return result
+
+
+def predict_language(
+    names: pd.DataFrame | list[str],
+    name_column: str | None = None,
+    top_k: int = 3,
+    model: str = "lstm",
+) -> pd.DataFrame:
+    """Predict most likely languages for given names.
+
+    Two methods available:
+    - "lstm": Neural network prediction using trained LSTM model
+    - "knn": K-nearest neighbor lookup in language database
+
+    Args:
+        names: DataFrame containing names or list of name strings.
+        name_column: If names is a DataFrame, the column containing names.
+        top_k: Number of top languages to return (default: 3).
+            Note: KNN method returns only the single best match.
+        model: Prediction method - "lstm" (neural) or "knn" (lookup).
+
+    Returns:
+        DataFrame with name and predicted_languages columns.
+        For LSTM: predicted_languages contains list of top_k languages.
+        For KNN: predicted_languages contains single best language.
+
+    Examples:
+        >>> names = ["sood", "chintalapati"]
+        >>> result = predict_language(names, model="lstm")
+        >>> result["predicted_languages"][0]
+        ['hindi', 'punjabi', 'urdu']
+
+        >>> result_knn = predict_language(names, model="knn")
+        >>> result_knn["predicted_languages"][0]
+        'hindi'
+
+        >>> df = pd.DataFrame({"name": ["patel", "sharma"]})
+        >>> result = predict_language(df, "name", model="lstm", top_k=2)
+        >>> len(result["predicted_languages"][0])
+        2
+    """
+    from ._utils import prepare_name_dataframe
+
+    # Prepare DataFrame
+    df = prepare_name_dataframe(names, name_column)
+    name_col = df.columns[0]
+
+    if model == "lstm":
+        predictions = _predict_language_lstm(df[name_col], top_k)
+    elif model == "knn":
+        predictions = _predict_language_knn(df[name_col])
+    else:
+        raise ValueError(f"Model '{model}' not supported. Use 'lstm' or 'knn'.")
+
+    # Add predictions to DataFrame
+    result = df.copy()
+    result["predicted_languages"] = predictions
+
+    return result
+
+
+def _predict_language_lstm(names: pd.Series, top_k: int = 3) -> list:
+    """
+    Internal function for LSTM language prediction.
+    """
+    from ._utils import clean_name, load_lstm_model
+
+    model, lstm_data = load_lstm_model()
+    char2idx = lstm_data["char2idx"]
+    idx2lang = lstm_data["idx2lang"]
+    device = lstm_data["device"]
+
+    predictions = []
+
+    for name in names:
+        cleaned = clean_name(name)
+        if not cleaned or len(cleaned) < 3:
+            predictions.append([])
+            continue
+
+        # Convert name to indices
+        try:
+            name_indices = [char2idx.get(char, 0) for char in cleaned]
+        except Exception:
+            predictions.append([])
+            continue
+
+        # Prepare tensor
+        with torch.no_grad():
+            name_tensor = (
+                torch.tensor(name_indices, dtype=torch.long).unsqueeze(0).to(device)
+            )
+            lengths = torch.tensor([len(cleaned)], dtype=torch.long)
+
+            # Get predictions for top 3 language outputs
+            out1, out2, out3 = model(name_tensor, lengths)
+
+            # Get top predictions from each output
+            pred_first = torch.argmax(out1, dim=1)
+            pred_second = torch.argmax(out2, dim=1)
+            pred_third = torch.argmax(out3, dim=1)
+
+            # Ensure unique predictions
+            if pred_second == pred_first:
+                pred_second = torch.topk(out2, k=2, dim=1)[1][0][1]
+            if pred_third == pred_first or pred_third == pred_second:
+                pred_third = torch.topk(out3, k=3, dim=1)[1][0][2]
+
+            # Convert to language names
+            langs = [
+                idx2lang[pred_first.item()],
+                idx2lang[pred_second.item()],
+                idx2lang[pred_third.item()],
+            ]
+
+            # Return only top_k languages
+            predictions.append(langs[:top_k])
+
+    return predictions
+
+
+def _predict_language_knn(names: pd.Series) -> list:
+    """
+    Internal function for KNN language lookup.
+    """
+    from ._utils import clean_name, load_language_lookup_data
+
+    lang_data = load_language_lookup_data()
+    lang_cols = lang_data.columns[1:]  # Skip lastname column
+
+    predictions = []
+
+    for name in names:
+        cleaned = clean_name(name)
+        if not cleaned or len(cleaned) < 3:
+            predictions.append("")
+            continue
+
+        # Calculate edit distance to all names in database
+        # Use partial to avoid lambda scope issue
+        distances = lang_data["last_name"].apply(partial(distance, cleaned))
+
+        # Get top 3 nearest names
+        nearest_indices = distances.nsmallest(3).index
+
+        # Sum language scores for nearest names and get max
+        lang_scores = lang_data.loc[nearest_indices, lang_cols].sum()
+        best_lang = lang_scores.idxmax()
+
+        predictions.append(best_lang)
+
+    return predictions

instate-1.0.0.dist-info/METADATA

@@ -0,0 +1,226 @@
+Metadata-Version: 2.3
+Name: instate
+Version: 1.0.0
+Summary: Instate: predict the state of residence from last name
+Keywords: predict,state,residence,last name
+Author: Atul Dhingra, Gaurav Sood, Rajashekar Chintalapati
+Author-email: Atul Dhingra <dhingra.atul92@gmail.com>, Gaurav Sood <gsood07@gmail.com>, Rajashekar Chintalapati <rajshekar.ch@gmail.com>
+License: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Utilities
+Requires-Dist: pandas>=1.5.0
+Requires-Dist: numpy>=1.21.0,<2.0.0
+Requires-Dist: torch>=2.0.0
+Requires-Dist: levenshtein>=0.20.0
+Requires-Dist: requests>=2.25.0
+Requires-Dist: tqdm>=4.64.0
+Requires-Dist: sphinx>=7.0 ; extra == 'docs'
+Requires-Dist: furo>=2024.1.29 ; extra == 'docs'
+Requires-Dist: sphinx-autodoc-typehints>=1.25 ; extra == 'docs'
+Requires-Dist: sphinx-copybutton>=0.5 ; extra == 'docs'
+Requires-Dist: myst-parser>=2.0 ; extra == 'docs'
+Requires-Python: >=3.11, <3.14
+Project-URL: Bug Tracker, https://github.com/appeler/instate/issues
+Project-URL: Homepage, https://github.com/appeler/instate
+Project-URL: Repository, https://github.com/appeler/instate
+Provides-Extra: docs
+Description-Content-Type: text/markdown
+
+## instate: predict spoken language and the state of residence from last name
+
+[![image](https://github.com/appeler/instate/workflows/test/badge.svg)](https://github.com/appeler/instate/actions?query=workflow%3Atest)
+[![image](https://img.shields.io/pypi/v/instate.svg)](https://pypi.python.org/pypi/instate)
+[![Documentation](https://github.com/appeler/instate/actions/workflows/docs.yml/badge.svg)](https://github.com/appeler/instate/actions/workflows/docs.yml)
+[![image](https://static.pepy.tech/badge/instate)](https://pepy.tech/project/instate)
+
+Using the Indian electoral rolls data (2017), we provide a Python
+package that takes the last name of a person and gives its distribution
+across states. This package can also predict the spoken language of the
+person based on the last name.
+
+# Potential Use Cases
+
+India has 22 official languages. To serve such a diverse language base
+is a challenge for businesses and surveyors. To the extent that
+businesses have access to the last name (and no other information) and
+in the absence of other data that allows us to model a person\'s spoken
+language, the distribution of last names across states is the best we
+have.
+
+# Dataset
+
+Refer
+[lastname_langs_india.csv.tar.gz](https://github.com/appeler/instate/blob/main/instate/data/lastname_langs_india.csv.tar.gz)
+for the dataset, that will be used to predict/lookup the spoken language
+based on the last name.
+
+Refer
+[lastname_langs_india_top3.csv.tar.gz](https://github.com/appeler/instate/blob/main/instate/data/lastname_langs_india_top3.csv.tar.gz)
+for the dataset, that will be used to predict the top-3 spoken languages
+based on the last name. A LSTM model has been trained on this dataset to
+predict the top-3 spoken languages.
+
+Refer
+[notebooks](https://github.com/appeler/instate/tree/main/instate/notebooks)
+for the notebooks that were used to prepare above datasets and train the
+models.
+
+# Web UI
+
+Streamlit App.:
+<https://appeler-instate-streamlitstreamlit-app-e39m4c.streamlit.app/>
+
+# Installation
+
+We strongly recommend installing [instate]{.title-ref} inside a Python
+virtual environment (see [venv
+documentation](https://docs.python.org/3/library/venv.html#creating-virtual-environments))
+
+    pip install instate
+
+# Examples
+
+    from instate import last_state
+    last_dat <- pd.read_csv("last_dat.csv")
+    last_state_dat <- last_state(last_dat, "dhingra")
+    print(last_state_dat)
+
+# API
+
+instate provides 4 main functions for predicting state and language from Indian lastnames.
+
+## Electoral Rolls Lookup
+
+- **get_state_distribution** - Get P(state|lastname) from 2017 electoral rolls data
+
+```python
+import instate
+
+# With list of names
+names = ["sharma", "patel", "singh"]
+result = instate.get_state_distribution(names)
+print(result[["name", "Delhi", "Gujarat", "Punjab"]].head())
+
+# With DataFrame
+import pandas as pd
+df = pd.DataFrame({"lastname": ["sharma", "patel"]})
+result = instate.get_state_distribution(df, "lastname")
+print(result.shape)  # (2, 33) - 2 names + 31 state columns
+```
+
+- **get_state_languages** - Map states to their official languages
+
+```python
+# Map states to languages
+states = ["Delhi", "Punjab", "Gujarat"]
+result = instate.get_state_languages(states)
+print(result[["state", "official_languages"]])
+
+#     state official_languages
+# 0   Delhi     Hindi, English  
+# 1  Punjab            Punjabi
+# 2 Gujarat           Gujarati
+```
+
+## Neural Network Predictions
+
+- **predict_state** - Predict likely states using trained GRU model
+
+```python
+# Predict top 3 most likely states
+names = ["sharma", "patel", "singh"]
+result = instate.predict_state(names, top_k=3)
+print(result["predicted_states"].iloc[0])
+# ['Delhi', 'Uttar Pradesh', 'Bihar']
+```
+
+- **predict_language** - Predict likely languages using LSTM or k-nearest neighbor
+
+```python
+# LSTM neural network prediction (top 3)
+result = instate.predict_language(names, model="lstm", top_k=3)
+print(result["predicted_languages"].iloc[0])
+# ['hindi', 'punjabi', 'urdu']
+
+# K-nearest neighbor lookup (single best)
+result = instate.predict_language(names, model="knn")
+print(result["predicted_languages"].iloc[0])
+# 'hindi'
+```
+
+## Complete Example
+
+```python
+import pandas as pd
+import instate
+
+# Sample data
+df = pd.DataFrame({
+    "person_id": [1, 2, 3],
+    "lastname": ["sharma", "patel", "singh"]
+})
+
+# Get state distributions from electoral rolls
+state_dist = instate.get_state_distribution(df, "lastname")
+print("Electoral rolls data shape:", state_dist.shape)
+
+# Predict states with neural network  
+predicted_states = instate.predict_state(df, "lastname", top_k=3)
+print("Top 3 predicted states:", predicted_states["predicted_states"].iloc[0])
+
+# Predict languages
+predicted_langs = instate.predict_language(df, "lastname", model="lstm", top_k=3)
+print("Top 3 predicted languages:", predicted_langs["predicted_languages"].iloc[0])
+
+# Map states to languages
+states_df = pd.DataFrame({"state": ["Delhi", "Gujarat", "Punjab"]})
+lang_map = instate.get_state_languages(states_df)
+print("State language mapping:")
+print(lang_map[["state", "official_languages"]])
+```
+
+# Data
+
+The underlying data for the package can be accessed at:
+<https://doi.org/10.7910/DVN/ZXMVTJ>
+
+# Evaluation
+
+The model has a top-3 accuracy of 85.3% on [unseen
+names](https://github.com/appeler/instate/blob/main/instate/models/model_dnn_gpu.ipynb).
+The KNN model does quite well. See the details
+[here](https://github.com/appeler/instate/blob/main/instate/models/KNN_cosine_distance_simple_avg_modal_state.ipynb)
+The name-to-language lookup has an accuracy of 67.9%. The
+name-to-language model prediction has an accuracy of 72.2%.
+
+# Authors
+
+Atul Dhingra, Gaurav Sood and Rajashekar Chintalapati
+
+# Contributor Code of Conduct
+
+The project welcomes contributions from everyone! In fact, it depends on
+it. To maintain this welcoming atmosphere, and to collaborate in a fun
+and productive way, we expect contributors to the project to abide by
+the [Contributor Code of
+Conduct](http://contributor-covenant.org/version/1/0/0/).
+
+# License
+
+The package is released under the [MIT
+License](https://opensource.org/licenses/MIT).
+
+## 🔗 Adjacent Repositories
+
+- [appeler/ethnicolr2](https://github.com/appeler/ethnicolr2) — Ethnicolr implementation with new models in pytorch
+- [appeler/naampy](https://github.com/appeler/naampy) — Infer Sociodemographic Characteristics from Names Using Indian Electoral Rolls
+- [appeler/ethnicolr](https://github.com/appeler/ethnicolr) — Predict Race and Ethnicity Based on the Sequence of Characters in a Name
+- [appeler/parsernaam](https://github.com/appeler/parsernaam) — AI name parsing. Predict first or last name using a DL model.
+- [appeler/ethnicolor](https://github.com/appeler/ethnicolor) — Race and Ethnicity based on name using data from census, voter reg. files, etc.

instate-1.0.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+instate/__init__.py,sha256=sfIy0tU2asluoXBjWRgQEOOEl8u_iKaLhEs9F_wV3as,924
+instate/_utils.py,sha256=ITEjqTcQIry_xxWEtOvXiGGxDI0Gvv9SqPfASCwfcEM,9692
+instate/constants.py,sha256=v12Bpw64UWqEUiH0ZWVld-dW4PFs_ZXvubkMQn3qzC4,2201
+instate/data/instate_unique_ln_state_prop_v1.csv.gz,sha256=IfmHWpnwdGi_o-rQVL45kVmndyV6Ex1eO8m6hbXJ_XY,11166988
+instate/data/lastname_langs_india.csv.tar.gz,sha256=DkwB0H6fNsD-qLAvxPSrbUSaHJEMX-l2aaYdAkytr-U,7470521
+instate/data/lastname_langs_india_top3.csv.tar.gz,sha256=AQYOTSiI0uKF2qIARDYQRwQnUzceD6uwrTMCq1JzU1s,7470581
+instate/data/state_lang_labels.pt,sha256=l_HDwC4fx0ebQdSheN10_SJrwjaRn76xUUcKmFIrfPE,434872
+instate/data/state_to_languages.csv,sha256=GkXL9qwstNIWOrGplSbZAnHrkD7T5Q9C9xZY095xcbk,2781
+instate/electoral.py,sha256=M2UnJrQtKWSEEMiFjIRVvw1RcNoTheSudOt3NyGzmu8,4634
+instate/nnets.py,sha256=rHvXpqryWtYoinTW3zN2MIn1I7KYcRcjpNVbGYzsXyI,1266
+instate/predict.py,sha256=dXl7CrxvnmXKcqtlI6DIkCir642DS8InjO1bPHo9eNg,7465
+instate-1.0.0.dist-info/WHEEL,sha256=z-mOpxbJHqy3cq6SvUThBZdaLGFZzdZPtgWLcP2NKjQ,79
+instate-1.0.0.dist-info/METADATA,sha256=ighh53kIzWntBIvj73_-5FQrw2oyKZKhUMjIZEIjwIw,8369
+instate-1.0.0.dist-info/RECORD,,

instate-1.0.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.9.15
+Root-Is-Purelib: true
+Tag: py3-none-any