ltc-code 0.1.94__tar.gz → 0.1.96__tar.gz

{ltc_code-0.1.94 → ltc_code-0.1.96}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ltc-code
-Version: 0.1.94
+Version: 0.1.96
 Summary: Add your description here
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown

{ltc_code-0.1.94 → ltc_code-0.1.96}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.1.94"
+version = "0.1.96"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"

ltc_code-0.1.96/src/ltc_code/aspire.py

@@ -0,0 +1,405 @@
+# --- Import Necessary Libraries ---
+import polars as pl
+import pandas as pd
+import sys
+sys.path.append(r"P:\Long_Term_Charter\programs\new_build")
+from paths import *
+from helpers import *
+from mappings import *
+from polars_readstat import scan_readstat
+import glob
+import unicodedata
+import re
+from typing import Union, List, Dict, Optional, Literal, Sequence
+import time
+import subprocess
+import logging
+import nameparser 
+import shutil 
+from nameparser import HumanName # Popular package for parsing out the subcomponents of a full name
+
+pl.Config.set_tbl_cols(13)
+pl.Config.set_tbl_rows(15)
+
+
+# --- Create log file and start timer ---
+logging.basicConfig(
+    filename=LOGS / "aspire_apps.log",
+    filemode="w", # "w"=overwrite existing log
+    level=logging.INFO,
+    format="%(asctime)s - %(message)s",
+    force=True, #reconfigure logging even if already set up
+)
+
+# Start timer
+start = time.perf_counter()
+
+
+###########################################################################################################################################################################
+# --- Define necessary paths ---
+###########################################################################################################################################################################
+
+# --- Input + Temp + Output ---
+INPUT = CMO_ID / "NO_PII" / "aspire.csv"
+TEMP_CSV = CMO_APPS / "csv" / "temp" / "aspire_temp.csv"
+OUTPUT_CSV = CMO_APPS / "csv" / "aspire.csv"
+OUTPUT_STATA = CMO_APPS / "stata" / "aspire.dta"
+OUTPUT_STATA_COLLAPSED = CMO_APPS / "stata_collapsed" / "aspire.dta"
+OUTPUT_SCHOOLS = CLEAN / "new_build" / "schools" / "aspire_schools.csv"
+
+
+
+###########################################################################################################################################################################
+# --- Scan in lottery dataset, code up offers/risksets and deduplicate the applications based on rules ---
+###########################################################################################################################################################################
+
+# Define keys for dedupping function
+DEDUP_KEYS = ["sid_cepr", "school_year", "entry_grade_clean", "school_name",]
+
+# Take the max of these for dupes (ORDER MATTERS HERE MAKE SURE MOST IMPORTANT IS LISTED AT TOP)
+EVER_COLS = ["offer_accepted", "offer", "waitlist", "withdraw",]
+
+# Prioritize records with more filled out info for these cols
+INFO_COLS = ["lottery_number", "waitlist_number", "priority_group_name", "status_timestamp",]
+
+
+# Scan in the lottery data and clean stuff
+lot = (
+    pl.scan_csv(INPUT, infer_schema=False, null_values=[], try_parse_dates=False, ignore_errors=False,)
+    .filter(pl.col("data_type")=="lottery") # filter to lotto
+    .pipe(drop_all_null)
+    .with_columns(
+        pl.lit("CALIFORNIA").alias("cmo_region"),
+        pl.lit(None, dtype=pl.Int64).alias("late_app"),
+    )
+    .with_columns(
+        apply_status_mapping("status", ASPIRE_STATUS_MAP), # map the statuses to relevant offer/waitlist variables
+    )
+    .with_columns( # code wl numb==0 as offer in this context)
+        pl.when(pl.col("waitlist_number") == "0")
+        .then(1)
+        .otherwise(pl.col("offer"))
+        .alias("offer")
+    )
+    .rename({
+        "priority_name": "priority_group_name",
+    })
+    .pipe(resolve_duplicate_applications, DEDUP_KEYS, EVER_COLS, INFO_COLS)
+    .pipe( # create priority specific flags based on priority group name
+        add_priority_flags,
+        priority_col="priority_group_name",
+        cmo="aspire",
+    )
+    .with_columns( # create number representing the priority group (is not ordinally ranked b/c i dont see reason to do so)
+        pl.col("priority_group_clean")
+        .rank(method="dense")
+        .cast(pl.Int32)
+        .alias("priority_group"),
+    )
+).collect()
+
+
+###########################################################################################################################################################################
+# --- Scan in waitlist, enrollment, and name datasets then join on waitlist/enrollment/name indicators, and years of enrollment ---
+###########################################################################################################################################################################
+
+# Create risk keys
+risk_keys = ["school_year", "entry_grade_clean", "school_name", "priority_group"]
+
+# Scan in the waitlist data and collapse to get wl flag
+wl = (
+    pl.scan_csv(INPUT, infer_schema=False, null_values=[], try_parse_dates=False, ignore_errors=False,)
+    .filter(pl.col("data_type")=="waitlist") # filter to waitlist
+    .select(DEDUP_KEYS)
+    .unique()
+    .with_columns(
+        pl.lit(1).alias("in_wl"),
+    )
+).collect()
+
+# Scan in the enrollment data and collapse to get enroll flags
+enr_student = (
+    pl.scan_csv(INPUT, infer_schema=False, null_values=[], try_parse_dates=False, ignore_errors=False,)
+    .filter(pl.col("data_type")=="enroll") # filter to waitlist
+    .select(["sid_cepr", "school_year"])
+    .unique()
+    .group_by("sid_cepr")
+    .agg(
+        pl.lit(1).alias("enroll"), # flag for if the student ever enrolled in the cmo
+        pl.len().alias("enrollment_years"), # number of years the student was enrolled in the cmo
+        pl.col("school_year").cast(pl.Int32, strict=False).min().alias("year_enrolled"),
+    )
+).collect()
+
+
+# Scan in the enrollment data and collapse to get flag for if enrolled at a given school 
+enr_school = (
+    pl.scan_csv(INPUT, infer_schema=False, null_values=[], try_parse_dates=False, ignore_errors=False,)
+    .filter(pl.col("data_type")=="enroll") # filter to waitlist
+    .select(["sid_cepr", "school_name"])
+    .unique()
+    .with_columns(
+        pl.lit(1).alias("enroll_school"), # flag for if the student ever enrolled at the school to which they applied
+    )
+).collect()
+
+# Scan in the enrollment data and collapse to get flag for if enrolled in a given year
+enr_year = (
+    pl.scan_csv(INPUT, infer_schema=False, null_values=[], try_parse_dates=False, ignore_errors=False,)
+    .filter(pl.col("data_type")=="enroll") # filter to waitlist
+    .select(["sid_cepr", "school_year"])
+    .unique()
+    .with_columns(
+        pl.lit(1).alias("enroll_year"), # flag for if the student enrolled at any schools from the lottery year they applied
+    )
+).collect()
+
+# Scan in the enrollment data and collapse to get flag for if enrolled in a given year
+enr_school_year = (
+    pl.scan_csv(INPUT, infer_schema=False, null_values=[], try_parse_dates=False, ignore_errors=False,)
+    .filter(pl.col("data_type")=="enroll") # filter to waitlist
+    .select(["sid_cepr", "school_year", "school_name"])
+    .unique()
+    .with_columns(
+        pl.lit(1).alias("enroll_school_year"), # flag for if the student enrolled in the school to which they applied in the year of the lottery
+    )
+).collect()
+
+# Scan in the lname data and filter to aspire
+lname = (
+    pl.scan_csv(XWALKS / "lname_clean_id_xwalk.csv", infer_schema=False, null_values=[], try_parse_dates=False, ignore_errors=False,)
+    .filter(pl.col("cmo_name")=="Aspire") 
+).collect()
+
+# Join on all the useful info from wl/enr, create risk sets/lottery variables, join on last names, and then keep just the variables in the desired schema
+final = (
+    lot
+    .join(wl, on=DEDUP_KEYS, how="left", validate="m:1")
+    .join(enr_student, on="sid_cepr", how="left", validate="m:1")
+    .join(enr_school, on=["sid_cepr", "school_name"], how="left", validate="m:1")
+    .join(enr_year, on=["sid_cepr", "school_year"], how="left", validate="m:1")
+    .join(enr_school_year, on=["sid_cepr", "school_year", "school_name"], how="left", validate="m:1")
+    .with_columns(
+        pl.coalesce(
+            pl.col("in_wl"),
+            pl.col("waitlist"),
+        ).alias("waitlist"),
+
+        pl.col("enroll").fill_null(0).alias("enroll"),
+        pl.col("enrollment_years").fill_null(0).alias("enrollment_years"),
+        pl.col("enroll_school").fill_null(0).alias("enroll_school"),
+        pl.col("enroll_year").fill_null(0).alias("enroll_year"),
+        pl.col("enroll_school_year").fill_null(0).alias("enroll_school_year"),
+    )
+    .with_columns(
+        waitlist_offer=(
+            (pl.col("offer") == 1)
+            & (pl.col("waitlist") == 1)
+        ).cast(pl.Int8),
+
+        initial_offer=(
+            (pl.col("offer") == 1)
+            & (pl.col("waitlist") != 1)
+        ).cast(pl.Int8),
+    )
+    .pipe(keep_final_apps_schema)
+    .join(lname, on=["lname_clean_id", "cmo_name"], how="left", validate="m:1")
+    .with_columns(
+        pl.concat_str(
+            [pl.col(c).cast(pl.String) for c in risk_keys],
+            separator="_",
+        ).alias("risk_id_name")
+    )
+    .with_columns( # create number representing the risk id
+        pl.col("risk_id_name")
+        .rank(method="dense")
+        .cast(pl.Int32)
+        .alias("risk_id"),
+    )
+    .pipe(drop_all_null)
+)
+
+
+
+###########################################################################################################################################################################
+# --- Run the R scripts to get race joined on  ---
+###########################################################################################################################################################################
+
+# Save to csv
+final.write_csv(TEMP_CSV)
+
+# Run the R script to get race (NOTE: this can not be run interactively and you must run from terminal with py -m main)
+subprocess.run(
+    ["Rscript", "impute_race.R", str(TEMP_CSV), str(OUTPUT_CSV)], check=True,
+)
+
+
+
+###########################################################################################################################################################################
+# --- Create collapsed version of the stata dataset  ---
+###########################################################################################################################################################################
+
+# Read the data back in 
+final = (
+    pl.read_csv(OUTPUT_CSV)    
+)
+
+
+# Define cols we want to take the max of and the grouping key
+max_cols = [
+    "offer",
+    "initial_offer",
+    "waitlist_offer",
+    "offer_accepted",
+    "enroll",
+    "sibling",
+    "transfer",
+    "zoned",
+    "staff",
+    "sibling_concur",
+]
+
+group_key = ["sid_cepr"]
+
+first_cols = [
+    c for c in final.columns
+    if c not in group_key + max_cols + ["risk_id"]
+]
+
+# Create the collapsed dataset
+final_collapsed = (
+    final
+    .filter(pl.col("application_cancel") !=1)
+    .group_by(group_key)
+    .agg(
+        # max variables
+        [pl.max(c).alias(c) for c in max_cols]
+
+        +
+
+        # all risk sets student participcated in
+        [
+            pl.col("risk_id")
+            .unique()
+            .sort()
+            .str.join("_")
+            .alias("risk_set")
+        ]
+
+        + 
+
+        # everything else from first row
+        [
+            pl.first(c).alias(c)
+            for c in first_cols
+        ]
+    )
+)
+
+
+###########################################################################################################################################################################
+# --- Create unique schools dataset  ---
+###########################################################################################################################################################################
+
+
+# Scan in the combined data and collaprse to unique schools
+sch_ren_map = (
+    pl.scan_csv(INPUT, infer_schema=False, null_values=[], try_parse_dates=False, ignore_errors=False,)
+    .select(["cmo_name", "school_name", "ren_num"])
+    .with_columns(
+        pl.lit("CALIFORNIA").alias("cmo_region"),
+    )
+    .drop_nulls(["school_name"])
+    .group_by(["cmo_name", "cmo_region", "school_name"])
+    .agg(
+        pl.len().alias("n_obs"),
+
+        pl.col("ren_num")
+        .drop_nulls()
+        .unique()
+        .sort()
+        .cast(pl.String)
+        .str.join("+")
+        .alias("ren_nums")
+    )
+    .select(["cmo_name", "cmo_region", "school_name", "n_obs"])
+    .sort("n_obs", descending=True)
+).collect()
+
+
+###########################################################################################################################################################################
+# --- Save data  ---
+###########################################################################################################################################################################
+
+# Save unique schools dataset
+sch_ren_map.write_csv(OUTPUT_SCHOOLS)
+
+# Save student apps dataset to stata
+final.to_pandas().to_stata(OUTPUT_STATA, write_index=False, version=118)
+
+# Save collapsed student dataset to stata
+final_collapsed.to_pandas().to_stata(OUTPUT_STATA_COLLAPSED, write_index=False, version=118)
+
+# End timer
+elapsed = time.perf_counter() - start
+print(f"Finished in {elapsed / 60:.2f} minutes")
+logging.info(f"Finished in {elapsed / 60:.2f} minutes")
+
+
+
+###########################################################################################################################################################################
+# --- LOG COUNTS ---
+###########################################################################################################################################################################
+
+"""
+logging.info(
+    "Lottery observations: %s",
+    final_no_pii
+    .filter(pl.col("data_type") == "lottery")
+    .height
+)
+logging.info(
+    "Waitlist observations: %s",
+    final_no_pii
+    .filter(pl.col("data_type") == "waitlist")
+    .height
+)
+logging.info(
+    "Enrollment observations: %s",
+    final_no_pii
+    .filter(pl.col("data_type") == "enroll")
+    .height
+)
+logging.info(
+    "Counts by school year:"
+)
+logging.info(
+    "\n%s",
+    final_no_pii
+    .group_by("school_year")
+    .len()
+    .sort("school_year")
+)
+logging.info(
+    "Counts by birth cohort:"
+)
+with pl.Config(
+    tbl_rows=-1,
+    tbl_cols=-1,
+    fmt_str_lengths=1000,
+):
+    logging.info(
+        "\n%s",
+        final_no_pii
+        .group_by("cohort_lottery")
+        .len()
+        .sort("cohort_lottery")    
+    )
+    logging.info(
+        "Schema: \n%s",
+        "\n".join(
+            f"{col}: {dtype}"
+            for col, dtype in final_no_pii.schema.items()
+        )
+    )
+"""