ltc-code 0.1.99__tar.gz → 0.2.0__tar.gz

{ltc_code-0.1.99 → ltc_code-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ltc-code
-Version: 0.1.99
+Version: 0.2.0
 Summary: Add your description here
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown

{ltc_code-0.1.99 → ltc_code-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.1.99"
+version = "0.2.0"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"

ltc_code-0.2.0/src/ltc_code/20260630_census_disclosure/aspire.py

@@ -0,0 +1,337 @@
+# --- Import Necessary Libraries ---
+import polars as pl
+import pandas as pd
+import sys
+sys.path.append("/projects/from_research1_arts867_opportunity/ltc/code")
+from paths import *
+from helpers import *
+from mappings import *
+from polars_readstat import scan_readstat
+import glob
+import unicodedata
+import re
+from typing import Union, List, Dict, Optional, Literal, Sequence
+import time
+import subprocess
+import logging
+import shutil 
+
+
+# --- Create log file and start timer ---
+logging.basicConfig(
+    filename=APPS_LOGS / "aspire_apps.log",
+    filemode="w", # "w"=overwrite existing log
+    level=logging.INFO,
+    format="%(asctime)s - %(message)s",
+    force=True, #reconfigure logging even if already set up
+)
+
+# Start timer
+start = time.perf_counter()
+
+
+###########################################################################################################################################################################
+# --- Define necessary paths ---
+###########################################################################################################################################################################
+
+# --- Input + Output ---
+INPUT = CMO_IDS / "harvard_ltc_2006_2021_asp_rsch.sas7bdat"
+OUTPUT_LONG = CMO_APPS / "long" / "aspire.dta"
+OUTPUT_WIDE = CMO_APPS / "wide" / "aspire.dta"
+OUTPUT_SCHOOLS = SCHOOLS / "aspire_schools.csv"
+
+
+
+###########################################################################################################################################################################
+# --- Scan in the cmos data, join on piks, filter to those who are piked, and drop sid_cepr ---
+###########################################################################################################################################################################
+
+# Store full data
+full = (
+    scan_readstat(INPUT)
+    .join(
+        pl.scan_csv(XWALKS / "sid_cepr_pik_xwalk.csv", infer_schema=False),
+        how="left",
+        on=["sid_cepr", "cmo_name"],
+        validate="m:1",
+    )
+    .filter(
+        (pl.col("pik").is_not_null())
+        & (pl.col("pik") != "")
+    )
+    .drop("DQB_SOURCE_ID", "sid_cepr")
+)
+
+
+###########################################################################################################################################################################
+# --- Scan in lottery dataset, code up offers/risksets and deduplicate the applications based on rules ---
+###########################################################################################################################################################################
+
+# Define keys for dedupping function
+DEDUP_KEYS = ["pik", "school_year", "entry_grade_clean", "school_name",]
+
+# Take the max of these for dupes (ORDER MATTERS HERE MAKE SURE MOST IMPORTANT IS LISTED AT TOP)
+EVER_COLS = ["offer_accepted", "offer", "waitlist", "withdraw",]
+
+# Prioritize records with more filled out info for these cols
+INFO_COLS = ["lottery_number", "waitlist_number", "priority_group_name", "status_timestamp",]
+
+
+# Scan in the lottery data and clean stuff
+lot = (
+    full
+    .filter(pl.col("data_type")=="lottery") # filter to lotto
+    .pipe(drop_all_null)
+    .with_columns(
+        pl.lit("CALIFORNIA").alias("cmo_region"),
+        pl.lit(None, dtype=pl.Int64).alias("late_app"),
+    )
+    .with_columns(
+        apply_status_mapping("status", ASPIRE_STATUS_MAP), # map the statuses to relevant offer/waitlist variables
+    )
+    .with_columns( # In this context, missing lottery waitlist numbers are offers
+        pl.when(pl.col("waitlist_number") == "0")
+        .then(1)
+        .otherwise(pl.col("offer"))
+        .alias("offer")
+    )
+    .rename({
+        "priority_name": "priority_group_name",
+    })
+    .pipe(resolve_duplicate_applications, DEDUP_KEYS, EVER_COLS, INFO_COLS)
+    .pipe( # create priority specific flags based on priority group name
+        add_priority_flags,
+        priority_col="priority_group_name",
+        cmo="aspire",
+    )
+    .with_columns( # create number representing the priority group (is not ordinally ranked b/c i dont see reason to do so)
+        pl.col("priority_group_clean")
+        .rank(method="dense")
+        .cast(pl.Int32)
+        .alias("priority_group"),
+    )
+).collect()
+
+
+###########################################################################################################################################################################
+# --- Scan in waitlist, enrollment, and name datasets then join on waitlist/enrollment/name indicators, and years of enrollment ---
+###########################################################################################################################################################################
+
+# Create risk keys
+risk_keys = ["school_year", "entry_grade_clean", "school_name", "priority_group"]
+
+# Scan in the waitlist data and collapse to get wl flag
+wl = (
+    full
+    .filter(pl.col("data_type")=="waitlist") # filter to waitlist
+    .select(DEDUP_KEYS)
+    .unique()
+    .with_columns(
+        pl.lit(1).alias("in_wl"),
+    )
+).collect()
+
+# Scan in the enrollment data and collapse to get enroll flags
+enr_student = (
+    full
+    .filter(pl.col("data_type")=="enroll") # filter to waitlist
+    .select(["pik", "school_year"])
+    .unique()
+    .group_by("pik")
+    .agg(
+        pl.lit(1).alias("enroll"), # flag for if the student ever enrolled in the cmo
+        pl.len().alias("enrollment_years"), # number of years the student was enrolled in the cmo
+        pl.col("school_year").cast(pl.Int32, strict=False).min().alias("year_enrolled"),
+    )
+).collect()
+
+
+# Scan in the enrollment data and collapse to get flag for if enrolled at a given school 
+enr_school = (
+    full
+    .filter(pl.col("data_type")=="enroll") # filter to waitlist
+    .select(["pik", "school_name"])
+    .unique()
+    .with_columns(
+        pl.lit(1).alias("enroll_school"), # flag for if the student ever enrolled at the school to which they applied
+    )
+).collect()
+
+# Scan in the enrollment data and collapse to get flag for if enrolled in a given year
+enr_year = (
+    full
+    .filter(pl.col("data_type")=="enroll") # filter to waitlist
+    .select(["pik", "school_year"])
+    .unique()
+    .with_columns(
+        pl.lit(1).alias("enroll_year"), # flag for if the student enrolled at any schools from the lottery year they applied
+    )
+).collect()
+
+# Scan in the enrollment data and collapse to get flag for if enrolled in a given year
+enr_school_year = (
+    full
+    .filter(pl.col("data_type")=="enroll") # filter to waitlist
+    .select(["pik", "school_year", "school_name"])
+    .unique()
+    .with_columns(
+        pl.lit(1).alias("enroll_school_year"), # flag for if the student enrolled in the school to which they applied in the year of the lottery
+    )
+).collect()
+
+
+# Join on all the useful info from wl/enr, create risk sets/lottery variables, and then keep just the variables in the desired schema
+final = (
+    lot
+    .join(wl, on=DEDUP_KEYS, how="left", validate="m:1")
+    .join(enr_student, on="pik", how="left", validate="m:1")
+    .join(enr_school, on=["pik", "school_name"], how="left", validate="m:1")
+    .join(enr_year, on=["pik", "school_year"], how="left", validate="m:1")
+    .join(enr_school_year, on=["pik", "school_year", "school_name"], how="left", validate="m:1")
+    .with_columns(
+        pl.coalesce(
+            pl.col("in_wl"),
+            pl.col("waitlist"),
+        ).alias("waitlist"),
+
+        pl.col("enroll").fill_null(0).alias("enroll"),
+        pl.col("enrollment_years").fill_null(0).alias("enrollment_years"),
+        pl.col("enroll_school").fill_null(0).alias("enroll_school"),
+        pl.col("enroll_year").fill_null(0).alias("enroll_year"),
+        pl.col("enroll_school_year").fill_null(0).alias("enroll_school_year"),
+    )
+    .with_columns(
+        waitlist_offer=(
+            (pl.col("offer") == 1)
+            & (pl.col("waitlist") == 1)
+        ).cast(pl.Int8),
+
+        initial_offer=(
+            (pl.col("offer") == 1)
+            & (pl.col("waitlist") != 1)
+        ).cast(pl.Int8),
+    )
+    .pipe(keep_final_apps_schema)
+    .with_columns(
+        pl.concat_str(
+            [pl.col(c).cast(pl.String) for c in risk_keys],
+            separator="_",
+        ).alias("risk_id_name")
+    )
+    .with_columns( # create number representing the risk id
+        pl.col("risk_id_name")
+        .rank(method="dense")
+        .cast(pl.Int32)
+        .alias("risk_id"),
+    )
+    .pipe(drop_all_null)
+)
+
+
+
+###########################################################################################################################################################################
+# --- Create collapsed version of the stata dataset  ---
+###########################################################################################################################################################################
+
+
+# Define cols we want to take the max of and the grouping key
+max_cols = [
+    "offer",
+    "initial_offer",
+    "waitlist_offer",
+    "offer_accepted",
+    "enroll",
+    "sibling",
+    "transfer",
+    "zoned",
+    "staff",
+    "sibling_concur",
+]
+
+group_key = ["pik"]
+
+first_cols = [
+    c for c in final.columns
+    if c not in group_key + max_cols + ["risk_id"]
+]
+
+# Create the collapsed dataset
+final_collapsed = (
+    final
+    .filter(pl.col("application_cancel") !=1)
+    .group_by(group_key)
+    .agg(
+        # max variables
+        [pl.max(c).alias(c) for c in max_cols]
+
+        +
+
+        # all risk sets student participcated in
+        [
+            pl.col("risk_id")
+            .unique()
+            .sort()
+            .str.join("_")
+            .alias("risk_set")
+        ]
+
+        + 
+
+        # everything else from first row
+        [
+            pl.first(c).alias(c)
+            for c in first_cols
+        ]
+    )
+)
+
+
+###########################################################################################################################################################################
+# --- Create unique schools dataset  ---
+###########################################################################################################################################################################
+
+
+# Scan in the combined data and collaprse to unique schools
+sch_ren_map = (
+    full
+    .select(["cmo_name", "school_name", "ren_num"])
+    .with_columns(
+        pl.lit("CALIFORNIA").alias("cmo_region"),
+    )
+    .drop_nulls(["school_name"])
+    .group_by(["cmo_name", "cmo_region", "school_name"])
+    .agg(
+        pl.len().alias("n_obs"),
+
+        pl.col("ren_num")
+        .drop_nulls()
+        .unique()
+        .sort()
+        .cast(pl.String)
+        .str.join("+")
+        .alias("ren_nums")
+    )
+    .select(["cmo_name", "cmo_region", "school_name", "n_obs", "ren_nums"])
+    .sort("n_obs", descending=True)
+).collect()
+
+
+###########################################################################################################################################################################
+# --- Save data  ---
+###########################################################################################################################################################################
+
+# Save unique schools dataset
+sch_ren_map.write_csv(OUTPUT_SCHOOLS)
+
+# Save student apps dataset to stata
+final.to_pandas().to_stata(OUTPUT_LONG, write_index=False, version=118)
+
+# Save collapsed student dataset to stata
+final_collapsed.to_pandas().to_stata(OUTPUT_WIDE, write_index=False, version=118)
+
+# End timer
+elapsed = time.perf_counter() - start
+print(f"Finished in {elapsed / 60:.2f} minutes")
+logging.info(f"Finished in {elapsed / 60:.2f} minutes")
+
+