ltc-code 0.2.1__tar.gz → 0.2.11__tar.gz

{ltc_code-0.2.1 → ltc_code-0.2.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ltc-code
-Version: 0.2.1
+Version: 0.2.11
 Summary: Add your description here
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown

{ltc_code-0.2.1 → ltc_code-0.2.11}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.2.1"
+version = "0.2.11"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"

ltc_code-0.2.11/src/ltc_code/20260712_uncommon_scripts/main.py

@@ -0,0 +1,638 @@
+# --- Import Necessary Libraries ---
+import polars as pl
+import sys
+sys.path.append(r"P:\Long_Term_Charter\programs\new_build")
+from paths import *
+from helpers import *
+from mappings import *
+from polars_readstat import scan_readstat
+import glob
+import unicodedata
+import re
+from typing import Union, List, Dict, Optional
+import time
+from datetime import datetime
+import logging
+
+# For parallel processing
+import subprocess
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+# set row/col config
+pl.Config.set_tbl_cols(13)
+pl.Config.set_tbl_rows(15)
+
+# --- Create log file and start timer ---
+logging.basicConfig(
+    filename=LOGS / "main.log",
+    filemode="w", # "w"=overwrite existing log
+    level=logging.INFO,
+    format="%(asctime)s - %(message)s",
+    force=True, #reconfigure logging even if already set up
+)
+
+
+
+###########################################################################################################################################################################
+# --- Define necessary paths and scripts ---
+###########################################################################################################################################################################
+
+BASE = Path("P:/Long_Term_Charter/programs/new_build")
+
+# Paths to code files
+CMO_ID_DIR = BASE / "cmo_ids"
+CMO_APPS_DIR = BASE / "cmo_apps"
+
+# Paths to input data
+OLD_SAMPLE = OLD_BUILD / "old_cmo_apps.parquet"
+KIPP_ENR = CMO_APPS / "kipp_enroll" / "kipp.dta" 
+
+# Paths to output data
+OUTPUT_LONG_DTA = CMO_APPS / "stata" / "cmo_apps.dta"
+OUTPUT_LONG_PARQUET = CMO_APPS / "stata" / "cmo_apps.parquet"
+OUTPUT_WIDE_SID_DTA = CMO_APPS/ "stata_collapsed" / "cmo_apps_sid.dta"
+OUTPUT_WIDE_SID_PARQUET = CMO_APPS / "stata_collapsed" / "cmo_apps_sid.parquet"
+OUTPUT_NSC_DTA = CMO_ANALYSIS / "cmo_analysis.dta"
+OUTPUT_NSC_PARQUET = CMO_ANALYSIS /  "cmo_analysis.parquet"
+OUTPUT_WIDE_NSC_DTA = CMO_ANALYSIS / "cmo_analysis_wide.dta"
+OUTPUT_WIDE_NSC_PARQUET = CMO_ANALYSIS /  "cmo_analysis_wide.parquet"
+OUTPUT_SCHOOLS_CSV = CLEAN / "new_build" / "schools" / "all_schools.csv"
+
+
+
+# Define which apps scripts we want to run
+apps_scripts = [
+    #CMO_APPS_DIR / "aspire.py",
+    #CMO_APPS_DIR / "christel_house.py",
+    #CMO_APPS_DIR / "democracy_prep.py",
+    #CMO_APPS_DIR / "green_dot.py",
+    #CMO_APPS_DIR / "harmony.py",
+    #CMO_APPS_DIR / "ilt.py",
+    #CMO_APPS_DIR / "kipp.py",
+    #CMO_APPS_DIR / "kipp_nj.py",
+    #CMO_APPS_DIR / "kipp_tx.py",
+    #CMO_APPS_DIR / "rocketship.py",
+    CMO_APPS_DIR / "uncommon.py",
+    #CMO_APPS_DIR / "yes_prep.py",
+]
+
+###########################################################################################################################################################################
+# --- Run scripts ---
+###########################################################################################################################################################################
+
+# Runs cmo scripts
+def run_cmo(script):
+    # Start timer
+    start = time.perf_counter()
+    logging.info(f"Starting {script.name}")
+    subprocess.run(
+        [sys.executable, str(script.relative_to(BASE))],
+        cwd=BASE,
+        check=True,
+    )
+    # log elapsed time
+    elapsed_min = (time.perf_counter() - start) / 60
+    logging.info(f"Finished {script.name}: took {elapsed_min:.2f} minutes")
+
+# Appends all the cmos together
+def append_cmos(*cmos):
+    # Scan in old build data
+    old_data = pl.scan_parquet(OLD_SAMPLE)
+    # Read in KIPP enrollment data 
+    enr = (
+        scan_readstat(KIPP_ENR)
+        .select("enrollment_years", "enroll",
+                "sid_cepr_kipp_nj", "sid_cepr_kipp_ma", 
+                "sid_cepr_kipp_nc", "sid_cepr_kipp_tx", )
+        .rename({
+            "enroll": "enroll_kipp",
+        })
+    )
+    dfs = []
+    for cmo in cmos:
+        """
+        if cmo == "harmony":
+            df = (
+                old_data
+                .filter(pl.col("cmo_name") == "HARMONY PUBLIC SCHOOLS")
+                .filter(pl.col("sid_cepr") != "")
+                .filter(pl.col("sample") == 1)
+                .filter(
+                    (pl.col("application_date").is_not_null())
+                    &
+                    (pl.col("application_date") != "")
+                )
+                # Create better late applicant variable specifically for harmony
+                .with_columns(
+                    pl.when(
+                        (
+                            pl.col("application_date")
+                            .str.strptime(pl.Date, "%m/%d/%Y", strict=False)
+                            .dt.year() == pl.col("school_year").cast(pl.Int64, strict=False)
+                        )
+                        &
+                        (
+                            (
+                                pl.col("application_date")
+                                .str.strptime(pl.Date, "%m/%d/%Y", strict=False)
+                                .dt.month() > 2
+                            )
+                            |
+                            (
+                                (
+                                    pl.col("application_date")
+                                    .str.strptime(pl.Date, "%m/%d/%Y", strict=False)
+                                    .dt.month() == 2
+                                )
+                                &
+                                (
+                                    pl.col("application_date")
+                                    .str.strptime(pl.Date, "%m/%d/%Y", strict=False)
+                                    .dt.day() >= 11
+                                )
+                            )
+                        )
+                    )
+                    .then(1)
+                    .otherwise(0)
+                    .alias("late_applicant_clean")
+                )
+                .filter(pl.col("late_applicant_clean") == 0)
+            )
+        
+        elif cmo == "uncommon":
+            df = (
+                old_data
+                .filter(pl.col("cmo_name") == "UNCOMMON SCHOOLS")
+                .filter(pl.col("sid_cepr") != "")
+                .filter(pl.col("sample") == 1)
+                # Drop troublesome cohorts
+                .filter(
+                     ~(
+                         (pl.col("cohort_lottery").is_not_null())
+                         & (pl.col("cohort_lottery").is_in([1990, 1991]))
+                     )
+                )
+                .filter(
+                    (pl.col("late_applicant") == "0")
+                    |
+                    (pl.col("late_applicant") == "")
+                    |
+                    (pl.col("late_applicant").is_null())
+                )
+            )
+        """
+        if cmo == "kipp_tx":
+            enr_filt = (
+                enr
+                .select("sid_cepr_kipp_tx", "enrollment_years", "enroll_kipp")
+                .unique("sid_cepr_kipp_tx")
+                .rename({
+                    "sid_cepr_kipp_tx": "sid_cepr",
+                })
+            )
+            df = (
+                old_data
+                .filter(pl.col("cmo_name") == "KIPP TX")
+                .filter(pl.col("sid_cepr") != "")
+                .filter(pl.col("sample") == 1)
+                # Restrict to the randomized years
+                .filter(pl.col("school_year").cast(pl.Int32, strict=False).is_between(2012, 2016))
+                # Create better filter out late apps
+                .with_columns(
+                    pl.when(
+                        (
+                            pl.col("application_date")
+                            .str.strptime(pl.Date, "%m/%d/%Y", strict=False)
+                            .dt.year() == pl.col("school_year").cast(pl.Int64, strict=False)
+                        )
+                        &
+                        (
+                            pl.col("application_date")
+                            .str.strptime(pl.Date, "%m/%d/%Y", strict=False)
+                            .dt.month() > 1
+                        )
+                    )
+                    .then(1)
+                    .otherwise(0)
+                    .alias("late_applicant_clean")
+                )
+                .filter(pl.col("late_applicant_clean") == 0)
+                # Join on KIPP foundation enrollment
+                .join(enr_filt, how = "left", on = "sid_cepr", validate = "m:1")
+                .rename({
+                    "enroll": "enroll_old"
+                })
+                .with_columns(
+                    pl.when(pl.col("cohort_lottery").cast(pl.Int32) <= 2003)
+                    .then(pl.col("enroll_kipp").fill_null(0))
+                    .alias("enroll_kipp")
+                )
+                .with_columns(
+                    pl.col("enrollment_years").fill_null(0).alias("enrollment_years"),
+
+                    pl.when(pl.col("cohort_lottery").cast(pl.Int32) <= 2003)
+                    .then(pl.col("enroll_kipp"))
+                    .otherwise(pl.col("enroll_old"))
+                    .alias("enroll")
+                )
+            )
+        elif cmo == "ascend":
+            df = (
+                old_data
+                .filter(pl.col("cmo_name") == "ASCEND PUBLIC CHARTER SCHOOLS")
+                .filter(pl.col("sid_cepr") != "")
+                .filter(pl.col("sample") == 1)
+            )
+        elif cmo == "kipp_nc":
+            enr_filt = (
+                enr
+                .select("sid_cepr_kipp_nc", "enrollment_years", "enroll_kipp")
+                .unique("sid_cepr_kipp_nc")
+                .rename({
+                    "sid_cepr_kipp_nc": "sid_cepr",
+                })
+            )
+            df = (
+                old_data
+                .filter(pl.col("cmo_name") == "KIPP NC")
+                .filter(pl.col("sid_cepr") != "")
+                .filter(pl.col("sample") == 1)
+                # Join on KIPP foundation enrollment
+                .join(enr_filt, how = "left", on = "sid_cepr", validate = "m:1")
+                .rename({
+                    "enroll": "enroll_old"
+                })
+                .with_columns(
+                    pl.when(pl.col("cohort_lottery").cast(pl.Int32) <= 2003)
+                    .then(pl.col("enroll_kipp").fill_null(0))
+                    .alias("enroll_kipp")
+                )
+                .with_columns(
+                    pl.col("enrollment_years").fill_null(0).alias("enrollment_years"),
+
+                    pl.when(pl.col("cohort_lottery").cast(pl.Int32) <= 2003)
+                    .then(pl.col("enroll_kipp"))
+                    .otherwise(pl.col("enroll_old"))
+                    .alias("enroll")
+                )
+            )
+        elif cmo == "kipp_ma":
+            enr_filt = (
+                enr
+                .select("sid_cepr_kipp_ma", "enrollment_years", "enroll_kipp")
+                .unique("sid_cepr_kipp_ma")
+                .rename({
+                    "sid_cepr_kipp_ma": "sid_cepr",
+                })
+            )
+            df = (
+                old_data
+                .filter(pl.col("cmo_name") == "KIPP MA")
+                .filter(pl.col("sid_cepr") != "")
+                .filter(pl.col("sample") == 1)
+                .filter(
+                    (pl.col("late_applicant") == "0")
+                    |
+                    (pl.col("late_applicant") == "")
+                    |
+                    (pl.col("late_applicant").is_null())
+                )
+                    # Join on KIPP foundation enrollment
+                .join(enr_filt, how = "left", on = "sid_cepr", validate = "m:1")
+                .rename({
+                    "enroll": "enroll_old"
+                })
+                .with_columns(
+                    pl.when(pl.col("cohort_lottery").cast(pl.Int32) <= 2003)
+                    .then(pl.col("enroll_kipp").fill_null(0))
+                    .alias("enroll_kipp")
+                )
+                .with_columns(
+                    pl.col("enrollment_years").fill_null(0).alias("enrollment_years"),
+
+                    pl.when(pl.col("cohort_lottery").cast(pl.Int32) <= 2003)
+                    .then(pl.col("enroll_kipp"))
+                    .otherwise(pl.col("enroll_old"))
+                    .alias("enroll")
+                )
+            )
+        elif cmo == "great_oaks":
+            df = (
+                old_data
+                .filter(pl.col("cmo_name") == "GREAT OAKS")
+                .filter(pl.col("sid_cepr") != "")
+                .filter(pl.col("sample") == 1)
+            ) 
+        elif cmo == "ilt":
+            df = (
+                old_data
+                .filter(pl.col("cmo_name") == "INTERNATIONAL LEADERSHIP OF TEXAS")
+                .filter(pl.col("sid_cepr") != "")
+                .filter(pl.col("sample") == 1)
+                .filter(
+                    (pl.col("late_applicant") == "0")
+                    |
+                    (pl.col("late_applicant") == "")
+                    |
+                    (pl.col("late_applicant").is_null())
+                )
+            )
+        else:
+            df = (
+                scan_readstat(CMO_APPS / "stata" / "{0}.dta".format(cmo))
+            )
+        dfs.append(df)
+    # Create stata version
+    out_stata = (
+        pl.concat(dfs, how = "diagonal_relaxed")
+        # Do some cleaning
+        .with_columns(
+            pl.col("entry_grade_clean").replace(grade_map).alias("entry_grade_clean"),
+            pl.col("late_app").fill_null(0),
+            pl.col("application_cancel").fill_null(0),
+            # Replace the null risk id names with the risk ids for CMOs we didn't change
+            pl.coalesce(["risk_id_name", "risk_id"]).alias("risk_id_name"),
+            # Fix initial_offer and waitlist_offer
+            pl.coalesce(["initial_offer", "offer_initial"]).alias("initial_offer"),
+            pl.coalesce(["waitlist_offer", "offer_waitlist"]).alias("waitlist_offer"),
+        )
+        .drop("cohort_lottery")
+        .with_columns([
+            pl.col("dob_clean")
+            .str
+            .strip_chars()
+            .str
+            .slice(-4)
+            .cast(pl.String)
+            .alias("cohort_lottery")
+        ])
+        .with_columns( # fix sid cepr (weirdly sometmes a cmo will have a decimanl in the string leading to errors later on) must convert 1010.0 -> 1010
+            pl.col("sid_cepr").cast(pl.Float64, strict=False).cast(pl.Int64).cast(pl.String).str.strip_chars().alias("sid_cepr"),
+        )
+        # Drop non randomized apps
+        .filter(
+            (pl.col("late_app") != 1)
+            &
+            (pl.col("application_cancel") != 1)
+        )
+    ).collect()
+    # Create parquet version
+    out_parquet = (
+        out_stata
+        .filter(
+            (pl.col("late_app") != 1)
+            &
+            (pl.col("application_cancel") != 1)
+        )
+    )
+    # Write datasets
+    out_stata.to_pandas().to_stata(OUTPUT_LONG_DTA, write_index=False, version = 118)
+    out_parquet.write_parquet(OUTPUT_LONG_PARQUET)
+    return out_stata, out_parquet
+
+# Collapse to wide
+def collapse_to_wide(
+    input_parquet_path,
+    group_key,
+    output_parquet_path,
+    output_dta_path,
+):
+    # Read in long parquet data (which already drops late apps)
+    long_input = pl.scan_parquet(input_parquet_path)
+    print(f"Starting N (Long): {long_input.select(pl.len()).collect().item()}")
+    # Define cols we need to take max of and the grouping key
+    max_cols = [
+        "enroll",
+        "enroll_kipp",
+        "enrollment_years",
+        "offer_accepted",
+        "offer",
+        "offer_imputed_enr",
+        "offer_imputed_ln",
+        "offer_imputed_wn",
+        "initial_offer",
+        "waitlist_offer",
+        "sibling",
+        "transfer",
+        "zoned",
+        "staff",
+        "sibling_concur",
+    ]
+    # Within each CMO, group to unique group_key level
+    first_cols = [
+        c for c in long_input.collect_schema().names()
+        if c not in group_key + max_cols + ["risk_id_name"]
+    ]
+    # Create collapsed dataset
+    collapsed = (
+        long_input
+        .group_by(group_key)
+        .agg(
+            # Max variables
+            [pl.max(c).alias(c) for c in max_cols]
+            +
+            # All risk ids concatenated together
+            [
+                pl.col("risk_id_name")
+                .unique()
+                .sort()
+                .str.join("_")
+                .alias("risk_set")
+            ]
+            +
+            # Everything else from first row
+            [
+                pl.first(c).alias(c) for c in first_cols
+            ]
+        )
+    )
+    collapsed_final = collapsed.collect()
+    print(f"Ending N (Wide): {len(collapsed_final)}")
+    # Write datasets
+    collapsed_final.to_pandas().to_stata(output_dta_path, write_index=False, version = 118)
+    collapsed_final.write_parquet(output_parquet_path)
+    return collapsed_final
+
+# Merge on nsc
+def merge_nsc(
+    input_parquet_path,
+    output_parquet_path,
+    output_dta_path,
+):
+    # Read in parquet data (assuming it already drops late apps)
+    input = (
+        pl.scan_parquet(input_parquet_path)
+    )
+    # Read in NSC outcomes
+    nsc = scan_readstat(NSC).with_columns(pl.col("sid_cepr").cast(pl.String))
+    output = (
+        input
+        .with_columns(
+            pl.col("sid_cepr").cast(pl.Float64, strict=False).cast(pl.Int64).cast(pl.String).str.strip_chars().alias("sid_cepr"),
+        )
+        .join(nsc, on="sid_cepr", how="left", validate="m:1")
+        .collect()
+    )
+    # Write datasets
+    output.to_pandas().to_stata(output_dta_path, write_index=False, version = 118)
+    output.write_parquet(output_parquet_path)
+    return output
+    
+
+
+# Creates kiteworks folder corresponding to current date
+def kiteworks():
+    today = datetime.today().strftime("%Y%m%d")
+    # kiteworks folders
+    KITEWORKS_DATE = KITEWORKS_FOLDER / today
+    KITEWORKS_DATE.mkdir(parents=True, exist_ok=True)
+    # define data and codebook output
+    DATA_OUTPUT = KITEWORKS_DATE / "combined_data.csv"
+    CODEBOOK_OUTPUT = KITEWORKS_DATE / "combined_codebook.csv"
+    # Create csv and codebook files set
+    # 7/13: subset
+    csv_files = [
+        CMO_ID_NO_PII / "uncommon.csv",
+    ]
+    codebook_files = [
+        CMO_ID_NO_PII / "uncommon_codebook.csv",
+    ]
+    # csv_files = [
+    #     f 
+    #     for f in sorted(CMO_ID_NO_PII.glob("*.csv"))
+    #     if "codebook" not in f.stem
+    # ]
+    # codebook_files = [
+    #     f 
+    #     for f in sorted(CMO_ID_NO_PII.glob("*.csv"))
+    #     if "codebook" in f.stem
+    # ]
+    # output combined data
+    if csv_files:
+        combined_lf = pl.concat(
+            [pl.scan_csv(f, infer_schema=False) for f in csv_files],
+            how = "diagonal",
+        )
+        combined_lf.collect().write_csv(DATA_OUTPUT)
+    # output combined codebook
+    if codebook_files:
+        combined_codebook = pl.concat(
+            [
+                pl.scan_csv(f, infer_schema=False)
+                for f in codebook_files
+            ],
+            how = "diagonal",
+        )
+        # De-dupe common vars
+        final_codebook = (
+            combined_codebook
+            .group_by("Variable Name")
+            .agg(
+                pl.col("Type").first(),
+                pl.col("Count Non-Null").cast(pl.Int64).sum().alias("Count Non-Null"),
+                pl.col("Description").first(),
+            )
+            .sort("Count Non-Null", descending=True)
+            .drop("Count Non-Null")
+        )
+        # Save it
+        final_codebook.collect().write_csv(CODEBOOK_OUTPUT)
+
+# Collapse to unique schools
+def summarize_schools(
+    input_parquet_path,
+    output_csv_path
+):
+    # Read in long parquet data (which already drops late apps)
+    long_input = pl.scan_parquet(input_parquet_path)
+    # Collapse to unique cmo, region, school_name combos
+    # NOTE: Old build CMOs may not have cmo_region and ren_num consistently filled
+    sch_ren_map = (
+        long_input
+        .filter(
+            pl.col("school_name").is_not_null()
+            & (pl.col("school_name") != "")
+        )
+        .select(["cmo_name", "cmo_region", "ren_num", "school_name"])
+        # unique row-school-ren_num combos
+        .group_by(["cmo_name", "cmo_region", "school_name"])
+        .agg(
+            pl.len().alias("n_obs"),
+            pl.col("ren_num")
+                .drop_nulls()
+                .unique()
+                .sort()
+                .cast(pl.String)
+                .str.join("+")
+                .alias("ren_nums")
+        )
+        .select(["cmo_name", "cmo_region", "school_name", "n_obs", "ren_nums"])
+        .sort(["cmo_name", "cmo_region", "school_name"])
+    )
+    # Save
+    sch_ren_map.collect().write_csv(output_csv_path)
+
+
+if __name__ == "__main__":
+    """
+    
+    # Create kiteworks output for cmo-id transfers
+    kiteworks()
+    
+    # Run cmo apps scripts
+    batch_start = time.perf_counter()
+    logging.info("Starting applications build")
+    with ThreadPoolExecutor(max_workers=2) as pool:
+        futures = [pool.submit(run_cmo, s) for s in apps_scripts]
+        for fut in as_completed(futures):
+            fut.result()
+    batch_elapsed_min_apps = (time.perf_counter() - batch_start) / 60
+    logging.info(f"Finished applications build; total runtime was {batch_elapsed_min_apps: .2f} minutes")
+    """
+    # Append the CMO apps builds together
+    batch_start = time.perf_counter()
+    logging.info("Starting CMO appending")
+    
+    append_cmos(
+        "aspire",
+        "yes_prep",
+        "christel_house",
+        "green_dot",
+        "democracy_prep",
+        "ilt",
+        "rocketship",
+        "kipp_nj",
+        "kipp_tx",
+        "uncommon",
+        "kipp_ma",
+        "great_oaks",
+        "harmony",
+        "ascend",
+        "kipp_nc",
+    )
+    batch_elapsed_min_cmo = (time.perf_counter() - batch_start) / 60
+    logging.info(f"Finished CMO appending; total runtime was {batch_elapsed_min_cmo: .2f} minutes")
+    
+    # Collapse to wide
+    batch_start = time.perf_counter()
+    logging.info("Starting wide collapse")
+
+    
+    collapse_to_wide(OUTPUT_LONG_PARQUET, ["cmo_name", "sid_cepr"], OUTPUT_WIDE_SID_PARQUET, OUTPUT_WIDE_SID_DTA)
+
+    batch_elapsed_min_wide = (time.perf_counter() - batch_start) / 60
+    logging.info(f"Finished wide collapse; total runtime was {batch_elapsed_min_wide: .2f} minutes")
+    
+    # Merge on nsc
+    batch_start = time.perf_counter()
+    logging.info("Starting NSC merge")
+    merge_nsc(OUTPUT_LONG_PARQUET, OUTPUT_NSC_PARQUET, OUTPUT_NSC_DTA)
+    merge_nsc(OUTPUT_WIDE_SID_PARQUET, OUTPUT_WIDE_NSC_PARQUET, OUTPUT_WIDE_NSC_DTA)
+    batch_elapsed_min_nsc = (time.perf_counter() - batch_start) / 60
+    logging.info(f"Finished NSC merge; total runtime was {batch_elapsed_min_nsc: .2f} minutes")
+    
+
+
+
+