ltc-code 0.2.17__tar.gz → 0.2.18__tar.gz

{ltc_code-0.2.17 → ltc_code-0.2.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ltc-code
-Version: 0.2.17
+Version: 0.2.18
 Summary: Add your description here
 Requires-Dist: fastexcel>=0.16,<0.20
 Requires-Dist: polars>=1.36.1,<1.42

{ltc_code-0.2.17 → ltc_code-0.2.18}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.2.17"
+version = "0.2.18"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"

{ltc_code-0.2.17 → ltc_code-0.2.18}/src/ltc_code/nsc/build_nsc_outcomes.py

@@ -54,13 +54,17 @@ CHETTY_MRC_TABLE11 = next(
 )
 # Chetty OPE-to-super-OPE crosswalk used to attach tier to NSC schools.
 
-
-APPS = INPUT_NSC / "apps.csv"
-# Application/student records; source for unique sid_cepr/cohort_lottery.
+IPEDS_DATA = RAW_NSC / "ipeds_data.dta"
+# Institution-level IPEDS outcomes, including completion within 150% of normal time.
 
 INPUT_NSC = PACKAGE_NSC / "input"
 # Folder for secured Census inputs that are not packaged with the code.
 
+OUTPUT_NSC = PACKAGE_NSC / "output"
+# Folder for final student-level NSC outcomes.
+
+APPS = INPUT_NSC / "apps.csv"
+# Application/student records; source for unique sid_cepr/cohort_lottery.
 
 INPUT_NSC_OLD = INPUT_NSC / "nsc_records_old.dta"
 # Older NSC pull, kept so the build can append old and new NSC versions.
@@ -68,7 +72,7 @@ INPUT_NSC_OLD = INPUT_NSC / "nsc_records_old.dta"
 INPUT_NSC_NEW = INPUT_NSC / "nsc_records_new.csv"
 # Newer NSC pull with studyid-based rows.
 
-OUTCOMES_OUTPUT = OUTPUT_NSC / "nsc_outcomes.parquet"
+OUTCOMES = OUTPUT_NSC / "nsc_outcomes.parquet"
 # Final student-level NSC outcomes file.
 
 
@@ -223,24 +227,32 @@ directory = (
     .select("unitid", "college_name_directory", "years_directory", "ownership_directory")
 )
 
-# Chetty/Barron tiers are keyed at super-OPE level. Table 11 maps school OPE IDs
-# to those super-OPE groups, which is the same route Sarah's Stata code uses.
-chetty_tiers = (
+# Chetty college outcomes are keyed at super-OPE level. Table 11 maps school
+# OPE IDs to those groups, which is the same route Sarah's Stata code uses.
+chetty_college_outcomes = (
     scan_readstat(
         CHETTY_MRC_TABLE2,
         missing_string_as_null=True,
         value_labels_as_strings=False,
     )
-    .select("super_opeid", "tier", "tier_name")
+    .select("super_opeid", "tier", "tier_name", "k_mean")
     .collect()
     .with_columns(
         pl.col("super_opeid").cast(pl.Int64, strict=False),
         pl.col("tier").cast(pl.Int8, strict=False),
+        pl.col("k_mean").cast(pl.Float64, strict=False),
     )
     .drop_nulls("super_opeid")
     .unique("super_opeid", keep="first")
 )
 
+k_mean_neverattend = chetty_college_outcomes.filter(
+    pl.col("super_opeid") == -99
+).item(0, "k_mean")
+k_mean_insuffdata = chetty_college_outcomes.filter(
+    pl.col("super_opeid") == -1
+).item(0, "k_mean")
+
 chetty_ope_crosswalk = (
     scan_readstat(
         CHETTY_MRC_TABLE11,
@@ -257,10 +269,30 @@ chetty_ope_crosswalk = (
 )
 
 chetty_by_opeid = (
-    chetty_ope_crosswalk.join(chetty_tiers, on="super_opeid", how="left")
+    chetty_ope_crosswalk.join(
+        chetty_college_outcomes, on="super_opeid", how="left"
+    )
     .unique("opeid", keep="first")
 )
 
+# IPEDS reports completion within 150% of normal time: six years at four-year
+# colleges and three years at two-year colleges.
+ipeds_completion = (
+    scan_readstat(
+        IPEDS_DATA,
+        missing_string_as_null=True,
+        value_labels_as_strings=False,
+    )
+    .select("unitid", "completion_rate_150pct_ip")
+    .collect()
+    .with_columns(
+        pl.col("unitid").cast(pl.Int64, strict=False),
+        pl.col("completion_rate_150pct_ip").cast(pl.Float64, strict=False),
+    )
+    .drop_nulls("unitid")
+    .unique("unitid", keep="first")
+)
+
 
 ###########################################################
 # Build college reference data
@@ -272,6 +304,7 @@ college_ref = (
     .join(ipeds_manual, on="unitid", how="left")
     .join(directory, on="unitid", how="left")
     .join(chetty_by_opeid, on="opeid", how="left", validate="m:1")
+    .join(ipeds_completion, on="unitid", how="left", validate="m:1")
     .with_columns(
         pl.coalesce(
             [
@@ -318,6 +351,8 @@ college_ref = (
         "college_2yr",
         "tier",
         "college_elite",
+        "k_mean",
+        "completion_rate_150pct_ip",
     )
 )
 
@@ -731,6 +766,8 @@ enroll_weeks = (
         "college_2yr",
         "tier",
         "college_elite",
+        "k_mean",
+        "completion_rate_150pct_ip",
         "term_start_date",
         "term_end_date",
         "week_start",
@@ -788,6 +825,8 @@ enroll = (
         pl.col("college_2yr").max(),
         pl.col("tier").drop_nulls().min(),
         pl.col("college_elite").max(),
+        pl.col("k_mean").drop_nulls().first(),
+        pl.col("completion_rate_150pct_ip").drop_nulls().first(),
         pl.col("_term_att").max(),
         pl.col("week").min().alias("week_start"),
         pl.col("week").max().alias("week_end"),
@@ -892,9 +931,20 @@ first_institution = (
         .first()
         .alias("college_name_firstinst"),
         pl.col("college_years").first().alias("college_years_firstinst"),
+        pl.col("k_mean").first().alias("k_mean_firstinst"),
+        pl.col("completion_rate_150pct_ip")
+        .first()
+        .alias("completion_rate_150pct_firstinst"),
     )
 )
 
+first_institution = first_institution.with_columns(
+    pl.col("k_mean_firstinst")
+    .fill_null(k_mean_insuffdata)
+    .alias("k_mean"),
+    pl.col("completion_rate_150pct_firstinst").alias("predicted_completion"),
+)
+
 enrollment_outcomes = enrollment_outcomes.join(
     first_institution, on="sid_cepr", how="left"
 )
@@ -1291,6 +1341,16 @@ degree_outcomes = degree_outcomes.join(degree_details, on="sid_cepr", how="left"
 nsc_outcomes = (
     apps.join(enrollment_outcomes, on="sid_cepr", how="left", validate="m:1")
     .join(degree_outcomes, on="sid_cepr", how="left", validate="m:1")
+    .with_columns(
+        pl.when(pl.col("ID_FSC_firstinst").is_null())
+        .then(k_mean_neverattend)
+        .otherwise(pl.col("k_mean"))
+        .alias("k_mean"),
+        pl.when(pl.col("ID_FSC_firstinst").is_null())
+        .then(0.0)
+        .otherwise(pl.col("predicted_completion"))
+        .alias("predicted_completion"),
+    )
 )
 
 for year in range(1, N_YEARS_OUT + 1):
@@ -1365,15 +1425,13 @@ outcome_columns = [
     "att_any_byY2", "att_any_byY3", "att_any_byY4",
     "att_4yr_byY2", "att_4yr_byY3", "att_4yr_byY4",
     "att_2yr_byY2", "att_2yr_byY3", "att_2yr_byY4", 
-    "att_elite_byY2", "att_elite_byY3", "att_elite_byY4", "att_elite_byY5", "att_elite_byY6"
+    "att_elite_byY2", "att_elite_byY3", "att_elite_byY4", "att_elite_byY5", "att_elite_byY6",
 
     "cmp_any_byY4", "cmp_any_byY5", "cmp_any_byY6", "cmp_any_byY7", "cmp_any_byY8",
     "cmp_AA_byY4", "cmp_AA_byY5", "cmp_AA_byY6", "cmp_AA_byY7", "cmp_AA_byY8",
     "cmp_BA_byY4", "cmp_BA_byY5", "cmp_BA_byY6", "cmp_BA_byY7", "cmp_BA_byY8",
 
-    "cmp_elite_byY4", "cmp_elite_byY5", "cmp_elite_byY6", "cmp_elite_byY7", "cmp_elite_byY8"
-
-
+    "cmp_elite_byY4", "cmp_elite_byY5", "cmp_elite_byY6", "cmp_elite_byY7", "cmp_elite_byY8",
 ]
 
 nsc_outcomes = nsc_outcomes.with_columns(
@@ -1384,6 +1442,11 @@ nsc_outcomes = nsc_outcomes.with_columns(
 
 keep_columns = [
     "sid_cepr",
+    "k_mean",
+    "predicted_completion",
+    "ID_FSC_firstinst",
+    "college_name_firstinst",
+    "completion_rate_150pct_firstinst",
 ] + outcome_columns
 
 nsc_outcomes_final = nsc_outcomes.select(keep_columns)
@@ -1397,6 +1460,25 @@ nsc_outcomes.with_columns(
 print(f"Wrote {OUTCOMES}")
 print(f"Wrote {OUTCOMES.with_suffix('.csv')}")
 print(f"Rows: {nsc_outcomes.height}, columns: {len(nsc_outcomes.columns)}")
+
+first_college_coverage = (
+    nsc_outcomes.filter(pl.col("ID_FSC_firstinst").is_not_null())
+    .group_by("ID_FSC_firstinst")
+    .agg(
+        pl.col("k_mean_firstinst").is_not_null().any().alias("has_k_mean"),
+        pl.col("completion_rate_150pct_firstinst")
+        .is_not_null()
+        .any()
+        .alias("has_completion_rate"),
+    )
+)
+print(
+    "First-college coverage: "
+    f"{first_college_coverage['has_k_mean'].sum()}/"
+    f"{first_college_coverage.height} with college-specific k_mean; "
+    f"{first_college_coverage['has_completion_rate'].sum()}/"
+    f"{first_college_coverage.height} with IPEDS completion rate"
+)
 print(
     nsc_outcomes.select("sid_cepr", "cohort_lottery", "recovered_nsc_outcome").sort(
         "sid_cepr"