ltc-code 0.2.16__tar.gz → 0.2.18__tar.gz

{ltc_code-0.2.16 → ltc_code-0.2.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ltc-code
-Version: 0.2.16
+Version: 0.2.18
 Summary: Add your description here
 Requires-Dist: fastexcel>=0.16,<0.20
 Requires-Dist: polars>=1.36.1,<1.42

{ltc_code-0.2.16 → ltc_code-0.2.18}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.2.16"
+version = "0.2.18"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"

{ltc_code-0.2.16 → ltc_code-0.2.18}/src/ltc_code/nsc/build_nsc_outcomes.py

@@ -54,24 +54,24 @@ CHETTY_MRC_TABLE11 = next(
 )
 # Chetty OPE-to-super-OPE crosswalk used to attach tier to NSC schools.
 
+IPEDS_DATA = RAW_NSC / "ipeds_data.dta"
+# Institution-level IPEDS outcomes, including completion within 150% of normal time.
+
 INPUT_NSC = PACKAGE_NSC / "input"
 # Folder for secured Census inputs that are not packaged with the code.
 
+OUTPUT_NSC = PACKAGE_NSC / "output"
+# Folder for final student-level NSC outcomes.
+
 APPS = INPUT_NSC / "apps.csv"
 # Application/student records; source for unique sid_cepr/cohort_lottery.
 
-NSC_RECORDS = INPUT_NSC / "nsc_records.csv"
-# Long NSC enrollment and degree records.
-
 INPUT_NSC_OLD = INPUT_NSC / "nsc_records_old.dta"
 # Older NSC pull, kept so the build can append old and new NSC versions.
 
-INPUT_NSC_NEW = INPUT_NSC / "nsc_records_new.dta"
+INPUT_NSC_NEW = INPUT_NSC / "nsc_records_new.csv"
 # Newer NSC pull with studyid-based rows.
 
-OUTPUT_NSC = PACKAGE_NSC / "output"
-# Package-local output folder for NSC outcome files.
-
 OUTCOMES = OUTPUT_NSC / "nsc_outcomes.parquet"
 # Final student-level NSC outcomes file.
 
@@ -99,17 +99,13 @@ college_crosswalk = (
         columns=["ID_IPEDS", "ID_OPE", "ID_FSC", "ID_FSC_long", "name"],
     )
     .with_columns(
-        pl.when(
-            pl.col("ID_FSC_long").is_not_null()
-            & (pl.col("ID_FSC_long").cast(pl.String).str.strip_chars() != "")
-        )
-        .then(pl.col("ID_FSC_long").cast(pl.String).str.strip_chars())
-        .when(pl.col("ID_FSC").is_not_null())
-        .then(
-            pl.col("ID_FSC").cast(pl.String).str.strip_chars().str.replace(r"\.0$", "")
-            + pl.lit("-00")
+        pl.coalesce(
+            [
+                pl.col("ID_FSC_long").cast(pl.String).str.strip_chars(),
+                pl.col("ID_FSC").cast(pl.String).str.strip_chars() + pl.lit("-00")
+            ]
         )
-        .otherwise(None)
+        .str.replace(r"\.0$", "")
         .alias("ID_FSC"),
         pl.col("ID_IPEDS").cast(pl.Int64, strict=False).alias("unitid"),
         pl.col("ID_OPE").cast(pl.Int64, strict=False),
@@ -231,24 +227,32 @@ directory = (
     .select("unitid", "college_name_directory", "years_directory", "ownership_directory")
 )
 
-# Chetty/Barron tiers are keyed at super-OPE level. Table 11 maps school OPE IDs
-# to those super-OPE groups, which is the same route Sarah's Stata code uses.
-chetty_tiers = (
+# Chetty college outcomes are keyed at super-OPE level. Table 11 maps school
+# OPE IDs to those groups, which is the same route Sarah's Stata code uses.
+chetty_college_outcomes = (
     scan_readstat(
         CHETTY_MRC_TABLE2,
         missing_string_as_null=True,
         value_labels_as_strings=False,
     )
-    .select("super_opeid", "tier", "tier_name")
+    .select("super_opeid", "tier", "tier_name", "k_mean")
     .collect()
     .with_columns(
         pl.col("super_opeid").cast(pl.Int64, strict=False),
         pl.col("tier").cast(pl.Int8, strict=False),
+        pl.col("k_mean").cast(pl.Float64, strict=False),
     )
     .drop_nulls("super_opeid")
     .unique("super_opeid", keep="first")
 )
 
+k_mean_neverattend = chetty_college_outcomes.filter(
+    pl.col("super_opeid") == -99
+).item(0, "k_mean")
+k_mean_insuffdata = chetty_college_outcomes.filter(
+    pl.col("super_opeid") == -1
+).item(0, "k_mean")
+
 chetty_ope_crosswalk = (
     scan_readstat(
         CHETTY_MRC_TABLE11,
@@ -265,10 +269,30 @@ chetty_ope_crosswalk = (
 )
 
 chetty_by_opeid = (
-    chetty_ope_crosswalk.join(chetty_tiers, on="super_opeid", how="left")
+    chetty_ope_crosswalk.join(
+        chetty_college_outcomes, on="super_opeid", how="left"
+    )
     .unique("opeid", keep="first")
 )
 
+# IPEDS reports completion within 150% of normal time: six years at four-year
+# colleges and three years at two-year colleges.
+ipeds_completion = (
+    scan_readstat(
+        IPEDS_DATA,
+        missing_string_as_null=True,
+        value_labels_as_strings=False,
+    )
+    .select("unitid", "completion_rate_150pct_ip")
+    .collect()
+    .with_columns(
+        pl.col("unitid").cast(pl.Int64, strict=False),
+        pl.col("completion_rate_150pct_ip").cast(pl.Float64, strict=False),
+    )
+    .drop_nulls("unitid")
+    .unique("unitid", keep="first")
+)
+
 
 ###########################################################
 # Build college reference data
@@ -280,6 +304,7 @@ college_ref = (
     .join(ipeds_manual, on="unitid", how="left")
     .join(directory, on="unitid", how="left")
     .join(chetty_by_opeid, on="opeid", how="left", validate="m:1")
+    .join(ipeds_completion, on="unitid", how="left", validate="m:1")
     .with_columns(
         pl.coalesce(
             [
@@ -318,8 +343,6 @@ college_ref = (
         "ID_FSC",
         "unitid",
         "ID_OPE",
-        "opeid",
-        "super_opeid",
         "college_name",
         "college_years",
         "college_ownership",
@@ -327,9 +350,9 @@ college_ref = (
         "college_4yr",
         "college_2yr",
         "tier",
-        "tier_name",
-        "superopeid_name",
         "college_elite",
+        "k_mean",
+        "completion_rate_150pct_ip",
     )
 )
 
@@ -340,22 +363,25 @@ college_ref = (
 
 # Start from unique students. cohort_lottery is the birth cohort; cohort_18 is
 # the year the student turns 18 and anchors Sarah's Y1/Y2/... outcome windows.
-if str(APPS).endswith(".dta"):
-    apps_raw = scan_readstat(
-        APPS, missing_string_as_null=True, value_labels_as_strings=True
-    ).collect()
-else:
-    apps_raw = pl.read_csv(APPS)
+
 
 apps = (
-    apps_raw.select("sid_cepr", "cohort_lottery")
+    scan_readstat(
+        APPS, missing_string_as_null=True, value_labels_as_strings=True
+    )
+    .select("sid_cepr", "cohort_lottery")
     .drop_nulls(["sid_cepr", "cohort_lottery"])
     .with_columns(
         pl.col("sid_cepr").cast(pl.Int64),
         pl.col("cohort_lottery").cast(pl.Int64),
     )
-    .unique(["sid_cepr", "cohort_lottery"])
+    .filter(
+        pl.col("cohort_lottery").is_between(1990, 2005)
+    )
+    .sort("cohort_lottery")
+    .unique(["sid_cepr"])
     .with_columns((pl.col("cohort_lottery") + 18).alias("cohort_18"))
+    .collect()
 )
 
 # These branch-code fixes mirror the Sarah code path that maps known bad FSC
@@ -441,7 +467,11 @@ fsc_branch_fixes = [
 # The real NSC file uses raw NSC names; Sarah renames these before building
 # outcomes. This block makes those same names explicit in Polars.
 nsc_old = (
-    scan_readstat(INPUT_NSC_OLD, missing_string_as_null=True, value_labels_as_strings=True)
+    scan_readstat(INPUT_NSC_OLD, missing_string_as_null=True, value_labels_as_strings=True, schema_overrides={
+            "searchdate": pl.String
+        }
+    )
+    .collect()
     .with_columns(
         pl.lit(2023).alias("NSCdatayear"),
 
@@ -450,11 +480,41 @@ nsc_old = (
         .cast(pl.Int32)
         .alias("searchbeginyear")
     )
-    .collect()
 )
 
 nsc_new = (
-    scan_readstat(INPUT_NSC_NEW, missing_string_as_null=True, value_labels_as_strings=True)
+    pl.scan_csv(INPUT_NSC_NEW, infer_schema=False, null_values=[""], try_parse_dates=False, ignore_errors=False,)
+    .rename({
+        "Study ID": "studyid",
+        "Record Found Y/N": "recordfoundyn",
+        "College Code/Branch": "collegecodebranch",
+        "College Name": "collegename",
+        "College State": "collegestate",
+        "2-year / 4-year": "2year4year",
+        "Public / Private": "publicprivate",
+        "Enrollment Begin": "enrollmentbegin",
+        "Enrollment End": "enrollmentend",
+        "Enrollment Status": "enrollmentstatus",
+        "Class Level": "classlevel",
+        "Enrollment Major 1": "enrollmentmajor1",
+        "Enrollment CIP 1": "enrollmentcip1",
+        "Enrollment Major 2": "enrollmentmajor2",
+        "Enrollment CIP 2": "enrollmentcip2",
+        "Graduated?": "graduated",
+        "Graduation Date": "graduationdate",
+        "Degree Title": "degreetitle",
+        "Degree Major 1": "degreemajor1",
+        "Degree CIP 1": "degreecip1",
+        "Degree Major 2": "degreemajor2",
+        "Degree CIP 2": "degreecip2",
+        "Degree Major 3": "degreemajor3",
+        "Degree CIP 3": "degreecip3",
+        "Degree Major 4": "degreemajor4",
+        "Degree CIP 4": "degreecip4",
+        "College Sequence": "collegesequence",
+        "Student Age at start of term": "studentageatstartofterm",
+        
+    })
     .with_row_index("origsrt", offset=1)
     .filter(pl.col("studyid").is_not_null())
     .with_columns(
@@ -463,7 +523,7 @@ nsc_new = (
     .explode("sid_cepr")
     .filter(pl.col("sid_cepr") != "")
     .with_columns(
-        pl.col("sid_cepr").cast(pl.Int64, strict=False)
+        pl.col("sid_cepr").cast(pl.String).str.replace_all("_", "").cast(pl.Int64, strict=False)
     )
     .filter(
         pl.col("sid_cepr").is_not_null()
@@ -706,6 +766,8 @@ enroll_weeks = (
         "college_2yr",
         "tier",
         "college_elite",
+        "k_mean",
+        "completion_rate_150pct_ip",
         "term_start_date",
         "term_end_date",
         "week_start",
@@ -763,6 +825,8 @@ enroll = (
         pl.col("college_2yr").max(),
         pl.col("tier").drop_nulls().min(),
         pl.col("college_elite").max(),
+        pl.col("k_mean").drop_nulls().first(),
+        pl.col("completion_rate_150pct_ip").drop_nulls().first(),
         pl.col("_term_att").max(),
         pl.col("week").min().alias("week_start"),
         pl.col("week").max().alias("week_end"),
@@ -867,9 +931,20 @@ first_institution = (
         .first()
         .alias("college_name_firstinst"),
         pl.col("college_years").first().alias("college_years_firstinst"),
+        pl.col("k_mean").first().alias("k_mean_firstinst"),
+        pl.col("completion_rate_150pct_ip")
+        .first()
+        .alias("completion_rate_150pct_firstinst"),
     )
 )
 
+first_institution = first_institution.with_columns(
+    pl.col("k_mean_firstinst")
+    .fill_null(k_mean_insuffdata)
+    .alias("k_mean"),
+    pl.col("completion_rate_150pct_firstinst").alias("predicted_completion"),
+)
+
 enrollment_outcomes = enrollment_outcomes.join(
     first_institution, on="sid_cepr", how="left"
 )
@@ -920,6 +995,7 @@ degree_title = pl.col("_degree_title_clean").fill_null("")
 degrees = (
     nsc.filter(pl.col("graduated") == "Y")
     .filter(pl.col("graduated_date").is_not_null())
+    .filter(pl.col("graduated_date") <= NSC_CUTOFF_DATE)
     .with_columns(
         pl.col("degree_title")
         .cast(pl.String)
@@ -1198,6 +1274,13 @@ for year in range(1, N_YEARS_OUT + 1):
             (pl.col("_by_end") * pl.col("_degree").is_in(["AA", "BA"]).cast(pl.Int8))
             .max()
             .alias(f"cmp_any_byY{year}"),
+            (
+                pl.col("_by_end")
+                * pl.col("_degree").is_in(["AA", "BA"]).cast(pl.Int8)
+                * pl.col("college_elite").fill_null(0).cast(pl.Int8)
+            )
+            .max()
+            .alias(f"cmp_elite_byY{year}"),
         )
     )
 
@@ -1256,20 +1339,24 @@ degree_outcomes = degree_outcomes.join(degree_details, on="sid_cepr", how="left"
 
 # Merge enrollment and degree outcomes back to the unique application universe.
 nsc_outcomes = (
-    apps.join(enrollment_outcomes, on="sid_cepr", how="left")
-    .join(degree_outcomes, on="sid_cepr", how="left")
+    apps.join(enrollment_outcomes, on="sid_cepr", how="left", validate="m:1")
+    .join(degree_outcomes, on="sid_cepr", how="left", validate="m:1")
+    .with_columns(
+        pl.when(pl.col("ID_FSC_firstinst").is_null())
+        .then(k_mean_neverattend)
+        .otherwise(pl.col("k_mean"))
+        .alias("k_mean"),
+        pl.when(pl.col("ID_FSC_firstinst").is_null())
+        .then(0.0)
+        .otherwise(pl.col("predicted_completion"))
+        .alias("predicted_completion"),
+    )
 )
 
 for year in range(1, N_YEARS_OUT + 1):
-    nsc_outcomes = nsc_outcomes.with_columns(
-        (pl.col("cohort_18") + year - 1).alias("_in_start_year"),
-        pl.col("cohort_18").alias("_by_start_year"),
-        (pl.col("cohort_18") + year).alias("_by_end_year"),
-    ).with_columns(
-        pl.date(pl.col("_in_start_year"), 7, 1).alias("_in_start"),
-        pl.date(pl.col("_by_start_year"), 7, 1).alias("_by_start"),
-        pl.date(pl.col("_by_end_year"), 6, 30).alias("_by_end"),
-    )
+    in_start = pl.date(pl.col("cohort_18") + year - 1, 7, 1)
+    by_start = pl.date(pl.col("cohort_18"), 7, 1)
+    by_end = pl.date(pl.col("cohort_18") + year, 6, 30)
 
     att_in_columns = [
         column
@@ -1291,7 +1378,7 @@ for year in range(1, N_YEARS_OUT + 1):
 
     nsc_outcomes = nsc_outcomes.with_columns(
         [
-            pl.when(pl.col("_in_start") > NSC_CUTOFF_DATE)
+            pl.when(in_start > NSC_CUTOFF_DATE)
             .then(None)
             .when(pl.col(column).is_null())
             .then(0)
@@ -1302,7 +1389,7 @@ for year in range(1, N_YEARS_OUT + 1):
         ]
         +
         [
-            pl.when(pl.col("_by_start") > NSC_CUTOFF_DATE)
+            pl.when(by_start > NSC_CUTOFF_DATE)
             .then(None)
             .when(pl.col(column).is_null())
             .then(0)
@@ -1313,7 +1400,7 @@ for year in range(1, N_YEARS_OUT + 1):
         ]
         +
         [
-            pl.when(pl.col("_by_end") > NSC_CUTOFF_DATE)
+            pl.when(by_end > NSC_CUTOFF_DATE)
             .then(None)
             .when(pl.col(column).is_null())
             .then(0)
@@ -1330,21 +1417,21 @@ for year in range(1, N_YEARS_OUT + 1):
 ###########################################################
 
 outcome_columns = [
-    "att_any_inY1", "att_any_inY2", "att_any_inY3", "att_any_inY4",
-    "att_4yr_inY1", "att_4yr_inY2", "att_4yr_inY3", "att_4yr_inY4",
+    "att_any_inY1", "att_any_inY2", "att_any_inY3", "att_any_inY4", "att_any_inY5", "att_any_inY6", "att_any_inY7", "att_any_inY8",
+    "att_4yr_inY1", "att_4yr_inY2", "att_4yr_inY3", "att_4yr_inY4", "att_4yr_inY5", "att_4yr_inY6", "att_4yr_inY7", "att_4yr_inY8",
     "att_2yr_inY1", "att_2yr_inY2", "att_2yr_inY3", "att_2yr_inY4",
     "att_elite_inY1", "att_elite_inY2", "att_elite_inY3", "att_elite_inY4",
 
     "att_any_byY2", "att_any_byY3", "att_any_byY4",
     "att_4yr_byY2", "att_4yr_byY3", "att_4yr_byY4",
     "att_2yr_byY2", "att_2yr_byY3", "att_2yr_byY4", 
-    "att_elite_byY2", "att_elite_byY3", "att_elite_byY4",
+    "att_elite_byY2", "att_elite_byY3", "att_elite_byY4", "att_elite_byY5", "att_elite_byY6",
 
     "cmp_any_byY4", "cmp_any_byY5", "cmp_any_byY6", "cmp_any_byY7", "cmp_any_byY8",
     "cmp_AA_byY4", "cmp_AA_byY5", "cmp_AA_byY6", "cmp_AA_byY7", "cmp_AA_byY8",
     "cmp_BA_byY4", "cmp_BA_byY5", "cmp_BA_byY6", "cmp_BA_byY7", "cmp_BA_byY8",
 
-
+    "cmp_elite_byY4", "cmp_elite_byY5", "cmp_elite_byY6", "cmp_elite_byY7", "cmp_elite_byY8",
 ]
 
 nsc_outcomes = nsc_outcomes.with_columns(
@@ -1355,6 +1442,11 @@ nsc_outcomes = nsc_outcomes.with_columns(
 
 keep_columns = [
     "sid_cepr",
+    "k_mean",
+    "predicted_completion",
+    "ID_FSC_firstinst",
+    "college_name_firstinst",
+    "completion_rate_150pct_firstinst",
 ] + outcome_columns
 
 nsc_outcomes_final = nsc_outcomes.select(keep_columns)
@@ -1368,6 +1460,25 @@ nsc_outcomes.with_columns(
 print(f"Wrote {OUTCOMES}")
 print(f"Wrote {OUTCOMES.with_suffix('.csv')}")
 print(f"Rows: {nsc_outcomes.height}, columns: {len(nsc_outcomes.columns)}")
+
+first_college_coverage = (
+    nsc_outcomes.filter(pl.col("ID_FSC_firstinst").is_not_null())
+    .group_by("ID_FSC_firstinst")
+    .agg(
+        pl.col("k_mean_firstinst").is_not_null().any().alias("has_k_mean"),
+        pl.col("completion_rate_150pct_firstinst")
+        .is_not_null()
+        .any()
+        .alias("has_completion_rate"),
+    )
+)
+print(
+    "First-college coverage: "
+    f"{first_college_coverage['has_k_mean'].sum()}/"
+    f"{first_college_coverage.height} with college-specific k_mean; "
+    f"{first_college_coverage['has_completion_rate'].sum()}/"
+    f"{first_college_coverage.height} with IPEDS completion rate"
+)
 print(
     nsc_outcomes.select("sid_cepr", "cohort_lottery", "recovered_nsc_outcome").sort(
         "sid_cepr"