ltc-code 0.2.25__tar.gz → 0.2.27__tar.gz

{ltc_code-0.2.25 → ltc_code-0.2.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ltc-code
-Version: 0.2.25
+Version: 0.2.27
 Summary: Add your description here
 Requires-Dist: fastexcel>=0.16,<0.20
 Requires-Dist: polars>=1.36.1,<1.42

{ltc_code-0.2.25 → ltc_code-0.2.27}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.2.25"
+version = "0.2.27"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"

{ltc_code-0.2.25 → ltc_code-0.2.27}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.2.25"
+version = "0.2.27"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"

{ltc_code-0.2.25 → ltc_code-0.2.27}/src/ltc_code/nsc/build_nsc_outcomes.py

@@ -461,7 +461,7 @@ ipeds_completion = (
 
 # Use the graduation cohort itself as the student weight. Summing completers and
 # cohorts is equivalent to weighting each college's rate by its IPEDS denominator.
-ipeds_sector_rates = (
+ipeds_completion_with_sector = (
     ipeds_completion
     .join(
         ipeds_2013.select("unitid", "years_ipeds_2013"),
@@ -501,6 +501,10 @@ ipeds_sector_rates = (
         .otherwise(pl.lit("2yr_or_less"))
         .alias("college_sector")
     )
+)
+
+ipeds_sector_rates = (
+    ipeds_completion_with_sector
     .group_by("college_sector")
     .agg(
         pl.col("completers_150pct_ip").sum().alias("completers"),
@@ -1120,6 +1124,7 @@ enroll = (
         pl.col("cohort_18").first(),
         pl.col("college").first(),
         pl.col("college_name").first(),
+        pl.col("unitid").drop_nulls().first(),
         pl.col("college_years").first(),
         pl.col("college_any").max(),
         pl.col("college_4yr").max(),
@@ -1331,6 +1336,7 @@ first_institution = (
         .first()
         .alias("college_name_firstinst"),
         pl.col("college_years").first().alias("college_years_firstinst"),
+        pl.col("unitid").first().alias("unitid_firstinst"),
         pl.col("completion_rate_150pct_ip")
         .first()
         .alias("completion_rate_150pct_firstinst"),
@@ -1725,61 +1731,163 @@ for age in AGE_ATTENDANCE_RANGE:
     )
 
 
-# Define expected completion over the cumulative by-Y4 attendance window. The
-# detailed measure uses the first institution's rate (with the tier-median fill
-# above), gives observed nonattenders zero, and leaves attendee rates missing
-# when neither the institution rate nor its tier median is available.
-nsc_outcomes = nsc_outcomes.with_columns(
-    pl.when(pl.col("att_any_byY4") == 0)
-    .then(None)
-    .otherwise(pl.col("completion_rate_150pct_ip"))
-    .alias("completion_rate_150pct_ip")
+# Calculate coarse averages, restricted to the first institutions among observed by-Y4 attendees
+sample_firstinst_unitids = (
+    nsc_outcomes.filter(pl.col("att_any_byY4") == 1)
+    .select(pl.col("unitid_firstinst").alias("unitid"))
+    .drop_nulls()
+    .unique()
+)
+
+ipeds_sample_sector_rates = (
+    ipeds_completion_with_sector.join(
+        sample_firstinst_unitids,
+        on="unitid",
+        how="semi",
+    )
+    .group_by("college_sector")
+    .agg(
+        pl.len().alias("institutions"),
+        pl.col("completers_150pct_ip").sum().alias("completers"),
+        pl.col("cohort_adj_150pct_ip").sum().alias("cohort"),
+    )
+    .with_columns(
+        (pl.col("completers") / pl.col("cohort")).alias("cmp_rate_coarse_sample")
+    )
 )
 
+cmp_rate_4yr_coarse_sample = ipeds_sample_sector_rates.filter(
+    pl.col("college_sector") == "4yr"
+).item(0, "cmp_rate_coarse_sample")
+cmp_rate_2yr_coarse_sample = ipeds_sample_sector_rates.filter(
+    pl.col("college_sector") == "2yr_or_less"
+).item(0, "cmp_rate_coarse_sample")
+sample_4yr_institutions = ipeds_sample_sector_rates.filter(
+    pl.col("college_sector") == "4yr"
+).item(0, "institutions")
+sample_2yr_institutions = ipeds_sample_sector_rates.filter(
+    pl.col("college_sector") == "2yr_or_less"
+).item(0, "institutions")
+
+for label, value in {
+    "sample four-year completion rate": cmp_rate_4yr_coarse_sample,
+    "sample two-year-or-less completion rate": cmp_rate_2yr_coarse_sample,
+}.items():
+    if value is None or not 0 <= value <= 1:
+        raise ValueError(f"Invalid IPEDS-cohort-weighted {label}: {value}")
+
+
+# Define completion-rate measures over the cumulative by-Y4 attendance window, just for attendees
 nsc_outcomes = nsc_outcomes.with_columns(
-    pl.col("completion_rate_150pct_ip").alias("cmp_rate"),
-    pl.when(pl.col("att_any_byY4").is_null())
-    .then(None)
-    .when(pl.col("att_4yr_byY4") == 1)
-    .then(cmp_rate_4yr_coarse)
-    .when(pl.col("att_2yr_byY4") == 1)
-    .then(cmp_rate_2yr_coarse)
-    .when(pl.col("att_any_byY4") == 0)
-    .then(0.0)
+    # Detailed cmp rate
+    pl.when(pl.col("att_any_byY4") == 1)
+    .then(pl.col("completion_rate_150pct_ip"))
+    .otherwise(None)
+    .alias("cmp_rate"),
+    # Coarse cmp rate
+    pl.when(pl.col("att_any_byY4") == 1)
+    .then(
+        pl.when(pl.col("college_years_firstinst") == 4)
+        .then(cmp_rate_4yr_coarse)
+        .when(pl.col("college_years_firstinst").is_in([1, 2]))
+        .then(cmp_rate_2yr_coarse)
+        .otherwise(None)
+    )
     .otherwise(None)
     .alias("cmp_rate_coarse"),
+    # Coarse cmp rate, using in-sample avgs
+    pl.when(pl.col("att_any_byY4") == 1)
+    .then(
+        pl.when(pl.col("college_years_firstinst") == 4)
+        .then(cmp_rate_4yr_coarse_sample)
+        .when(pl.col("college_years_firstinst").is_in([1, 2]))
+        .then(cmp_rate_2yr_coarse_sample)
+        .otherwise(None)
+    )
+    .otherwise(None)
+    .alias("cmp_rate_coarse_sample"),
+    # Coarsen only the 4yr cmp rates
+    pl.when(pl.col("att_any_byY4") == 1)
+    .then(
+        pl.when(pl.col("college_years_firstinst") == 4)
+        .then(cmp_rate_4yr_coarse)
+        .when(pl.col("college_years_firstinst").is_in([1, 2]))
+        .then(pl.col("completion_rate_150pct_ip"))
+        .otherwise(None)
+    )
+    .otherwise(None)
+    .alias("cmp_rate_coarsen_4yr"),
+    # Coarsen only the 2yr cmp rates
+    pl.when(pl.col("att_any_byY4") == 1)
+    .then(
+        pl.when(pl.col("college_years_firstinst") == 4)
+        .then(pl.col("completion_rate_150pct_ip"))
+        .when(pl.col("college_years_firstinst").is_in([1, 2]))
+        .then(cmp_rate_2yr_coarse)
+        .otherwise(None)
+    )
+    .otherwise(None)
+    .alias("cmp_rate_coarsen_2yr"),
 )
 
+# Now set observed nonattenders to zero for each measure
 nsc_outcomes = nsc_outcomes.with_columns(
-    pl.when(pl.col("att_any_byY4").is_null())
-    .then(None)
-    .when(pl.col("att_any_byY4") == 0)
+    # Detailed cmp rate
+    pl.when(pl.col("att_any_byY4") == 0)
     .then(0.0)
     .otherwise(pl.col("cmp_rate"))
     .alias("adj_cmp_rate"),
-    pl.col("cmp_rate_coarse")
-    .alias("adj_cmp_rate_coarse"),
-).with_columns(
-    pl.when(pl.col("att_any_byY4").is_null())
-    .then(None)
-    .otherwise(pl.col("cmp_rate"))
-    .alias("cmp_rate"),
-    pl.when(pl.col("att_any_byY4").is_null())
-    .then(None)
-    .otherwise(pl.col("adj_cmp_rate"))
-    .alias("adj_cmp_rate"),
-    pl.when(pl.col("att_any_byY4").is_null())
-    .then(None)
+    # Coarse cmp rate
+    pl.when(pl.col("att_any_byY4") == 0)
+    .then(0.0)
     .otherwise(pl.col("cmp_rate_coarse"))
-    .alias("cmp_rate_coarse"),
-    pl.when(pl.col("att_any_byY4").is_null())
-    .then(None)
-    .otherwise(pl.col("adj_cmp_rate_coarse"))
     .alias("adj_cmp_rate_coarse"),
-    pl.when(pl.col("att_any_byY4").is_null())
-    .then(None)
+    # Coarse cmp rate, using in-sample avgs
+    pl.when(pl.col("att_any_byY4") == 0)
+    .then(0.0)
+    .otherwise(pl.col("cmp_rate_coarse_sample"))
+    .alias("adj_cmp_rate_coarse_sample"),
+    # Coarsen only the 4yr cmp rates
+    pl.when(pl.col("att_any_byY4") == 0)
+    .then(0.0)
+    .otherwise(pl.col("cmp_rate_coarsen_4yr"))
+    .alias("adj_cmp_rate_coarsen_4yr"),
+    # Coarsen only the 2yr cmp rates
+    pl.when(pl.col("att_any_byY4") == 0)
+    .then(0.0)
+    .otherwise(pl.col("cmp_rate_coarsen_2yr"))
+    .alias("adj_cmp_rate_coarsen_2yr"),
+)
+
+
+# Split the existing prediction by first institution, preserving missing rates.
+nsc_outcomes = nsc_outcomes.with_columns(
+    pl.when(
+        pl.col("adj_cmp_rate").is_not_null()
+        & pl.col("college_years_firstinst").is_in([1, 2])
+    )
+    .then(0.0)
+    .otherwise(pl.col("adj_cmp_rate"))
+    .alias("adj_cmp_rate_4yr"),
+    pl.when(
+        pl.col("adj_cmp_rate").is_not_null()
+        & (pl.col("college_years_firstinst") == 4)
+    )
+    .then(0.0)
     .otherwise(pl.col("adj_cmp_rate"))
-    .alias("predicted_completion"),
+    .alias("adj_cmp_rate_2yr"),
+)
+
+# Match the available 1098-T calendar years, including the missing 2015 year.
+tax_years = list(range(2011, 2015)) + list(range(2016, 2023))
+nsc_outcomes = nsc_outcomes.with_columns(
+    [
+        pl.when((pl.col("cohort_lottery") + age).is_in(tax_years))
+        .then(pl.col(f"att_any_{age}"))
+        .otherwise(None)
+        .alias(f"att_any_1098_{age}")
+        for age in AGE_ATTENDANCE_RANGE
+    ]
 )
 
 
@@ -1799,6 +1907,7 @@ outcome_columns = [
     "att_elite_byY2", "att_elite_byY3", "att_elite_byY4", "att_elite_byY5", "att_elite_byY6",
 
     *[f"att_any_{age}" for age in AGE_ATTENDANCE_RANGE],
+    *[f"att_any_1098_{age}" for age in AGE_ATTENDANCE_RANGE],
     *[f"att_4yr_{age}" for age in AGE_ATTENDANCE_RANGE],
     *[f"att_2yr_{age}" for age in AGE_ATTENDANCE_RANGE],
 
@@ -1855,9 +1964,12 @@ keep_columns = [
     "completion_rate_150pct_ip",
     "cmp_rate",
     "adj_cmp_rate",
-    "cmp_rate_coarse",
+    "adj_cmp_rate_2yr",
+    "adj_cmp_rate_4yr",
     "adj_cmp_rate_coarse",
-    "predicted_completion",
+    "adj_cmp_rate_coarse_sample",
+    "adj_cmp_rate_coarsen_4yr",
+    "adj_cmp_rate_coarsen_2yr",
     "ID_FSC_firstinst",
     "college_name_firstinst",
     "tier_firstinst",
@@ -1902,6 +2014,13 @@ print(
     f"completion 2yr-or-less={cmp_rate_2yr_coarse:.4f}; "
     "observed by-Y4 nonattender completion=0"
 )
+print(
+    "Sample-institution IPEDS-cohort-weighted coarse values: "
+    f"completion 4yr={cmp_rate_4yr_coarse_sample:.4f} "
+    f"({sample_4yr_institutions} institutions), "
+    f"completion 2yr-or-less={cmp_rate_2yr_coarse_sample:.4f} "
+    f"({sample_2yr_institutions} institutions)"
+)
 
 observable_y4 = nsc_outcomes.filter(pl.col("att_any_byY4").is_not_null())
 coarse_audit = observable_y4.select(
@@ -1914,6 +2033,15 @@ coarse_audit = observable_y4.select(
     .alias("adj_cmp_rate_coarse_missing"),
     pl.col("adj_cmp_rate_coarse").min().alias("adj_cmp_rate_coarse_min"),
     pl.col("adj_cmp_rate_coarse").max().alias("adj_cmp_rate_coarse_max"),
+    pl.col("adj_cmp_rate_coarse_sample")
+    .null_count()
+    .alias("adj_cmp_rate_coarse_sample_missing"),
+    pl.col("adj_cmp_rate_coarse_sample")
+    .min()
+    .alias("adj_cmp_rate_coarse_sample_min"),
+    pl.col("adj_cmp_rate_coarse_sample")
+    .max()
+    .alias("adj_cmp_rate_coarse_sample_max"),
 )
 
 if coarse_audit.item(0, "k_mean_coarse_missing") > 0:
@@ -1922,6 +2050,10 @@ if coarse_audit.item(0, "adj_cmp_rate_coarse_missing") > 0:
     raise ValueError(
         "adj_cmp_rate_coarse is unexpectedly missing for observable students."
     )
+if coarse_audit.item(0, "adj_cmp_rate_coarse_sample_missing") > 0:
+    raise ValueError(
+        "adj_cmp_rate_coarse_sample is unexpectedly missing for observable students."
+    )
 
 print("Coarse outcome audit:")
 print(coarse_audit)
@@ -1940,22 +2072,22 @@ attendee_completion_audit = (
         )
         .sum()
         .alias("filled_by_tier_median"),
-        pl.col("predicted_completion")
+        pl.col("adj_cmp_rate")
         .is_null()
         .sum()
         .alias("missing_after_tier_median"),
-        pl.col("predicted_completion")
+        pl.col("adj_cmp_rate")
         .is_null()
         .mean()
         .alias("missing_after_tier_median_share"),
         (
-            pl.col("predicted_completion").is_null()
+            pl.col("adj_cmp_rate").is_null()
             & pl.col("tier_firstinst").is_null()
         )
         .sum()
         .alias("missing_after_tier_median_no_tier"),
         (
-            pl.col("predicted_completion").is_null()
+            pl.col("adj_cmp_rate").is_null()
             & pl.col("tier_firstinst").is_not_null()
         )
         .sum()
@@ -1969,3 +2101,18 @@ print(
         "sid_cepr"
     )
 )
+
+# Nonmissing year-eight completion identifies students with the full Y8 window.
+year8_attendance_audit = (
+    nsc_outcomes.filter(pl.col("cmp_BA_byY8").is_not_null())
+    .select(
+        pl.len().alias("N_year8_sample"),
+        (pl.col("att_any_byY4") == 0).sum().alias("N_no_attendance_byY4"),
+        (
+            (pl.col("att_any_byY4") == 0)
+            & (pl.col("cmp_BA_byY8") == 1)
+        ).sum().alias("N_no_attendance_byY4_BA_byY8"),
+    )
+)
+print("Year-eight sample attendance and late BA completion check:")
+print(year8_attendance_audit)