ltc-code 0.2.20__tar.gz → 0.2.22__tar.gz

{ltc_code-0.2.20 → ltc_code-0.2.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ltc-code
-Version: 0.2.20
+Version: 0.2.22
 Summary: Add your description here
 Requires-Dist: fastexcel>=0.16,<0.20
 Requires-Dist: polars>=1.36.1,<1.42

{ltc_code-0.2.20 → ltc_code-0.2.22}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.2.20"
+version = "0.2.22"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"

{ltc_code-0.2.20 → ltc_code-0.2.22}/src/ltc_code/nsc/build_nsc_outcomes.py

@@ -86,6 +86,8 @@ NSC_CUTOFF_DATE = date(2026, 12, 31)
 RANDOM_SEED = 3852804
 STEM_CIP_FAMILIES = {11, 14, 15, 26, 27, 40, 41}
 COLLEGE_TYPES = ["any", "4yr", "2yr", "elite"]
+AGE_ATTENDANCE_TYPES = ["any", "4yr", "2yr"]
+AGE_ATTENDANCE_RANGE = range(18, 27)
 
 
 ###########################################################
@@ -235,11 +237,13 @@ chetty_college_outcomes = (
         missing_string_as_null=True,
         value_labels_as_strings=False,
     )
-    .select("super_opeid", "tier", "tier_name", "k_mean")
+    .select("super_opeid", "tier", "tier_name", "iclevel", "count", "k_mean")
     .collect()
     .with_columns(
         pl.col("super_opeid").cast(pl.Int64, strict=False),
         pl.col("tier").cast(pl.Int8, strict=False),
+        pl.col("iclevel").cast(pl.Int8, strict=False),
+        pl.col("count").cast(pl.Float64, strict=False),
         pl.col("k_mean").cast(pl.Float64, strict=False),
     )
     .drop_nulls("super_opeid")
@@ -253,6 +257,38 @@ k_mean_insuffdata = chetty_college_outcomes.filter(
     pl.col("super_opeid") == -1
 ).item(0, "k_mean")
 
+# Collapse MRC college-specific earnings to national student-weighted means.
+# MRC count is the mean number of children per cohort represented by each row.
+mrc_sector_means = (
+    chetty_college_outcomes
+    .filter(
+        (pl.col("super_opeid") > 0)
+        & pl.col("iclevel").is_in([1, 2, 3])
+        & (pl.col("count") > 0)
+        & pl.col("k_mean").is_not_null()
+    )
+    .with_columns(
+        pl.when(pl.col("iclevel") == 1)
+        .then(pl.lit("4yr"))
+        .otherwise(pl.lit("2yr_or_less"))
+        .alias("college_sector")
+    )
+    .group_by("college_sector")
+    .agg(
+        (
+            (pl.col("k_mean") * pl.col("count")).sum()
+            / pl.col("count").sum()
+        ).alias("k_mean_coarse")
+    )
+)
+
+k_mean_4yr_coarse = mrc_sector_means.filter(
+    pl.col("college_sector") == "4yr"
+).item(0, "k_mean_coarse")
+k_mean_2yr_coarse = mrc_sector_means.filter(
+    pl.col("college_sector") == "2yr_or_less"
+).item(0, "k_mean_coarse")
+
 chetty_ope_crosswalk = (
     scan_readstat(
         CHETTY_MRC_TABLE11,
@@ -275,24 +311,102 @@ chetty_by_opeid = (
     .unique("opeid", keep="first")
 )
 
-# IPEDS reports completion within 150% of normal time: six years at four-year
-# colleges and three years at two-year colleges.
+# The IPEDS 150%-of-normal-time rate is six-year completion at four-year
+# institutions and three-year completion at two-year institutions.
 ipeds_completion = (
     scan_readstat(
         IPEDS_DATA,
         missing_string_as_null=True,
         value_labels_as_strings=False,
     )
-    .select("unitid", "completion_rate_150pct_ip")
+    .select(
+        "unitid",
+        "completion_rate_150pct_ip",
+        "cohort_adj_150pct_ip",
+        "completers_150pct_ip",
+    )
     .collect()
     .with_columns(
         pl.col("unitid").cast(pl.Int64, strict=False),
         pl.col("completion_rate_150pct_ip").cast(pl.Float64, strict=False),
+        pl.col("cohort_adj_150pct_ip").cast(pl.Float64, strict=False),
+        pl.col("completers_150pct_ip").cast(pl.Float64, strict=False),
     )
     .drop_nulls("unitid")
     .unique("unitid", keep="first")
 )
 
+# Use the graduation cohort itself as the student weight. Summing completers and
+# cohorts is equivalent to weighting each college's rate by its IPEDS denominator.
+ipeds_sector_rates = (
+    ipeds_completion
+    .join(
+        ipeds_2013.select("unitid", "years_ipeds_2013"),
+        on="unitid",
+        how="left",
+        validate="1:1",
+    )
+    .join(
+        ipeds_manual.select("unitid", "years_manual"),
+        on="unitid",
+        how="left",
+        validate="1:1",
+    )
+    .join(
+        directory.select("unitid", "years_directory"),
+        on="unitid",
+        how="left",
+        validate="1:1",
+    )
+    .with_columns(
+        pl.coalesce(
+            [
+                pl.col("years_manual"),
+                pl.col("years_ipeds_2013"),
+                pl.col("years_directory"),
+            ]
+        ).alias("college_years")
+    )
+    .filter(
+        pl.col("college_years").is_in([1, 2, 4])
+        & (pl.col("cohort_adj_150pct_ip") > 0)
+        & pl.col("completers_150pct_ip").is_not_null()
+    )
+    .with_columns(
+        pl.when(pl.col("college_years") == 4)
+        .then(pl.lit("4yr"))
+        .otherwise(pl.lit("2yr_or_less"))
+        .alias("college_sector")
+    )
+    .group_by("college_sector")
+    .agg(
+        pl.col("completers_150pct_ip").sum().alias("completers"),
+        pl.col("cohort_adj_150pct_ip").sum().alias("cohort"),
+    )
+    .with_columns((pl.col("completers") / pl.col("cohort")).alias("cmp_rate_coarse"))
+)
+
+cmp_rate_4yr_coarse = ipeds_sector_rates.filter(
+    pl.col("college_sector") == "4yr"
+).item(0, "cmp_rate_coarse")
+cmp_rate_2yr_coarse = ipeds_sector_rates.filter(
+    pl.col("college_sector") == "2yr_or_less"
+).item(0, "cmp_rate_coarse")
+
+for label, value in {
+    "four-year k_mean": k_mean_4yr_coarse,
+    "two-year-or-less k_mean": k_mean_2yr_coarse,
+}.items():
+    if value is None or value <= 0:
+        raise ValueError(f"Invalid national student-weighted {label}: {value}")
+
+for label, value in {
+    "four-year completion rate": cmp_rate_4yr_coarse,
+    "two-year-or-less completion rate": cmp_rate_2yr_coarse,
+}.items():
+    if value is None or not 0 <= value <= 1:
+        raise ValueError(f"Invalid national student-weighted {label}: {value}")
+
 
 ###########################################################
 # Build college reference data
@@ -331,7 +445,7 @@ college_ref = (
     )
     .with_columns(
         (pl.col("college_years") == 4).cast(pl.Int8).alias("college_4yr"),
-        (pl.col("college_years") == 2).cast(pl.Int8).alias("college_2yr"),
+        pl.col("college_years").is_in([1, 2]).cast(pl.Int8).alias("college_2yr"),
         pl.col("tier").is_in([1, 2]).cast(pl.Int8).alias("college_elite"),
     )
     .with_columns(
@@ -908,6 +1022,40 @@ for year in range(1, N_YEARS_OUT + 1):
                 )
             )
 
+# Build calendar-year attendance indicators for each age. These use birth year
+# plus age, rather than the July-to-June school-year windows above.
+for age in AGE_ATTENDANCE_RANGE:
+    bounded = enroll.with_columns(
+        (pl.col("cohort_lottery") + age).alias("_age_window_year")
+    ).with_columns(
+        pl.date(pl.col("_age_window_year"), 1, 1).alias("_age_window_start"),
+        pl.date(pl.col("_age_window_year"), 12, 31).alias("_age_window_end"),
+    ).with_columns(
+        (
+            (pl.col("term_start_date") <= pl.col("_age_window_end"))
+            & (pl.col("term_end_date") >= pl.col("_age_window_start"))
+            & (pl.col("_term_att") == 1)
+        )
+        .cast(pl.Int8)
+        .alias("_overlaps")
+    )
+
+    enrollment_pieces.append(
+        bounded.group_by("sid_cepr").agg(
+            [
+                (
+                    pl.col("_overlaps")
+                    * pl.col(f"college_{college_type}")
+                    .fill_null(0)
+                    .cast(pl.Int8)
+                )
+                .max()
+                .alias(f"att_{college_type}_{age}")
+                for college_type in AGE_ATTENDANCE_TYPES
+            ]
+        )
+    )
+
 enrollment_outcomes = enrollment_pieces[0]
 for piece in enrollment_pieces[1:]:
     enrollment_outcomes = enrollment_outcomes.join(
@@ -939,7 +1087,6 @@ first_institution = (
         .first()
         .alias("college_name_firstinst"),
         pl.col("college_years").first().alias("college_years_firstinst"),
-        pl.col("k_mean").first().alias("k_mean_firstinst"),
         pl.col("completion_rate_150pct_ip")
         .first()
         .alias("completion_rate_150pct_firstinst"),
@@ -1370,6 +1517,14 @@ nsc_outcomes = (
         .then(k_mean_neverattend)
         .otherwise(pl.col("k_mean"))
         .alias("k_mean"),
+        pl.when(pl.col("ID_FSC_firstinst").is_null())
+        .then(k_mean_neverattend)
+        .when(pl.col("college_years_firstinst") == 4)
+        .then(k_mean_4yr_coarse)
+        .when(pl.col("college_years_firstinst").is_in([1, 2]))
+        .then(k_mean_2yr_coarse)
+        .otherwise(k_mean_insuffdata)
+        .alias("k_mean_coarse"),
     )
 )
 
@@ -1466,6 +1621,27 @@ for year in range(1, N_YEARS_OUT + 1):
         ]
     )
 
+for age in AGE_ATTENDANCE_RANGE:
+    age_start = pl.date(pl.col("cohort_lottery") + age, 1, 1)
+    age_columns = [
+        column
+        for column in nsc_outcomes.columns
+        if column.startswith("att_") and column.endswith(f"_{age}")
+    ]
+
+    nsc_outcomes = nsc_outcomes.with_columns(
+        [
+            pl.when(age_start > NSC_CUTOFF_DATE)
+            .then(None)
+            .when(pl.col(column).is_null())
+            .then(0)
+            .otherwise(pl.col(column))
+            .cast(pl.Int8)
+            .alias(column)
+            for column in age_columns
+        ]
+    )
+
 
 # Sarah removes the raw school-specific rate for students who were not observed
 # attending by Y2 before creating cmp_rate.
@@ -1479,10 +1655,18 @@ nsc_outcomes = nsc_outcomes.with_columns(
 # Sarah creates cmp_rate as a copy of the restricted raw completion rate, then
 # fills missing cmp_rate values with the late-enrollee/control-group mean.
 nsc_outcomes = nsc_outcomes.with_columns(
-    pl.col("completion_rate_150pct_ip").alias("cmp_rate")
+    pl.col("completion_rate_150pct_ip").alias("cmp_rate"),
+    pl.when(pl.col("att_any_byY2") == 0)
+    .then(None)
+    .when(pl.col("college_years_firstinst") == 4)
+    .then(cmp_rate_4yr_coarse)
+    .when(pl.col("college_years_firstinst").is_in([1, 2]))
+    .then(cmp_rate_2yr_coarse)
+    .otherwise(None)
+    .alias("cmp_rate_coarse"),
 )
 
-late_enrollee_mean = (
+late_enrollee_means = (
     nsc_outcomes
     .filter(
         (pl.col("att_4yr_inY2") == 0)
@@ -1494,15 +1678,29 @@ late_enrollee_mean = (
         )
         & (pl.col("offer") == 0)
     )
-    .select(pl.col("cmp_rate").mean())
-    .item()
+    .select(
+        pl.col("cmp_rate").mean().alias("cmp_rate"),
+        pl.col("cmp_rate_coarse").mean().alias("cmp_rate_coarse"),
+    )
 )
 
+late_enrollee_mean = late_enrollee_means.item(0, "cmp_rate")
+late_enrollee_coarse_mean = late_enrollee_means.item(0, "cmp_rate_coarse")
+
+if late_enrollee_mean is None or late_enrollee_coarse_mean is None:
+    raise ValueError(
+        "The late-enrollee control donor group has no usable completion-rate values."
+    )
+
 nsc_outcomes = nsc_outcomes.with_columns(
     pl.when(pl.col("cmp_rate").is_null())
     .then(pl.lit(late_enrollee_mean, dtype=pl.Float64))
     .otherwise(pl.col("cmp_rate"))
-    .alias("adj_cmp_rate")
+    .alias("adj_cmp_rate"),
+    pl.when(pl.col("cmp_rate_coarse").is_null())
+    .then(pl.lit(late_enrollee_coarse_mean, dtype=pl.Float64))
+    .otherwise(pl.col("cmp_rate_coarse"))
+    .alias("adj_cmp_rate_coarse"),
 ).with_columns(
     pl.when(pl.col("att_4yr_byY2").is_null())
     .then(None)
@@ -1512,6 +1710,14 @@ nsc_outcomes = nsc_outcomes.with_columns(
     .then(None)
     .otherwise(pl.col("adj_cmp_rate"))
     .alias("adj_cmp_rate"),
+    pl.when(pl.col("att_4yr_byY2").is_null())
+    .then(None)
+    .otherwise(pl.col("cmp_rate_coarse"))
+    .alias("cmp_rate_coarse"),
+    pl.when(pl.col("att_4yr_byY2").is_null())
+    .then(None)
+    .otherwise(pl.col("adj_cmp_rate_coarse"))
+    .alias("adj_cmp_rate_coarse"),
     # Preserve the package's existing name as an alias for Sarah's adjusted rate.
     pl.when(pl.col("att_4yr_byY2").is_null())
     .then(None)
@@ -1535,6 +1741,10 @@ outcome_columns = [
     "att_2yr_byY2", "att_2yr_byY3", "att_2yr_byY4", 
     "att_elite_byY2", "att_elite_byY3", "att_elite_byY4", "att_elite_byY5", "att_elite_byY6",
 
+    *[f"att_any_{age}" for age in AGE_ATTENDANCE_RANGE],
+    *[f"att_4yr_{age}" for age in AGE_ATTENDANCE_RANGE],
+    *[f"att_2yr_{age}" for age in AGE_ATTENDANCE_RANGE],
+
     "cmp_any_byY4", "cmp_any_byY5", "cmp_any_byY6", "cmp_any_byY7", "cmp_any_byY8",
     "cmp_AA_byY4", "cmp_AA_byY5", "cmp_AA_byY6", "cmp_AA_byY7", "cmp_AA_byY8",
     "cmp_BA_byY4", "cmp_BA_byY5", "cmp_BA_byY6", "cmp_BA_byY7", "cmp_BA_byY8",
@@ -1551,9 +1761,12 @@ nsc_outcomes = nsc_outcomes.with_columns(
 keep_columns = [
     "sid_cepr",
     "k_mean",
+    "k_mean_coarse",
     "completion_rate_150pct_ip",
     "cmp_rate",
     "adj_cmp_rate",
+    "cmp_rate_coarse",
+    "adj_cmp_rate_coarse",
     "predicted_completion",
     "ID_FSC_firstinst",
     "college_name_firstinst",
@@ -1591,6 +1804,38 @@ print(
     f"{first_college_coverage['has_completion_rate'].sum()}/"
     f"{first_college_coverage.height} with IPEDS completion rate"
 )
+print(
+    "National student-weighted coarse values: "
+    f"k_mean 4yr={k_mean_4yr_coarse:.2f}, "
+    f"k_mean 2yr-or-less={k_mean_2yr_coarse:.2f}, "
+    f"completion 4yr={cmp_rate_4yr_coarse:.4f}, "
+    f"completion 2yr-or-less={cmp_rate_2yr_coarse:.4f}, "
+    f"nonattender completion imputation={late_enrollee_mean:.4f}, "
+    f"coarse nonattender imputation={late_enrollee_coarse_mean:.4f}"
+)
+
+observable_y2 = nsc_outcomes.filter(pl.col("att_4yr_byY2").is_not_null())
+coarse_audit = observable_y2.select(
+    pl.len().alias("students"),
+    pl.col("k_mean_coarse").null_count().alias("k_mean_coarse_missing"),
+    pl.col("k_mean_coarse").min().alias("k_mean_coarse_min"),
+    pl.col("k_mean_coarse").max().alias("k_mean_coarse_max"),
+    pl.col("adj_cmp_rate_coarse")
+    .null_count()
+    .alias("adj_cmp_rate_coarse_missing"),
+    pl.col("adj_cmp_rate_coarse").min().alias("adj_cmp_rate_coarse_min"),
+    pl.col("adj_cmp_rate_coarse").max().alias("adj_cmp_rate_coarse_max"),
+)
+
+if coarse_audit.item(0, "k_mean_coarse_missing") > 0:
+    raise ValueError("k_mean_coarse is unexpectedly missing for observable students.")
+if coarse_audit.item(0, "adj_cmp_rate_coarse_missing") > 0:
+    raise ValueError(
+        "adj_cmp_rate_coarse is unexpectedly missing for observable students."
+    )
+
+print("Coarse outcome audit:")
+print(coarse_audit)
 print(
     nsc_outcomes.select("sid_cepr", "cohort_lottery", "recovered_nsc_outcome").sort(
         "sid_cepr"

ltc_code-0.2.22/src/ltc_code/signal_var_calcs.py

@@ -0,0 +1,41 @@
+# --- Import necessary packages ---
+import polars as pl
+
+
+SIGNAL_DEP_VARS = ["wages25", "college20"]
+
+
+def calculate_signal_variance(results: pl.DataFrame) -> pl.DataFrame:
+    """Calculate signal variance and its implied 90-10 spread by outcome."""
+    return (
+        results
+        .filter(pl.col("depvar").is_in(SIGNAL_DEP_VARS))
+        .drop_nulls(["sample", "depvar", "coef", "se"])
+        .group_by("depvar")
+        .agg(
+            pl.col("sample").n_unique().alias("n_cmos"),
+            pl.col("coef").mean().alias("mean_te"),
+            pl.col("coef").var().alias("raw_variance"),
+            (pl.col("se") ** 2).mean().alias("noise_variance"),
+        )
+        .with_columns(
+            (
+                pl.col("raw_variance")
+                - pl.col("noise_variance")
+            ).alias("signal_variance_raw")
+        )
+        # Retain the raw estimate, but use a nonnegative value for the spread.
+        .with_columns(
+            pl.col("signal_variance_raw")
+            .clip(lower_bound=0)
+            .alias("signal_variance")
+        )
+        .with_columns(
+            pl.col("signal_variance").sqrt().alias("signal_sd")
+        )
+        # Under normality, P90 - P10 equals 2.563103 standard deviations.
+        .with_columns(
+            (2.563103 * pl.col("signal_sd")).alias("spread_90_10")
+        )
+        .sort("depvar")
+    )