ltc-code 0.2.19__tar.gz → 0.2.21__tar.gz

{ltc_code-0.2.19 → ltc_code-0.2.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ltc-code
-Version: 0.2.19
+Version: 0.2.21
 Summary: Add your description here
 Requires-Dist: fastexcel>=0.16,<0.20
 Requires-Dist: polars>=1.36.1,<1.42

{ltc_code-0.2.19 → ltc_code-0.2.21}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.2.19"
+version = "0.2.21"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -12,6 +12,7 @@ dependencies = [
 
 [project.scripts]
 ltc-code = "ltc_code:main"
+ltc-summary-stats = "ltc_code.make_summary_stats_table:main"
 
 [build-system]
 requires = ["uv_build>=0.11.16,<0.12.0"]

ltc_code-0.2.21/src/ltc_code/make_summary_stats_table.py

@@ -0,0 +1,235 @@
+"""Create a paper-ready two-panel summary-statistics table from a CSV."""
+
+from __future__ import annotations
+
+import argparse
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Optional, Union
+
+import polars as pl
+
+
+REQUIRED_COLUMNS = ["sample", "Variable", "Mean", "P50", "SD"]
+COUNT_COLUMNS = ["N", "N_in_group"]
+
+
+def clean_latex(value: object) -> str:
+    text = "" if value is None else str(value)
+    for old, new in {
+        "\\": r"\textbackslash{}",
+        "&": r"\&",
+        "%": r"\%",
+        "$": r"\$",
+        "#": r"\#",
+        "_": r"\_",
+        "{": r"\{",
+        "}": r"\}",
+        "~": r"\textasciitilde{}",
+        "^": r"\textasciicircum{}",
+    }.items():
+        text = text.replace(old, new)
+    return text
+
+
+def format_cell(column: str, value: object) -> str:
+    if value is None:
+        return ""
+    if column in COUNT_COLUMNS:
+        try:
+            return f"{float(value):,.0f}"
+        except (TypeError, ValueError):
+            return clean_latex(value)
+    try:
+        return f"{float(value):,.2f}"
+    except (TypeError, ValueError):
+        return clean_latex(value)
+
+
+def standardize_columns(stats: pl.DataFrame) -> pl.DataFrame:
+    rename = {}
+    aliases = {
+        "sample": "sample",
+        "samples": "sample",
+        "variable": "Variable",
+        "var": "Variable",
+        "name": "Variable",
+        "mean": "Mean",
+        "avg": "Mean",
+        "average": "Mean",
+        "p50": "P50",
+        "median": "P50",
+        "med": "P50",
+        "sd": "SD",
+        "std": "SD",
+        "stddev": "SD",
+        "standard deviation": "SD",
+        "n": "N",
+        "obs": "N",
+        "observations": "N",
+        "n_in_group": "N_in_group",
+        "n group": "N_in_group",
+        "group n": "N_in_group",
+        "groups": "N_in_group",
+        "unique count": "N_in_group",
+        "unique counts": "N_in_group",
+    }
+    for column in stats.columns:
+        target = aliases.get(column.strip().lower())
+        if target is not None:
+            rename[column] = target
+    return stats.rename(rename)
+
+
+def panel_for(sample: str) -> str:
+    if "Age 20+" in sample:
+        return "A"
+    if "Age 25+" in sample:
+        return "B"
+    return "Other"
+
+
+def sample_label(sample: str) -> str:
+    return sample.replace(" Age 20+", "").replace(" Age 25+", "")
+
+
+def make_tex(
+    stats: pl.DataFrame,
+    *,
+    caption: str = "Summary Statistics by Sample",
+    label: str = "tab:summary_stats_by_sample",
+    note: str = "Notes: Means, medians, and standard deviations are reported by analysis sample.",
+) -> str:
+    numeric_columns = [
+        column for column in ["Mean", "P50", "SD", "N", "N_in_group"] if column in stats.columns
+    ]
+    header_labels = {"Mean": "Mean", "P50": "Median", "SD": "SD", "N": "N", "N_in_group": "Group N"}
+    panel_width = len(numeric_columns)
+    num_columns = 1 + 2 * panel_width
+    headers = ["Variable"] + [header_labels[column] for column in numeric_columns] * 2
+    column_spec = (
+        r"@{}p{1.35in}"
+        + "r" * panel_width
+        + r"@{\hspace{0.22in}}"
+        + "r" * panel_width
+        + "@{}"
+    )
+
+    lines = [
+        r"\documentclass[11pt]{article}",
+        r"\usepackage[margin=0.25in]{geometry}",
+        r"\usepackage{booktabs}",
+        r"\usepackage{caption}",
+        r"\usepackage{longtable}",
+        r"\usepackage{pdflscape}",
+        r"\usepackage[table]{xcolor}",
+        r"\begin{document}",
+        r"\begin{landscape}",
+        r"\scriptsize",
+        r"\setlength{\tabcolsep}{3.5pt}",
+        r"\renewcommand{\arraystretch}{1.08}",
+        rf"\begin{{longtable}}{{{column_spec}}}",
+        rf"\caption{{{clean_latex(caption)}}}\label{{{label}}}\\",
+        r"\toprule",
+        rf" & \multicolumn{{{panel_width}}}{{c}}{{\textbf{{Panel A: Age 20+}}}} & \multicolumn{{{panel_width}}}{{c}}{{\textbf{{Panel B: Age 25+}}}} \\",
+        rf"\cmidrule(lr){{2-{1 + panel_width}}}\cmidrule(l){{{2 + panel_width}-{num_columns}}}",
+        " & ".join(clean_latex(column) for column in headers) + r" \\",
+        r"\midrule",
+        r"\endfirsthead",
+        rf"\caption*{{{clean_latex(caption)} (continued)}}\\",
+        r"\toprule",
+        rf" & \multicolumn{{{panel_width}}}{{c}}{{\textbf{{Panel A: Age 20+}}}} & \multicolumn{{{panel_width}}}{{c}}{{\textbf{{Panel B: Age 25+}}}} \\",
+        rf"\cmidrule(lr){{2-{1 + panel_width}}}\cmidrule(l){{{2 + panel_width}-{num_columns}}}",
+        " & ".join(clean_latex(column) for column in headers) + r" \\",
+        r"\midrule",
+        r"\endhead",
+        r"\midrule",
+        rf"\multicolumn{{{num_columns}}}{{r}}{{Continued on next page}} \\",
+        r"\endfoot",
+        r"\bottomrule",
+        r"\endlastfoot",
+    ]
+
+    groups: list[str] = []
+    variables: dict[str, list[str]] = {}
+    values: dict[tuple[str, str, str], dict[str, object]] = {}
+    for row in stats.iter_rows(named=True):
+        group = sample_label(str(row["sample"]))
+        panel = panel_for(str(row["sample"]))
+        variable = str(row["Variable"])
+        if group not in groups:
+            groups.append(group)
+        variables.setdefault(group, [])
+        if variable not in variables[group]:
+            variables[group].append(variable)
+        values[(panel, group, variable)] = row
+
+    for group_index, group in enumerate(groups):
+        if group_index:
+            lines.append(r"\addlinespace[0.25em]")
+        lines.append(
+            rf"\rowcolor{{black!8}}\multicolumn{{{num_columns}}}{{@{{}}l@{{}}}}{{\textbf{{{clean_latex(group)}}}}} \\"
+        )
+        for variable in variables[group]:
+            cells = [clean_latex(variable)]
+            for panel in ["A", "B"]:
+                row = values.get((panel, group, variable))
+                cells.extend(format_cell(column, row[column]) if row is not None else "" for column in numeric_columns)
+            lines.append(" & ".join(cells) + r" \\")
+
+    lines.extend([
+        r"\end{longtable}",
+        rf"\noindent\footnotesize {clean_latex(note)}",
+        r"\end{landscape}",
+        r"\end{document}",
+    ])
+    return "\n".join(lines) + "\n"
+
+
+def build_summary_stats_table(
+    input_csv: Union[str, Path],
+    tex_out: Union[str, Path],
+    pdf_out: Optional[Union[str, Path]] = None,
+    *,
+    caption: str = "Summary Statistics by Sample",
+    label: str = "tab:summary_stats_by_sample",
+    note: str = "Notes: Means, medians, and standard deviations are reported by analysis sample.",
+    compile_pdf: bool = True,
+) -> Optional[Path]:
+    stats = standardize_columns(pl.read_csv(input_csv))
+    missing = [column for column in REQUIRED_COLUMNS if column not in stats.columns]
+    if missing:
+        raise ValueError(f"Missing required columns: {', '.join(missing)}")
+    display_columns = REQUIRED_COLUMNS + [column for column in COUNT_COLUMNS if column in stats.columns]
+    stats = stats.select(display_columns)
+
+    tex_path = Path(tex_out)
+    tex_path.parent.mkdir(parents=True, exist_ok=True)
+    tex_path.write_text(make_tex(stats, caption=caption, label=label, note=note))
+
+    if pdf_out is None or not compile_pdf:
+        return None
+    engine = shutil.which("tectonic") or shutil.which("pdflatex")
+    if engine is None:
+        return None
+    pdf_path = Path(pdf_out)
+    pdf_path.parent.mkdir(parents=True, exist_ok=True)
+    command = [engine, tex_path.name]
+    if Path(engine).name == "pdflatex":
+        command.insert(1, "-interaction=nonstopmode")
+    subprocess.run(command, cwd=tex_path.parent, check=True)
+    return pdf_path
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("input_csv", type=Path)
+    parser.add_argument("--tex-out", type=Path, required=True)
+    parser.add_argument("--pdf-out", type=Path)
+    args = parser.parse_args()
+    build_summary_stats_table(args.input_csv, args.tex_out, args.pdf_out)
+
+
+if __name__ == "__main__":
+    main()

{ltc_code-0.2.19 → ltc_code-0.2.21}/src/ltc_code/nsc/build_nsc_outcomes.py

@@ -235,11 +235,13 @@ chetty_college_outcomes = (
         missing_string_as_null=True,
         value_labels_as_strings=False,
     )
-    .select("super_opeid", "tier", "tier_name", "k_mean")
+    .select("super_opeid", "tier", "tier_name", "iclevel", "count", "k_mean")
     .collect()
     .with_columns(
         pl.col("super_opeid").cast(pl.Int64, strict=False),
         pl.col("tier").cast(pl.Int8, strict=False),
+        pl.col("iclevel").cast(pl.Int8, strict=False),
+        pl.col("count").cast(pl.Float64, strict=False),
         pl.col("k_mean").cast(pl.Float64, strict=False),
     )
     .drop_nulls("super_opeid")
@@ -253,6 +255,38 @@ k_mean_insuffdata = chetty_college_outcomes.filter(
     pl.col("super_opeid") == -1
 ).item(0, "k_mean")
 
+# Collapse MRC college-specific earnings to national student-weighted means.
+# MRC count is the mean number of children per cohort represented by each row.
+mrc_sector_means = (
+    chetty_college_outcomes
+    .filter(
+        (pl.col("super_opeid") > 0)
+        & pl.col("iclevel").is_in([1, 2, 3])
+        & (pl.col("count") > 0)
+        & pl.col("k_mean").is_not_null()
+    )
+    .with_columns(
+        pl.when(pl.col("iclevel") == 1)
+        .then(pl.lit("4yr"))
+        .otherwise(pl.lit("2yr_or_less"))
+        .alias("college_sector")
+    )
+    .group_by("college_sector")
+    .agg(
+        (
+            (pl.col("k_mean") * pl.col("count")).sum()
+            / pl.col("count").sum()
+        ).alias("k_mean_coarse")
+    )
+)
+
+k_mean_4yr_coarse = mrc_sector_means.filter(
+    pl.col("college_sector") == "4yr"
+).item(0, "k_mean_coarse")
+k_mean_2yr_coarse = mrc_sector_means.filter(
+    pl.col("college_sector") == "2yr_or_less"
+).item(0, "k_mean_coarse")
+
 chetty_ope_crosswalk = (
     scan_readstat(
         CHETTY_MRC_TABLE11,
@@ -275,24 +309,102 @@ chetty_by_opeid = (
     .unique("opeid", keep="first")
 )
 
-# IPEDS reports completion within 150% of normal time: six years at four-year
-# colleges and three years at two-year colleges.
+# The IPEDS 150%-of-normal-time rate is six-year completion at four-year
+# institutions and three-year completion at two-year institutions.
 ipeds_completion = (
     scan_readstat(
         IPEDS_DATA,
         missing_string_as_null=True,
         value_labels_as_strings=False,
     )
-    .select("unitid", "completion_rate_150pct_ip")
+    .select(
+        "unitid",
+        "completion_rate_150pct_ip",
+        "cohort_adj_150pct_ip",
+        "completers_150pct_ip",
+    )
     .collect()
     .with_columns(
         pl.col("unitid").cast(pl.Int64, strict=False),
         pl.col("completion_rate_150pct_ip").cast(pl.Float64, strict=False),
+        pl.col("cohort_adj_150pct_ip").cast(pl.Float64, strict=False),
+        pl.col("completers_150pct_ip").cast(pl.Float64, strict=False),
     )
     .drop_nulls("unitid")
     .unique("unitid", keep="first")
 )
 
+# Use the graduation cohort itself as the student weight. Summing completers and
+# cohorts is equivalent to weighting each college's rate by its IPEDS denominator.
+ipeds_sector_rates = (
+    ipeds_completion
+    .join(
+        ipeds_2013.select("unitid", "years_ipeds_2013"),
+        on="unitid",
+        how="left",
+        validate="1:1",
+    )
+    .join(
+        ipeds_manual.select("unitid", "years_manual"),
+        on="unitid",
+        how="left",
+        validate="1:1",
+    )
+    .join(
+        directory.select("unitid", "years_directory"),
+        on="unitid",
+        how="left",
+        validate="1:1",
+    )
+    .with_columns(
+        pl.coalesce(
+            [
+                pl.col("years_manual"),
+                pl.col("years_ipeds_2013"),
+                pl.col("years_directory"),
+            ]
+        ).alias("college_years")
+    )
+    .filter(
+        pl.col("college_years").is_in([1, 2, 4])
+        & (pl.col("cohort_adj_150pct_ip") > 0)
+        & pl.col("completers_150pct_ip").is_not_null()
+    )
+    .with_columns(
+        pl.when(pl.col("college_years") == 4)
+        .then(pl.lit("4yr"))
+        .otherwise(pl.lit("2yr_or_less"))
+        .alias("college_sector")
+    )
+    .group_by("college_sector")
+    .agg(
+        pl.col("completers_150pct_ip").sum().alias("completers"),
+        pl.col("cohort_adj_150pct_ip").sum().alias("cohort"),
+    )
+    .with_columns((pl.col("completers") / pl.col("cohort")).alias("cmp_rate_coarse"))
+)
+
+cmp_rate_4yr_coarse = ipeds_sector_rates.filter(
+    pl.col("college_sector") == "4yr"
+).item(0, "cmp_rate_coarse")
+cmp_rate_2yr_coarse = ipeds_sector_rates.filter(
+    pl.col("college_sector") == "2yr_or_less"
+).item(0, "cmp_rate_coarse")
+
+for label, value in {
+    "four-year k_mean": k_mean_4yr_coarse,
+    "two-year-or-less k_mean": k_mean_2yr_coarse,
+}.items():
+    if value is None or value <= 0:
+        raise ValueError(f"Invalid national student-weighted {label}: {value}")
+
+for label, value in {
+    "four-year completion rate": cmp_rate_4yr_coarse,
+    "two-year-or-less completion rate": cmp_rate_2yr_coarse,
+}.items():
+    if value is None or not 0 <= value <= 1:
+        raise ValueError(f"Invalid national student-weighted {label}: {value}")
+
 
 ###########################################################
 # Build college reference data
@@ -331,7 +443,7 @@ college_ref = (
     )
     .with_columns(
         (pl.col("college_years") == 4).cast(pl.Int8).alias("college_4yr"),
-        (pl.col("college_years") == 2).cast(pl.Int8).alias("college_2yr"),
+        pl.col("college_years").is_in([1, 2]).cast(pl.Int8).alias("college_2yr"),
         pl.col("tier").is_in([1, 2]).cast(pl.Int8).alias("college_elite"),
     )
     .with_columns(
@@ -365,20 +477,27 @@ college_ref = (
 # the year the student turns 18 and anchors Sarah's Y1/Y2/... outcome windows.
 
 
+apps_scan = pl.scan_csv(APPS)
+app_columns = apps_scan.collect_schema().names()
+app_select = ["sid_cepr", "cohort_lottery"]
+if "offer" in app_columns:
+    app_select.append("offer")
+
 apps = (
-    scan_readstat(
-        APPS, missing_string_as_null=True, value_labels_as_strings=True
-    )
-    .select("sid_cepr", "cohort_lottery")
+    apps_scan
+    .select(app_select)
     .drop_nulls(["sid_cepr", "cohort_lottery"])
     .with_columns(
         pl.col("sid_cepr").cast(pl.Int64),
         pl.col("cohort_lottery").cast(pl.Int64),
+        pl.col("offer").cast(pl.Int8, strict=False)
+        if "offer" in app_columns
+        else pl.lit(None).cast(pl.Int8).alias("offer"),
     )
     .filter(
         pl.col("cohort_lottery").is_between(1990, 2005)
     )
-    .sort("cohort_lottery")
+    .sort(["cohort_lottery", "offer"], nulls_last=True)
     .unique(["sid_cepr"])
     .with_columns((pl.col("cohort_lottery") + 18).alias("cohort_18"))
     .collect()
@@ -919,11 +1038,12 @@ for column in enrollment_outcomes.columns:
             .alias(column)
         )
 
-# First post-high-school institution is useful for auditing the match.
+# First post-high-school institution is useful for auditing the match. Sarah's
+# final Stata sort keeps the highest ID_FSC when enrollment start dates tie.
 first_institution = (
     enroll.with_columns(pl.date(pl.col("cohort_18"), 7, 1).alias("_hs_grad_start"))
     .filter(pl.col("term_start_date") >= pl.col("_hs_grad_start"))
-    .sort(["sid_cepr", "term_start_date", "ID_FSC"])
+    .sort(["sid_cepr", "term_start_date", "ID_FSC"], descending=[False, False, True])
     .group_by("sid_cepr", maintain_order=True)
     .agg(
         pl.col("ID_FSC").first().alias("ID_FSC_firstinst"),
@@ -931,18 +1051,33 @@ first_institution = (
         .first()
         .alias("college_name_firstinst"),
         pl.col("college_years").first().alias("college_years_firstinst"),
-        pl.col("k_mean").first().alias("k_mean_firstinst"),
         pl.col("completion_rate_150pct_ip")
         .first()
         .alias("completion_rate_150pct_firstinst"),
     )
+    .with_columns(
+        pl.col("ID_FSC_firstinst")
+        .str.slice(0, 6)
+        .cast(pl.Int64, strict=False)
+        .alias("opeid_firstinst")
+    )
+    .join(
+        chetty_by_opeid.select("opeid", "k_mean", "tier"),
+        left_on="opeid_firstinst",
+        right_on="opeid",
+        how="left",
+        validate="m:1",
+    )
+    .rename({"k_mean": "k_mean_firstinst", "tier": "tier_firstinst"})
 )
 
+# Carry Sarah's raw IPEDS completion-rate variable from the first institution.
+# Nonattenders remain missing here; Sarah handles the adjusted version later.
 first_institution = first_institution.with_columns(
     pl.col("k_mean_firstinst")
     .fill_null(k_mean_insuffdata)
     .alias("k_mean"),
-    pl.col("completion_rate_150pct_firstinst").alias("predicted_completion"),
+    pl.col("completion_rate_150pct_firstinst").alias("completion_rate_150pct_ip"),
 )
 
 enrollment_outcomes = enrollment_outcomes.join(
@@ -1347,9 +1482,48 @@ nsc_outcomes = (
         .otherwise(pl.col("k_mean"))
         .alias("k_mean"),
         pl.when(pl.col("ID_FSC_firstinst").is_null())
-        .then(0.0)
-        .otherwise(pl.col("predicted_completion"))
-        .alias("predicted_completion"),
+        .then(k_mean_neverattend)
+        .when(pl.col("college_years_firstinst") == 4)
+        .then(k_mean_4yr_coarse)
+        .when(pl.col("college_years_firstinst").is_in([1, 2]))
+        .then(k_mean_2yr_coarse)
+        .otherwise(k_mean_insuffdata)
+        .alias("k_mean_coarse"),
+    )
+)
+
+# Sarah's later cleanup takes medians over the student-level analysis file by
+# tier. Apply the same residual fill only when an attendee is still missing.
+tier_medians = (
+    nsc_outcomes.group_by("tier_firstinst")
+    .agg(
+        pl.col("k_mean").median().alias("k_mean_tier_median"),
+        pl.col("completion_rate_150pct_ip")
+        .median()
+        .alias("completion_rate_150pct_tier_median"),
+    )
+    .drop_nulls("tier_firstinst")
+)
+
+nsc_outcomes = (
+    nsc_outcomes.join(
+        tier_medians,
+        on="tier_firstinst",
+        how="left",
+        validate="m:1",
+    )
+    .with_columns(
+        pl.when(pl.col("k_mean").is_null() & (pl.col("att_any_byY2") == 1))
+        .then(pl.col("k_mean_tier_median"))
+        .otherwise(pl.col("k_mean"))
+        .alias("k_mean"),
+        pl.when(
+            pl.col("completion_rate_150pct_ip").is_null()
+            & (pl.col("att_any_byY2") == 1)
+        )
+        .then(pl.col("completion_rate_150pct_tier_median"))
+        .otherwise(pl.col("completion_rate_150pct_ip"))
+        .alias("completion_rate_150pct_ip"),
     )
 )
 
@@ -1412,6 +1586,89 @@ for year in range(1, N_YEARS_OUT + 1):
     )
 
 
+# Sarah removes the raw school-specific rate for students who were not observed
+# attending by Y2 before creating cmp_rate.
+nsc_outcomes = nsc_outcomes.with_columns(
+    pl.when(pl.col("att_any_byY2") == 0)
+    .then(None)
+    .otherwise(pl.col("completion_rate_150pct_ip"))
+    .alias("completion_rate_150pct_ip")
+)
+
+# Sarah creates cmp_rate as a copy of the restricted raw completion rate, then
+# fills missing cmp_rate values with the late-enrollee/control-group mean.
+nsc_outcomes = nsc_outcomes.with_columns(
+    pl.col("completion_rate_150pct_ip").alias("cmp_rate"),
+    pl.when(pl.col("att_any_byY2") == 0)
+    .then(None)
+    .when(pl.col("college_years_firstinst") == 4)
+    .then(cmp_rate_4yr_coarse)
+    .when(pl.col("college_years_firstinst").is_in([1, 2]))
+    .then(cmp_rate_2yr_coarse)
+    .otherwise(None)
+    .alias("cmp_rate_coarse"),
+)
+
+late_enrollee_means = (
+    nsc_outcomes
+    .filter(
+        (pl.col("att_4yr_inY2") == 0)
+        & (pl.col("att_4yr_inY3") == 1)
+        & (
+            (pl.col("cmp_BA_byY6") == 1)
+            | (pl.col("cmp_BA_byY7") == 1)
+            | (pl.col("cmp_BA_byY8") == 1)
+        )
+        & (pl.col("offer") == 0)
+    )
+    .select(
+        pl.col("cmp_rate").mean().alias("cmp_rate"),
+        pl.col("cmp_rate_coarse").mean().alias("cmp_rate_coarse"),
+    )
+)
+
+late_enrollee_mean = late_enrollee_means.item(0, "cmp_rate")
+late_enrollee_coarse_mean = late_enrollee_means.item(0, "cmp_rate_coarse")
+
+if late_enrollee_mean is None or late_enrollee_coarse_mean is None:
+    raise ValueError(
+        "The late-enrollee control donor group has no usable completion-rate values."
+    )
+
+nsc_outcomes = nsc_outcomes.with_columns(
+    pl.when(pl.col("cmp_rate").is_null())
+    .then(pl.lit(late_enrollee_mean, dtype=pl.Float64))
+    .otherwise(pl.col("cmp_rate"))
+    .alias("adj_cmp_rate"),
+    pl.when(pl.col("cmp_rate_coarse").is_null())
+    .then(pl.lit(late_enrollee_coarse_mean, dtype=pl.Float64))
+    .otherwise(pl.col("cmp_rate_coarse"))
+    .alias("adj_cmp_rate_coarse"),
+).with_columns(
+    pl.when(pl.col("att_4yr_byY2").is_null())
+    .then(None)
+    .otherwise(pl.col("cmp_rate"))
+    .alias("cmp_rate"),
+    pl.when(pl.col("att_4yr_byY2").is_null())
+    .then(None)
+    .otherwise(pl.col("adj_cmp_rate"))
+    .alias("adj_cmp_rate"),
+    pl.when(pl.col("att_4yr_byY2").is_null())
+    .then(None)
+    .otherwise(pl.col("cmp_rate_coarse"))
+    .alias("cmp_rate_coarse"),
+    pl.when(pl.col("att_4yr_byY2").is_null())
+    .then(None)
+    .otherwise(pl.col("adj_cmp_rate_coarse"))
+    .alias("adj_cmp_rate_coarse"),
+    # Preserve the package's existing name as an alias for Sarah's adjusted rate.
+    pl.when(pl.col("att_4yr_byY2").is_null())
+    .then(None)
+    .otherwise(pl.col("adj_cmp_rate"))
+    .alias("predicted_completion"),
+)
+
+
 ###########################################################
 # Write final outcomes
 ###########################################################
@@ -1443,9 +1700,16 @@ nsc_outcomes = nsc_outcomes.with_columns(
 keep_columns = [
     "sid_cepr",
     "k_mean",
+    "k_mean_coarse",
+    "completion_rate_150pct_ip",
+    "cmp_rate",
+    "adj_cmp_rate",
+    "cmp_rate_coarse",
+    "adj_cmp_rate_coarse",
     "predicted_completion",
     "ID_FSC_firstinst",
     "college_name_firstinst",
+    "tier_firstinst",
     "completion_rate_150pct_firstinst",
 ] + outcome_columns
 
@@ -1479,6 +1743,38 @@ print(
     f"{first_college_coverage['has_completion_rate'].sum()}/"
     f"{first_college_coverage.height} with IPEDS completion rate"
 )
+print(
+    "National student-weighted coarse values: "
+    f"k_mean 4yr={k_mean_4yr_coarse:.2f}, "
+    f"k_mean 2yr-or-less={k_mean_2yr_coarse:.2f}, "
+    f"completion 4yr={cmp_rate_4yr_coarse:.4f}, "
+    f"completion 2yr-or-less={cmp_rate_2yr_coarse:.4f}, "
+    f"nonattender completion imputation={late_enrollee_mean:.4f}, "
+    f"coarse nonattender imputation={late_enrollee_coarse_mean:.4f}"
+)
+
+observable_y2 = nsc_outcomes.filter(pl.col("att_4yr_byY2").is_not_null())
+coarse_audit = observable_y2.select(
+    pl.len().alias("students"),
+    pl.col("k_mean_coarse").null_count().alias("k_mean_coarse_missing"),
+    pl.col("k_mean_coarse").min().alias("k_mean_coarse_min"),
+    pl.col("k_mean_coarse").max().alias("k_mean_coarse_max"),
+    pl.col("adj_cmp_rate_coarse")
+    .null_count()
+    .alias("adj_cmp_rate_coarse_missing"),
+    pl.col("adj_cmp_rate_coarse").min().alias("adj_cmp_rate_coarse_min"),
+    pl.col("adj_cmp_rate_coarse").max().alias("adj_cmp_rate_coarse_max"),
+)
+
+if coarse_audit.item(0, "k_mean_coarse_missing") > 0:
+    raise ValueError("k_mean_coarse is unexpectedly missing for observable students.")
+if coarse_audit.item(0, "adj_cmp_rate_coarse_missing") > 0:
+    raise ValueError(
+        "adj_cmp_rate_coarse is unexpectedly missing for observable students."
+    )
+
+print("Coarse outcome audit:")
+print(coarse_audit)
 print(
     nsc_outcomes.select("sid_cepr", "cohort_lottery", "recovered_nsc_outcome").sort(
         "sid_cepr"

ltc_code-0.2.21/src/ltc_code/signal_var_calcs.py

@@ -0,0 +1,41 @@
+# --- Import necessary packages ---
+import polars as pl
+
+
+SIGNAL_DEP_VARS = ["wages25", "college20"]
+
+
+def calculate_signal_variance(results: pl.DataFrame) -> pl.DataFrame:
+    """Calculate signal variance and its implied 90-10 spread by outcome."""
+    return (
+        results
+        .filter(pl.col("depvar").is_in(SIGNAL_DEP_VARS))
+        .drop_nulls(["sample", "depvar", "coef", "se"])
+        .group_by("depvar")
+        .agg(
+            pl.col("sample").n_unique().alias("n_cmos"),
+            pl.col("coef").mean().alias("mean_te"),
+            pl.col("coef").var().alias("raw_variance"),
+            (pl.col("se") ** 2).mean().alias("noise_variance"),
+        )
+        .with_columns(
+            (
+                pl.col("raw_variance")
+                - pl.col("noise_variance")
+            ).alias("signal_variance_raw")
+        )
+        # Retain the raw estimate, but use a nonnegative value for the spread.
+        .with_columns(
+            pl.col("signal_variance_raw")
+            .clip(lower_bound=0)
+            .alias("signal_variance")
+        )
+        .with_columns(
+            pl.col("signal_variance").sqrt().alias("signal_sd")
+        )
+        # Under normality, P90 - P10 equals 2.563103 standard deviations.
+        .with_columns(
+            (2.563103 * pl.col("signal_sd")).alias("spread_90_10")
+        )
+        .sort("depvar")
+    )