ltc-code 0.2.18__tar.gz → 0.2.20__tar.gz

{ltc_code-0.2.18 → ltc_code-0.2.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ltc-code
-Version: 0.2.18
+Version: 0.2.20
 Summary: Add your description here
 Requires-Dist: fastexcel>=0.16,<0.20
 Requires-Dist: polars>=1.36.1,<1.42

{ltc_code-0.2.18 → ltc_code-0.2.20}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.2.18"
+version = "0.2.20"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -12,6 +12,7 @@ dependencies = [
 
 [project.scripts]
 ltc-code = "ltc_code:main"
+ltc-summary-stats = "ltc_code.make_summary_stats_table:main"
 
 [build-system]
 requires = ["uv_build>=0.11.16,<0.12.0"]

ltc_code-0.2.20/src/ltc_code/make_summary_stats_table.py

@@ -0,0 +1,235 @@
+"""Create a paper-ready two-panel summary-statistics table from a CSV."""
+
+from __future__ import annotations
+
+import argparse
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Optional, Union
+
+import polars as pl
+
+
+REQUIRED_COLUMNS = ["sample", "Variable", "Mean", "P50", "SD"]
+COUNT_COLUMNS = ["N", "N_in_group"]
+
+
+def clean_latex(value: object) -> str:
+    text = "" if value is None else str(value)
+    for old, new in {
+        "\\": r"\textbackslash{}",
+        "&": r"\&",
+        "%": r"\%",
+        "$": r"\$",
+        "#": r"\#",
+        "_": r"\_",
+        "{": r"\{",
+        "}": r"\}",
+        "~": r"\textasciitilde{}",
+        "^": r"\textasciicircum{}",
+    }.items():
+        text = text.replace(old, new)
+    return text
+
+
+def format_cell(column: str, value: object) -> str:
+    if value is None:
+        return ""
+    if column in COUNT_COLUMNS:
+        try:
+            return f"{float(value):,.0f}"
+        except (TypeError, ValueError):
+            return clean_latex(value)
+    try:
+        return f"{float(value):,.2f}"
+    except (TypeError, ValueError):
+        return clean_latex(value)
+
+
+def standardize_columns(stats: pl.DataFrame) -> pl.DataFrame:
+    rename = {}
+    aliases = {
+        "sample": "sample",
+        "samples": "sample",
+        "variable": "Variable",
+        "var": "Variable",
+        "name": "Variable",
+        "mean": "Mean",
+        "avg": "Mean",
+        "average": "Mean",
+        "p50": "P50",
+        "median": "P50",
+        "med": "P50",
+        "sd": "SD",
+        "std": "SD",
+        "stddev": "SD",
+        "standard deviation": "SD",
+        "n": "N",
+        "obs": "N",
+        "observations": "N",
+        "n_in_group": "N_in_group",
+        "n group": "N_in_group",
+        "group n": "N_in_group",
+        "groups": "N_in_group",
+        "unique count": "N_in_group",
+        "unique counts": "N_in_group",
+    }
+    for column in stats.columns:
+        target = aliases.get(column.strip().lower())
+        if target is not None:
+            rename[column] = target
+    return stats.rename(rename)
+
+
+def panel_for(sample: str) -> str:
+    if "Age 20+" in sample:
+        return "A"
+    if "Age 25+" in sample:
+        return "B"
+    return "Other"
+
+
+def sample_label(sample: str) -> str:
+    return sample.replace(" Age 20+", "").replace(" Age 25+", "")
+
+
+def make_tex(
+    stats: pl.DataFrame,
+    *,
+    caption: str = "Summary Statistics by Sample",
+    label: str = "tab:summary_stats_by_sample",
+    note: str = "Notes: Means, medians, and standard deviations are reported by analysis sample.",
+) -> str:
+    numeric_columns = [
+        column for column in ["Mean", "P50", "SD", "N", "N_in_group"] if column in stats.columns
+    ]
+    header_labels = {"Mean": "Mean", "P50": "Median", "SD": "SD", "N": "N", "N_in_group": "Group N"}
+    panel_width = len(numeric_columns)
+    num_columns = 1 + 2 * panel_width
+    headers = ["Variable"] + [header_labels[column] for column in numeric_columns] * 2
+    column_spec = (
+        r"@{}p{1.35in}"
+        + "r" * panel_width
+        + r"@{\hspace{0.22in}}"
+        + "r" * panel_width
+        + "@{}"
+    )
+
+    lines = [
+        r"\documentclass[11pt]{article}",
+        r"\usepackage[margin=0.25in]{geometry}",
+        r"\usepackage{booktabs}",
+        r"\usepackage{caption}",
+        r"\usepackage{longtable}",
+        r"\usepackage{pdflscape}",
+        r"\usepackage[table]{xcolor}",
+        r"\begin{document}",
+        r"\begin{landscape}",
+        r"\scriptsize",
+        r"\setlength{\tabcolsep}{3.5pt}",
+        r"\renewcommand{\arraystretch}{1.08}",
+        rf"\begin{{longtable}}{{{column_spec}}}",
+        rf"\caption{{{clean_latex(caption)}}}\label{{{label}}}\\",
+        r"\toprule",
+        rf" & \multicolumn{{{panel_width}}}{{c}}{{\textbf{{Panel A: Age 20+}}}} & \multicolumn{{{panel_width}}}{{c}}{{\textbf{{Panel B: Age 25+}}}} \\",
+        rf"\cmidrule(lr){{2-{1 + panel_width}}}\cmidrule(l){{{2 + panel_width}-{num_columns}}}",
+        " & ".join(clean_latex(column) for column in headers) + r" \\",
+        r"\midrule",
+        r"\endfirsthead",
+        rf"\caption*{{{clean_latex(caption)} (continued)}}\\",
+        r"\toprule",
+        rf" & \multicolumn{{{panel_width}}}{{c}}{{\textbf{{Panel A: Age 20+}}}} & \multicolumn{{{panel_width}}}{{c}}{{\textbf{{Panel B: Age 25+}}}} \\",
+        rf"\cmidrule(lr){{2-{1 + panel_width}}}\cmidrule(l){{{2 + panel_width}-{num_columns}}}",
+        " & ".join(clean_latex(column) for column in headers) + r" \\",
+        r"\midrule",
+        r"\endhead",
+        r"\midrule",
+        rf"\multicolumn{{{num_columns}}}{{r}}{{Continued on next page}} \\",
+        r"\endfoot",
+        r"\bottomrule",
+        r"\endlastfoot",
+    ]
+
+    groups: list[str] = []
+    variables: dict[str, list[str]] = {}
+    values: dict[tuple[str, str, str], dict[str, object]] = {}
+    for row in stats.iter_rows(named=True):
+        group = sample_label(str(row["sample"]))
+        panel = panel_for(str(row["sample"]))
+        variable = str(row["Variable"])
+        if group not in groups:
+            groups.append(group)
+        variables.setdefault(group, [])
+        if variable not in variables[group]:
+            variables[group].append(variable)
+        values[(panel, group, variable)] = row
+
+    for group_index, group in enumerate(groups):
+        if group_index:
+            lines.append(r"\addlinespace[0.25em]")
+        lines.append(
+            rf"\rowcolor{{black!8}}\multicolumn{{{num_columns}}}{{@{{}}l@{{}}}}{{\textbf{{{clean_latex(group)}}}}} \\"
+        )
+        for variable in variables[group]:
+            cells = [clean_latex(variable)]
+            for panel in ["A", "B"]:
+                row = values.get((panel, group, variable))
+                cells.extend(format_cell(column, row[column]) if row is not None else "" for column in numeric_columns)
+            lines.append(" & ".join(cells) + r" \\")
+
+    lines.extend([
+        r"\end{longtable}",
+        rf"\noindent\footnotesize {clean_latex(note)}",
+        r"\end{landscape}",
+        r"\end{document}",
+    ])
+    return "\n".join(lines) + "\n"
+
+
+def build_summary_stats_table(
+    input_csv: Union[str, Path],
+    tex_out: Union[str, Path],
+    pdf_out: Optional[Union[str, Path]] = None,
+    *,
+    caption: str = "Summary Statistics by Sample",
+    label: str = "tab:summary_stats_by_sample",
+    note: str = "Notes: Means, medians, and standard deviations are reported by analysis sample.",
+    compile_pdf: bool = True,
+) -> Optional[Path]:
+    stats = standardize_columns(pl.read_csv(input_csv))
+    missing = [column for column in REQUIRED_COLUMNS if column not in stats.columns]
+    if missing:
+        raise ValueError(f"Missing required columns: {', '.join(missing)}")
+    display_columns = REQUIRED_COLUMNS + [column for column in COUNT_COLUMNS if column in stats.columns]
+    stats = stats.select(display_columns)
+
+    tex_path = Path(tex_out)
+    tex_path.parent.mkdir(parents=True, exist_ok=True)
+    tex_path.write_text(make_tex(stats, caption=caption, label=label, note=note))
+
+    if pdf_out is None or not compile_pdf:
+        return None
+    engine = shutil.which("tectonic") or shutil.which("pdflatex")
+    if engine is None:
+        return None
+    pdf_path = Path(pdf_out)
+    pdf_path.parent.mkdir(parents=True, exist_ok=True)
+    command = [engine, tex_path.name]
+    if Path(engine).name == "pdflatex":
+        command.insert(1, "-interaction=nonstopmode")
+    subprocess.run(command, cwd=tex_path.parent, check=True)
+    return pdf_path
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("input_csv", type=Path)
+    parser.add_argument("--tex-out", type=Path, required=True)
+    parser.add_argument("--pdf-out", type=Path)
+    args = parser.parse_args()
+    build_summary_stats_table(args.input_csv, args.tex_out, args.pdf_out)
+
+
+if __name__ == "__main__":
+    main()

{ltc_code-0.2.18 → ltc_code-0.2.20}/src/ltc_code/nsc/build_nsc_outcomes.py

@@ -247,7 +247,7 @@ chetty_college_outcomes = (
 )
 
 k_mean_neverattend = chetty_college_outcomes.filter(
-    pl.col("super_opeid") == -99
+    pl.col("super_opeid") == -9
 ).item(0, "k_mean")
 k_mean_insuffdata = chetty_college_outcomes.filter(
     pl.col("super_opeid") == -1
@@ -365,20 +365,27 @@ college_ref = (
 # the year the student turns 18 and anchors Sarah's Y1/Y2/... outcome windows.
 
 
+apps_scan = pl.scan_csv(APPS)
+app_columns = apps_scan.collect_schema().names()
+app_select = ["sid_cepr", "cohort_lottery"]
+if "offer" in app_columns:
+    app_select.append("offer")
+
 apps = (
-    scan_readstat(
-        APPS, missing_string_as_null=True, value_labels_as_strings=True
-    )
-    .select("sid_cepr", "cohort_lottery")
+    apps_scan
+    .select(app_select)
     .drop_nulls(["sid_cepr", "cohort_lottery"])
     .with_columns(
         pl.col("sid_cepr").cast(pl.Int64),
         pl.col("cohort_lottery").cast(pl.Int64),
+        pl.col("offer").cast(pl.Int8, strict=False)
+        if "offer" in app_columns
+        else pl.lit(None).cast(pl.Int8).alias("offer"),
     )
     .filter(
         pl.col("cohort_lottery").is_between(1990, 2005)
     )
-    .sort("cohort_lottery")
+    .sort(["cohort_lottery", "offer"], nulls_last=True)
     .unique(["sid_cepr"])
     .with_columns((pl.col("cohort_lottery") + 18).alias("cohort_18"))
     .collect()
@@ -919,11 +926,12 @@ for column in enrollment_outcomes.columns:
             .alias(column)
         )
 
-# First post-high-school institution is useful for auditing the match.
+# First post-high-school institution is useful for auditing the match. Sarah's
+# final Stata sort keeps the highest ID_FSC when enrollment start dates tie.
 first_institution = (
     enroll.with_columns(pl.date(pl.col("cohort_18"), 7, 1).alias("_hs_grad_start"))
     .filter(pl.col("term_start_date") >= pl.col("_hs_grad_start"))
-    .sort(["sid_cepr", "term_start_date", "ID_FSC"])
+    .sort(["sid_cepr", "term_start_date", "ID_FSC"], descending=[False, False, True])
     .group_by("sid_cepr", maintain_order=True)
     .agg(
         pl.col("ID_FSC").first().alias("ID_FSC_firstinst"),
@@ -936,13 +944,29 @@ first_institution = (
         .first()
         .alias("completion_rate_150pct_firstinst"),
     )
+    .with_columns(
+        pl.col("ID_FSC_firstinst")
+        .str.slice(0, 6)
+        .cast(pl.Int64, strict=False)
+        .alias("opeid_firstinst")
+    )
+    .join(
+        chetty_by_opeid.select("opeid", "k_mean", "tier"),
+        left_on="opeid_firstinst",
+        right_on="opeid",
+        how="left",
+        validate="m:1",
+    )
+    .rename({"k_mean": "k_mean_firstinst", "tier": "tier_firstinst"})
 )
 
+# Carry Sarah's raw IPEDS completion-rate variable from the first institution.
+# Nonattenders remain missing here; Sarah handles the adjusted version later.
 first_institution = first_institution.with_columns(
     pl.col("k_mean_firstinst")
     .fill_null(k_mean_insuffdata)
     .alias("k_mean"),
-    pl.col("completion_rate_150pct_firstinst").alias("predicted_completion"),
+    pl.col("completion_rate_150pct_firstinst").alias("completion_rate_150pct_ip"),
 )
 
 enrollment_outcomes = enrollment_outcomes.join(
@@ -1346,10 +1370,41 @@ nsc_outcomes = (
         .then(k_mean_neverattend)
         .otherwise(pl.col("k_mean"))
         .alias("k_mean"),
-        pl.when(pl.col("ID_FSC_firstinst").is_null())
-        .then(0.0)
-        .otherwise(pl.col("predicted_completion"))
-        .alias("predicted_completion"),
+    )
+)
+
+# Sarah's later cleanup takes medians over the student-level analysis file by
+# tier. Apply the same residual fill only when an attendee is still missing.
+tier_medians = (
+    nsc_outcomes.group_by("tier_firstinst")
+    .agg(
+        pl.col("k_mean").median().alias("k_mean_tier_median"),
+        pl.col("completion_rate_150pct_ip")
+        .median()
+        .alias("completion_rate_150pct_tier_median"),
+    )
+    .drop_nulls("tier_firstinst")
+)
+
+nsc_outcomes = (
+    nsc_outcomes.join(
+        tier_medians,
+        on="tier_firstinst",
+        how="left",
+        validate="m:1",
+    )
+    .with_columns(
+        pl.when(pl.col("k_mean").is_null() & (pl.col("att_any_byY2") == 1))
+        .then(pl.col("k_mean_tier_median"))
+        .otherwise(pl.col("k_mean"))
+        .alias("k_mean"),
+        pl.when(
+            pl.col("completion_rate_150pct_ip").is_null()
+            & (pl.col("att_any_byY2") == 1)
+        )
+        .then(pl.col("completion_rate_150pct_tier_median"))
+        .otherwise(pl.col("completion_rate_150pct_ip"))
+        .alias("completion_rate_150pct_ip"),
     )
 )
 
@@ -1412,6 +1467,59 @@ for year in range(1, N_YEARS_OUT + 1):
     )
 
 
+# Sarah removes the raw school-specific rate for students who were not observed
+# attending by Y2 before creating cmp_rate.
+nsc_outcomes = nsc_outcomes.with_columns(
+    pl.when(pl.col("att_any_byY2") == 0)
+    .then(None)
+    .otherwise(pl.col("completion_rate_150pct_ip"))
+    .alias("completion_rate_150pct_ip")
+)
+
+# Sarah creates cmp_rate as a copy of the restricted raw completion rate, then
+# fills missing cmp_rate values with the late-enrollee/control-group mean.
+nsc_outcomes = nsc_outcomes.with_columns(
+    pl.col("completion_rate_150pct_ip").alias("cmp_rate")
+)
+
+late_enrollee_mean = (
+    nsc_outcomes
+    .filter(
+        (pl.col("att_4yr_inY2") == 0)
+        & (pl.col("att_4yr_inY3") == 1)
+        & (
+            (pl.col("cmp_BA_byY6") == 1)
+            | (pl.col("cmp_BA_byY7") == 1)
+            | (pl.col("cmp_BA_byY8") == 1)
+        )
+        & (pl.col("offer") == 0)
+    )
+    .select(pl.col("cmp_rate").mean())
+    .item()
+)
+
+nsc_outcomes = nsc_outcomes.with_columns(
+    pl.when(pl.col("cmp_rate").is_null())
+    .then(pl.lit(late_enrollee_mean, dtype=pl.Float64))
+    .otherwise(pl.col("cmp_rate"))
+    .alias("adj_cmp_rate")
+).with_columns(
+    pl.when(pl.col("att_4yr_byY2").is_null())
+    .then(None)
+    .otherwise(pl.col("cmp_rate"))
+    .alias("cmp_rate"),
+    pl.when(pl.col("att_4yr_byY2").is_null())
+    .then(None)
+    .otherwise(pl.col("adj_cmp_rate"))
+    .alias("adj_cmp_rate"),
+    # Preserve the package's existing name as an alias for Sarah's adjusted rate.
+    pl.when(pl.col("att_4yr_byY2").is_null())
+    .then(None)
+    .otherwise(pl.col("adj_cmp_rate"))
+    .alias("predicted_completion"),
+)
+
+
 ###########################################################
 # Write final outcomes
 ###########################################################
@@ -1443,9 +1551,13 @@ nsc_outcomes = nsc_outcomes.with_columns(
 keep_columns = [
     "sid_cepr",
     "k_mean",
+    "completion_rate_150pct_ip",
+    "cmp_rate",
+    "adj_cmp_rate",
     "predicted_completion",
     "ID_FSC_firstinst",
     "college_name_firstinst",
+    "tier_firstinst",
     "completion_rate_150pct_firstinst",
 ] + outcome_columns
 

ltc_code-0.2.20/src/ltc_code/school_name_xwalk/__init__.py

@@ -0,0 +1,3 @@
+from ltc_code.school_name_xwalk.merge_school_ccd import add_school_ccd
+
+__all__ = ["add_school_ccd"]