ltc-code 0.2.29__tar.gz → 0.2.31__tar.gz

{ltc_code-0.2.29 → ltc_code-0.2.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ltc-code
-Version: 0.2.29
+Version: 0.2.31
 Summary: Add your description here
 Requires-Dist: fastexcel>=0.16,<0.20
 Requires-Dist: polars>=1.36.1,<1.42

{ltc_code-0.2.29 → ltc_code-0.2.31}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.2.29"
+version = "0.2.31"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"

{ltc_code-0.2.29 → ltc_code-0.2.31}/src/ltc_code/nsc/NEW_CROSSWALK.md

@@ -21,4 +21,6 @@ Required files in `nsc/raw/`:
 
 The old IPEDS_IC_manual.xlsx is not needed by this variant. Student inputs remain input/apps.csv, input/nsc_records_old.dta, and input/nsc_records_new.csv. Do not supply synthetic records for actual analysis. Institutional support files are bundled in the locally built wheel and source distribution but are Git-ignored. GitHub Actions publishes these verified distributions from release assets; it does not rebuild from a checkout that lacks the data. Student inputs are never bundled.
 
-Institution-only audit CSVs are written alongside the outcomes, including mapping choices, appended candidates, missing UNITIDs, and missing classifications. The 99.48% matched subset is an audit denominator, not a filter that drops student records from the build.
+Intermediate diagnostic CSVs and printed audit tables are not generated. Only the full and selected final outcome files are written, each in Parquet and CSV format. The 99.48% matched subset is an audit denominator, not a filter that drops student records from the build.
+
+Version 0.2.30 adds final audit exports and selectivity/BA sensitivity outcomes to both builds; see [OUTCOMES.md](OUTCOMES.md) for names and definitions.

ltc_code-0.2.31/src/ltc_code/nsc/OUTCOMES.md

@@ -0,0 +1,59 @@
+# NSC outcomes added in 0.2.30
+
+Both `build_nsc_outcomes.py` and `build_nsc_outcomes_new.py` provide these fields.
+The new-crosswalk variant remains an explicit separate build.
+
+## Final institution audit fields
+
+`nsc_outcomes_final` retains `ID_FSC_firstinst`, `college_name_firstinst`,
+`unitid_firstinst`, `tier_firstinst`, `college_years_firstinst`, and
+`completion_rate_150pct_firstinst`. Institution years are 4 (four-year),
+2 (two-year), 1 (less than two-year), or null if unknown.
+
+The first institution remains the earliest retained enrollment spell starting
+on or after July 1 of the year the student turns 18, with the existing tie rule.
+It need not meet the attendance-status rule or fall within the by-year-four
+window. Attendance indicators describe qualifying enrollment in their window;
+they do not describe the first institution. A transfer can therefore have a
+two-year first institution and four-year attendance. Prediction formulas and
+first-institution selection have not changed.
+
+The build writes the selected table to `nsc_outcomes_final.parquet` and `.csv`
+alongside the existing full `nsc_outcomes.parquet` and `.csv` outputs. These
+are the only four files written; intermediate diagnostic exports and printed
+audit tables have been removed.
+
+## Selectivity outcomes
+
+Selective means tiers 3–4. The existing `elite` name means highly selective,
+tiers 1–2. These institution groups do not overlap. A student can attend or
+earn degrees at institutions in both groups, so their student indicators
+are not forced to be mutually exclusive.
+
+| Example | Meaning |
+| --- | --- |
+| `att_selective_byY4` | Qualifying attendance at a tier 3–4 institution by year 4 |
+| `att_elite_byY4` | Qualifying attendance at a tier 1–2 institution by year 4 |
+| `cmp_selective_byY8` | AA or BA from a tier 3–4 institution by year 8 |
+| `cmp_elite_byY8` | Existing AA-or-BA measure for tiers 1–2 |
+| `cmp_BA_selective_byY8` | BA from a tier 3–4 institution by year 8 |
+| `cmp_BA_elite_byY8` | BA from a tier 1–2 institution by year 8 |
+| `cmp_BA_noimpute_byY8` | Any BA without institution-type credential imputation |
+| `cmp_BA_selective_noimpute_byY8` | Same sensitivity definition for tiers 3–4 |
+| `cmp_BA_elite_noimpute_byY8` | Same sensitivity definition for tiers 1–2 |
+
+Attendance is available for `inY1`–`inY8`, `byY1`–`byY8`, their `fall` and
+`spring` variants, and calendar ages 18–26 (e.g. `att_selective_20`). Completion
+is available for `byY1`–`byY8`. All these selectivity and sensitivity fields are
+retained in the selected final table. They follow the existing observation
+windows and zero/null rules. Unknown institutional tiers do not count as
+selective. Completion uses the degree-granting institution's tier, not the
+first institution's tier.
+
+The default BA measures retain the existing institution-type imputation.
+The `noimpute` versions use credential-lookup/title evidence before that step:
+an unknown award at a four-year institution alone does not count as a BA.
+Students remain in the sample, and another identified BA can still qualify.
+This does not change institutional IPEDS rates or coarse predictions.
+
+Verification: run `uv run python tests/verify_nsc_outcomes.py` from the repository root with the local institutional support files present. This creates temporary synthetic records and does not require secured student inputs.

{ltc_code-0.2.29 → ltc_code-0.2.31}/src/ltc_code/nsc/build_nsc_outcomes.py

@@ -93,8 +93,8 @@ RANDOM_SEED = 3852804
 SARAH_STEM_CIP_FAMILIES = {"11", "14", "15", "26", "27", "40", "41"}
 DHS_STEM_CIP_FAMILIES = {"14", "26", "27", "40"}
 STEM_DEFINITIONS = ["sarah", "dhs"]
-COLLEGE_TYPES = ["any", "4yr", "2yr", "elite"]
-AGE_ATTENDANCE_TYPES = ["any", "4yr", "2yr"]
+COLLEGE_TYPES = ["any", "4yr", "2yr", "elite", "selective"]
+AGE_ATTENDANCE_TYPES = ["any", "4yr", "2yr", "elite", "selective"]
 AGE_ATTENDANCE_RANGE = range(18, 27)
 
 # These ordered rules exactly reproduce Sarah's Stata replacements. Every rule
@@ -573,7 +573,9 @@ college_ref = (
     .with_columns(
         (pl.col("college_years") == 4).cast(pl.Int8).alias("college_4yr"),
         pl.col("college_years").is_in([1, 2]).cast(pl.Int8).alias("college_2yr"),
+        # Institution groups: highly selective (elite) is tiers 1–2; selective is 3–4.
         pl.col("tier").is_in([1, 2]).cast(pl.Int8).alias("college_elite"),
+        pl.col("tier").is_in([3, 4]).cast(pl.Int8).alias("college_selective"),
     )
     .with_columns(
         (pl.col("college_4yr") + pl.col("college_2yr") > 0)
@@ -592,6 +594,7 @@ college_ref = (
         "college_2yr",
         "tier",
         "college_elite",
+        "college_selective",
         "k_mean",
         "completion_rate_150pct_ip",
     )
@@ -1061,6 +1064,7 @@ enroll_weeks = (
         "college_2yr",
         "tier",
         "college_elite",
+        "college_selective",
         "k_mean",
         "completion_rate_150pct_ip",
         "term_start_date",
@@ -1131,6 +1135,7 @@ enroll = (
         pl.col("college_2yr").max(),
         pl.col("tier").drop_nulls().min(),
         pl.col("college_elite").max(),
+        pl.col("college_selective").max(),
         pl.col("k_mean").drop_nulls().first(),
         pl.col("completion_rate_150pct_ip").drop_nulls().first(),
         pl.col("_term_att").max(),
@@ -1453,6 +1458,8 @@ degrees = (
         .otherwise(pl.lit(None))
         .alias("_degree")
     )
+    # Preserve credential/title evidence before filling unknown awards by sector.
+    .with_columns(pl.col("_degree").alias("_degree_no_type_fill"))
     .with_columns(
         pl.when(pl.col("_degree").is_null() & (pl.col("college_years") == 4))
         .then(pl.lit("BA"))
@@ -1504,6 +1511,12 @@ for year in range(1, N_YEARS_OUT + 1):
             (pl.col("_by_end") * (pl.col("_degree") == "BA").cast(pl.Int8))
             .max()
             .alias(f"cmp_BA_byY{year}"),
+            (
+                pl.col("_by_end")
+                * (pl.col("_degree_no_type_fill") == "BA").fill_null(False).cast(pl.Int8)
+            )
+            .max()
+            .alias(f"cmp_BA_noimpute_byY{year}"),
             (pl.col("_by_end") * pl.col("_degree").is_in(["AA", "BA"]).cast(pl.Int8))
             .max()
             .alias(f"cmp_any_byY{year}"),
@@ -1514,6 +1527,28 @@ for year in range(1, N_YEARS_OUT + 1):
             )
             .max()
             .alias(f"cmp_elite_byY{year}"),
+            (
+                pl.col("_by_end")
+                * pl.col("_degree").is_in(["AA", "BA"]).cast(pl.Int8)
+                * pl.col("college_selective").fill_null(0).cast(pl.Int8)
+            )
+            .max()
+            .alias(f"cmp_selective_byY{year}"),
+            # BA-only outcomes support comparisons with any-BA completion.
+            *[
+                (
+                    pl.col("_by_end")
+                    * (pl.col(degree_column) == "BA").fill_null(False).cast(pl.Int8)
+                    * pl.col(f"college_{college_type}").fill_null(0).cast(pl.Int8)
+                )
+                .max()
+                .alias(f"cmp_BA_{college_type}{suffix}_byY{year}")
+                for college_type in ["selective", "elite"]
+                for degree_column, suffix in [
+                    ("_degree", ""),
+                    ("_degree_no_type_fill", "_noimpute"),
+                ]
+            ],
             *[
                 expression
                 for definition in STEM_DEFINITIONS
@@ -1932,7 +1967,6 @@ for label, tiers in tier_buckets.items():
         .otherwise(pl.col("adj_cmp_rate_4yr"))
         .alias(f"adj_cmp_rate_4yr_coarse_{label}")
     )
-    print(f"Four-year IPEDS-cohort-weighted completion rate, {label}: {bucket_rate}")
 
 # Match the available 1098-T calendar years, including the missing 2015 year.
 tax_years = list(range(2011, 2015)) + list(range(2016, 2023))
@@ -2014,6 +2048,16 @@ nsc_outcomes = nsc_outcomes.with_columns(
     .alias("recovered_nsc_outcome")
 )
 
+# Export every selectivity window, plus the BA sensitivity outcomes. Keep the
+# existing recovered-outcome definition above independent of this export list.
+selectivity_outcome_columns = [
+    column for column in nsc_outcomes.columns
+    if column.startswith((
+        "att_selective_", "att_elite_", "cmp_selective_", "cmp_elite_",
+        "cmp_BA_selective_", "cmp_BA_elite_", "cmp_BA_noimpute_",
+    ))
+]
+
 keep_columns = [
     "sid_cepr",
     "k_mean",
@@ -2032,140 +2076,36 @@ keep_columns = [
     "adj_cmp_rate_coarsen_2yr",
     "ID_FSC_firstinst",
     "college_name_firstinst",
+    "unitid_firstinst",
+    "college_years_firstinst",
     "tier_firstinst",
     "completion_rate_150pct_firstinst",
-] + outcome_columns + stem_outcome_columns
+] + outcome_columns + stem_outcome_columns + selectivity_outcome_columns
+keep_columns = list(dict.fromkeys(keep_columns))
 
 nsc_outcomes_final = nsc_outcomes.select(keep_columns)
 
+# Retain the existing completeness check without generating audit tables.
+if nsc_outcomes.filter(
+    pl.col("att_any_byY4").is_not_null() & pl.col("k_mean_coarse").is_null()
+).height > 0:
+    raise ValueError("k_mean_coarse is unexpectedly missing for observable students.")
+
 OUTCOMES.parent.mkdir(parents=True, exist_ok=True)
 nsc_outcomes.write_parquet(OUTCOMES)
 nsc_outcomes.with_columns(
     pl.all().exclude(pl.Date).cast(pl.String, strict=False)
 ).write_csv(OUTCOMES.with_suffix(".csv"))
 
+# Save the selected analysis/audit table as well as the existing full output.
+final_path = OUTCOMES.with_name(f"{OUTCOMES.stem}_final.parquet")
+nsc_outcomes_final.write_parquet(final_path)
+nsc_outcomes_final.with_columns(
+    pl.all().exclude(pl.Date).cast(pl.String, strict=False)
+).write_csv(final_path.with_suffix(".csv"))
+
+print(f"Wrote {final_path}")
+print(f"Wrote {final_path.with_suffix('.csv')}")
 print(f"Wrote {OUTCOMES}")
 print(f"Wrote {OUTCOMES.with_suffix('.csv')}")
 print(f"Rows: {nsc_outcomes.height}, columns: {len(nsc_outcomes.columns)}")
-
-first_college_coverage = (
-    nsc_outcomes.filter(pl.col("ID_FSC_firstinst").is_not_null())
-    .group_by("ID_FSC_firstinst")
-    .agg(
-        pl.col("k_mean_firstinst").is_not_null().any().alias("has_k_mean"),
-        pl.col("completion_rate_150pct_firstinst")
-        .is_not_null()
-        .any()
-        .alias("has_completion_rate"),
-    )
-)
-print(
-    "First-college coverage: "
-    f"{first_college_coverage['has_k_mean'].sum()}/"
-    f"{first_college_coverage.height} with college-specific k_mean; "
-    f"{first_college_coverage['has_completion_rate'].sum()}/"
-    f"{first_college_coverage.height} with IPEDS completion rate"
-)
-print(
-    "National student-weighted coarse values: "
-    f"k_mean 4yr={k_mean_4yr_coarse:.2f}, "
-    f"k_mean 2yr-or-less={k_mean_2yr_coarse:.2f}, "
-    f"completion 4yr={cmp_rate_4yr_coarse:.4f}, "
-    f"completion 2yr-or-less={cmp_rate_2yr_coarse:.4f}; "
-    "observed by-Y4 nonattender completion=0"
-)
-print(
-    "Sample-institution IPEDS-cohort-weighted coarse values: "
-    f"completion 4yr={cmp_rate_4yr_coarse_sample:.4f} "
-    f"({sample_4yr_institutions} institutions), "
-    f"completion 2yr-or-less={cmp_rate_2yr_coarse_sample:.4f} "
-    f"({sample_2yr_institutions} institutions)"
-)
-
-observable_y4 = nsc_outcomes.filter(pl.col("att_any_byY4").is_not_null())
-coarse_audit = observable_y4.select(
-    pl.len().alias("students"),
-    pl.col("k_mean_coarse").null_count().alias("k_mean_coarse_missing"),
-    pl.col("k_mean_coarse").min().alias("k_mean_coarse_min"),
-    pl.col("k_mean_coarse").max().alias("k_mean_coarse_max"),
-    pl.col("adj_cmp_rate_coarse")
-    .null_count()
-    .alias("adj_cmp_rate_coarse_missing"),
-    pl.col("adj_cmp_rate_coarse").min().alias("adj_cmp_rate_coarse_min"),
-    pl.col("adj_cmp_rate_coarse").max().alias("adj_cmp_rate_coarse_max"),
-    pl.col("adj_cmp_rate_coarse_sample")
-    .null_count()
-    .alias("adj_cmp_rate_coarse_sample_missing"),
-    pl.col("adj_cmp_rate_coarse_sample")
-    .min()
-    .alias("adj_cmp_rate_coarse_sample_min"),
-    pl.col("adj_cmp_rate_coarse_sample")
-    .max()
-    .alias("adj_cmp_rate_coarse_sample_max"),
-)
-
-if coarse_audit.item(0, "k_mean_coarse_missing") > 0:
-    raise ValueError("k_mean_coarse is unexpectedly missing for observable students.")
-# Attendees without a classified first college can have missing coarse rates.
-# Report those counts below rather than treating them as a build failure.
-print("Coarse outcome audit:")
-print(coarse_audit)
-
-attendee_completion_audit = (
-    observable_y4.filter(pl.col("att_any_byY4") == 1)
-    .select(
-        pl.len().alias("by_y4_attendees"),
-        pl.col("completion_rate_150pct_firstinst")
-        .is_null()
-        .sum()
-        .alias("missing_first_institution_rate"),
-        (
-            pl.col("completion_rate_150pct_firstinst").is_null()
-            & pl.col("completion_rate_150pct_ip").is_not_null()
-        )
-        .sum()
-        .alias("filled_by_tier_median"),
-        pl.col("adj_cmp_rate")
-        .is_null()
-        .sum()
-        .alias("missing_after_tier_median"),
-        pl.col("adj_cmp_rate")
-        .is_null()
-        .mean()
-        .alias("missing_after_tier_median_share"),
-        (
-            pl.col("adj_cmp_rate").is_null()
-            & pl.col("tier_firstinst").is_null()
-        )
-        .sum()
-        .alias("missing_after_tier_median_no_tier"),
-        (
-            pl.col("adj_cmp_rate").is_null()
-            & pl.col("tier_firstinst").is_not_null()
-        )
-        .sum()
-        .alias("missing_after_tier_median_with_tier"),
-    )
-)
-print("By-Y4 attendee completion-rate audit:")
-print(attendee_completion_audit)
-print(
-    nsc_outcomes.select("sid_cepr", "cohort_lottery", "recovered_nsc_outcome").sort(
-        "sid_cepr"
-    )
-)
-
-# Nonmissing year-eight completion identifies students with the full Y8 window.
-year8_attendance_audit = (
-    nsc_outcomes.filter(pl.col("cmp_BA_byY8").is_not_null())
-    .select(
-        pl.len().alias("N_year8_sample"),
-        (pl.col("att_any_byY4") == 0).sum().alias("N_no_attendance_byY4"),
-        (
-            (pl.col("att_any_byY4") == 0)
-            & (pl.col("cmp_BA_byY8") == 1)
-        ).sum().alias("N_no_attendance_byY4_BA_byY8"),
-    )
-)
-print("Year-eight sample attendance and late BA completion check:")
-print(year8_attendance_audit)

{ltc_code-0.2.29 → ltc_code-0.2.31}/src/ltc_code/nsc/build_nsc_outcomes_new.py

@@ -60,7 +60,6 @@ INPUT_NSC = PACKAGE_NSC / "input"
 # Folder for secured Census inputs; no student records are packaged.
 
 OUTPUT_NSC = PACKAGE_NSC / "output" / "new_crosswalk"
-OUTPUT_NSC.mkdir(parents=True, exist_ok=True)
 # Folder for final student-level NSC outcomes.
 
 APPS = INPUT_NSC / "apps.csv"
@@ -93,8 +92,8 @@ RANDOM_SEED = 3852804
 SARAH_STEM_CIP_FAMILIES = {"11", "14", "15", "26", "27", "40", "41"}
 DHS_STEM_CIP_FAMILIES = {"14", "26", "27", "40"}
 STEM_DEFINITIONS = ["sarah", "dhs"]
-COLLEGE_TYPES = ["any", "4yr", "2yr", "elite"]
-AGE_ATTENDANCE_TYPES = ["any", "4yr", "2yr"]
+COLLEGE_TYPES = ["any", "4yr", "2yr", "elite", "selective"]
+AGE_ATTENDANCE_TYPES = ["any", "4yr", "2yr", "elite", "selective"]
 AGE_ATTENDANCE_RANGE = range(18, 27)
 
 # These ordered rules exactly reproduce Sarah's Stata replacements. Every rule
@@ -249,14 +248,12 @@ legacy_absent_candidates = legacy_raw.join(
     official_crosswalk.select("ID_FSC"), on="ID_FSC", how="anti"
 )
 # Reproducible legacy fallback selection; approved sole-rate exceptions below
-# still take precedence. Save all candidates so ambiguity is reviewable.
-legacy_absent_candidates.write_csv(OUTPUT_NSC / "legacy_absent_candidates.csv")
+# still take precedence.
 legacy_append = legacy_absent_candidates.sort(
     ["ID_FSC", "legacy_unitid", "ID_OPE", "name"], nulls_last=True
 ).unique("ID_FSC", keep="first", maintain_order=True).select(
     "ID_FSC", "legacy_unitid", pl.col("name").alias("college_name_crosswalk")
 )
-legacy_append.write_csv(OUTPUT_NSC / "legacy_appended_codes.csv")
 combined_crosswalk = pl.concat([
     official_crosswalk.with_columns(pl.lit("official").alias("mapping_source")),
     legacy_append.with_columns(pl.lit("legacy_absent_code").alias("mapping_source")),
@@ -300,7 +297,6 @@ college_crosswalk = (
         pl.col("ID_FSC").str.slice(0, 6).cast(pl.Int64, strict=False).alias("opeid"),
     )
 )
-college_crosswalk.write_csv(OUTPUT_NSC / "mapping_decisions.csv")
 
 
 ###########################################################
@@ -616,7 +612,9 @@ college_ref = (
     .with_columns(
         (pl.col("college_years") == 4).cast(pl.Int8).alias("college_4yr"),
         pl.col("college_years").is_in([1, 2]).cast(pl.Int8).alias("college_2yr"),
+        # Institution groups: highly selective (elite) is tiers 1–2; selective is 3–4.
         pl.col("tier").is_in([1, 2]).cast(pl.Int8).alias("college_elite"),
+        pl.col("tier").is_in([3, 4]).cast(pl.Int8).alias("college_selective"),
     )
     .with_columns(
         (pl.col("college_4yr") + pl.col("college_2yr") > 0)
@@ -635,26 +633,13 @@ college_ref = (
         "college_2yr",
         "tier",
         "college_elite",
+        "college_selective",
         "k_mean",
         "completion_rate_150pct_ip",
     )
 )
 
 
-# Institution-universe coverage: code counts, not student counts.
-coverage = college_crosswalk.join(
-    college_ref.select("ID_FSC", "college_years", "completion_rate_150pct_ip"),
-    on="ID_FSC", how="left", validate="1:1",
-)
-coverage.write_csv(OUTPUT_NSC / "institution_coverage.csv")
-coverage.filter(pl.col("unitid").is_null()).write_csv(OUTPUT_NSC / "codes_without_unitid.csv")
-coverage.filter(pl.col("unitid").is_not_null() & pl.col("college_years").is_null()).write_csv(OUTPUT_NSC / "mapped_codes_without_level.csv")
-print("CROSSWALK COVERAGE", coverage.select(
-    pl.len().alias("codes"), pl.col("unitid").is_not_null().sum().alias("mapped"),
-    pl.col("college_years").is_not_null().sum().alias("classified"),
-    pl.col("completion_rate_150pct_ip").is_not_null().sum().alias("with_rate"),
-))
-
 ###########################################################
 # Load student universe and NSC records
 ###########################################################
@@ -1042,6 +1027,7 @@ enroll_weeks = (
         "college_2yr",
         "tier",
         "college_elite",
+        "college_selective",
         "k_mean",
         "completion_rate_150pct_ip",
         "term_start_date",
@@ -1112,6 +1098,7 @@ enroll = (
         pl.col("college_2yr").max(),
         pl.col("tier").drop_nulls().min(),
         pl.col("college_elite").max(),
+        pl.col("college_selective").max(),
         pl.col("k_mean").drop_nulls().first(),
         pl.col("completion_rate_150pct_ip").drop_nulls().first(),
         pl.col("_term_att").max(),
@@ -1306,11 +1293,34 @@ for column in enrollment_outcomes.columns:
             .alias(column)
         )
 
-# First post-high-school institution is useful for auditing the match. Sarah's
-# final Stata sort keeps the highest ID_FSC when enrollment start dates tie.
+# Use exactly the status and window-overlap rules for att_any_byY4. A spell
+# starting before July 1 can count if it continues into the attendance window.
+qualifying_enroll = enroll.with_columns(
+    pl.date(pl.col("cohort_18"), 7, 1).alias("_window_start"),
+    pl.date(pl.col("cohort_18") + 4, 6, 30).alias("_window_end"),
+).filter(
+    (pl.col("_term_att") == 1)
+    & (pl.col("term_end_date") >= pl.col("_window_start"))
+)
+
+# Before excluding late starts, count students whose first observed qualifying
+# enrollment is after Y4. Denominator: observed qualifying enrollees with a
+# fully observable Y4 window; not all applicants or an eventual-enrollment rate.
+first_enrollment_timing = qualifying_enroll.group_by("sid_cepr").agg(
+    pl.col("term_start_date").min().alias("first_start"),
+    pl.col("_window_end").first().alias("y4_end"),
+).filter(pl.col("y4_end") <= NSC_CUTOFF_DATE)
+
+print("FIRST QUALIFYING ENROLLMENT AFTER Y4", first_enrollment_timing.select(
+    pl.len().alias("observed_enrollees_with_Y4_followup"),
+    (pl.col("first_start") > pl.col("y4_end")).sum().alias("first_after_Y4"),
+    (100 * (pl.col("first_start") > pl.col("y4_end")).mean()).alias("percent_after_Y4"),
+))
+
+# Select the earliest qualifying spell overlapping the by-Y4 window. Preserve
+# Sarah's tie-break: highest ID_FSC when enrollment start dates are equal.
 first_institution = (
-    enroll.with_columns(pl.date(pl.col("cohort_18"), 7, 1).alias("_hs_grad_start"))
-    .filter(pl.col("term_start_date") >= pl.col("_hs_grad_start"))
+    qualifying_enroll.filter(pl.col("term_start_date") <= pl.col("_window_end"))
     .sort(["sid_cepr", "term_start_date", "ID_FSC"], descending=[False, False, True])
     .group_by("sid_cepr", maintain_order=True)
     .agg(
@@ -1434,6 +1444,8 @@ degrees = (
         .otherwise(pl.lit(None))
         .alias("_degree")
     )
+    # Preserve credential/title evidence before filling unknown awards by sector.
+    .with_columns(pl.col("_degree").alias("_degree_no_type_fill"))
     .with_columns(
         pl.when(pl.col("_degree").is_null() & (pl.col("college_years") == 4))
         .then(pl.lit("BA"))
@@ -1485,6 +1497,12 @@ for year in range(1, N_YEARS_OUT + 1):
             (pl.col("_by_end") * (pl.col("_degree") == "BA").cast(pl.Int8))
             .max()
             .alias(f"cmp_BA_byY{year}"),
+            (
+                pl.col("_by_end")
+                * (pl.col("_degree_no_type_fill") == "BA").fill_null(False).cast(pl.Int8)
+            )
+            .max()
+            .alias(f"cmp_BA_noimpute_byY{year}"),
             (pl.col("_by_end") * pl.col("_degree").is_in(["AA", "BA"]).cast(pl.Int8))
             .max()
             .alias(f"cmp_any_byY{year}"),
@@ -1495,6 +1513,28 @@ for year in range(1, N_YEARS_OUT + 1):
             )
             .max()
             .alias(f"cmp_elite_byY{year}"),
+            (
+                pl.col("_by_end")
+                * pl.col("_degree").is_in(["AA", "BA"]).cast(pl.Int8)
+                * pl.col("college_selective").fill_null(0).cast(pl.Int8)
+            )
+            .max()
+            .alias(f"cmp_selective_byY{year}"),
+            # BA-only outcomes support comparisons with any-BA completion.
+            *[
+                (
+                    pl.col("_by_end")
+                    * (pl.col(degree_column) == "BA").fill_null(False).cast(pl.Int8)
+                    * pl.col(f"college_{college_type}").fill_null(0).cast(pl.Int8)
+                )
+                .max()
+                .alias(f"cmp_BA_{college_type}{suffix}_byY{year}")
+                for college_type in ["selective", "elite"]
+                for degree_column, suffix in [
+                    ("_degree", ""),
+                    ("_degree_no_type_fill", "_noimpute"),
+                ]
+            ],
             *[
                 expression
                 for definition in STEM_DEFINITIONS
@@ -1923,7 +1963,6 @@ for label, tiers in tier_buckets.items():
         .otherwise(pl.col("adj_cmp_rate_4yr"))
         .alias(f"adj_cmp_rate_4yr_coarse_{label}")
     )
-    print(f"Four-year IPEDS-cohort-weighted completion rate, {label}: {bucket_rate}")
 
 # Match the available 1098-T calendar years, including the missing 2015 year.
 tax_years = list(range(2011, 2015)) + list(range(2016, 2023))
@@ -2005,6 +2044,16 @@ nsc_outcomes = nsc_outcomes.with_columns(
     .alias("recovered_nsc_outcome")
 )
 
+# Export every selectivity window, plus the BA sensitivity outcomes. Keep the
+# existing recovered-outcome definition above independent of this export list.
+selectivity_outcome_columns = [
+    column for column in nsc_outcomes.columns
+    if column.startswith((
+        "att_selective_", "att_elite_", "cmp_selective_", "cmp_elite_",
+        "cmp_BA_selective_", "cmp_BA_elite_", "cmp_BA_noimpute_",
+    ))
+]
+
 keep_columns = [
     "sid_cepr",
     "k_mean",
@@ -2023,140 +2072,36 @@ keep_columns = [
     "adj_cmp_rate_coarsen_2yr",
     "ID_FSC_firstinst",
     "college_name_firstinst",
+    "unitid_firstinst",
+    "college_years_firstinst",
     "tier_firstinst",
     "completion_rate_150pct_firstinst",
-] + outcome_columns + stem_outcome_columns
+] + outcome_columns + stem_outcome_columns + selectivity_outcome_columns
+keep_columns = list(dict.fromkeys(keep_columns))
 
 nsc_outcomes_final = nsc_outcomes.select(keep_columns)
 
+# Retain the existing completeness check without generating audit tables.
+if nsc_outcomes.filter(
+    pl.col("att_any_byY4").is_not_null() & pl.col("k_mean_coarse").is_null()
+).height > 0:
+    raise ValueError("k_mean_coarse is unexpectedly missing for observable students.")
+
 OUTCOMES.parent.mkdir(parents=True, exist_ok=True)
 nsc_outcomes.write_parquet(OUTCOMES)
 nsc_outcomes.with_columns(
     pl.all().exclude(pl.Date).cast(pl.String, strict=False)
 ).write_csv(OUTCOMES.with_suffix(".csv"))
 
+# Save the selected analysis/audit table as well as the existing full output.
+final_path = OUTCOMES.with_name(f"{OUTCOMES.stem}_final.parquet")
+nsc_outcomes_final.write_parquet(final_path)
+nsc_outcomes_final.with_columns(
+    pl.all().exclude(pl.Date).cast(pl.String, strict=False)
+).write_csv(final_path.with_suffix(".csv"))
+
+print(f"Wrote {final_path}")
+print(f"Wrote {final_path.with_suffix('.csv')}")
 print(f"Wrote {OUTCOMES}")
 print(f"Wrote {OUTCOMES.with_suffix('.csv')}")
 print(f"Rows: {nsc_outcomes.height}, columns: {len(nsc_outcomes.columns)}")
-
-first_college_coverage = (
-    nsc_outcomes.filter(pl.col("ID_FSC_firstinst").is_not_null())
-    .group_by("ID_FSC_firstinst")
-    .agg(
-        pl.col("k_mean_firstinst").is_not_null().any().alias("has_k_mean"),
-        pl.col("completion_rate_150pct_firstinst")
-        .is_not_null()
-        .any()
-        .alias("has_completion_rate"),
-    )
-)
-print(
-    "First-college coverage: "
-    f"{first_college_coverage['has_k_mean'].sum()}/"
-    f"{first_college_coverage.height} with college-specific k_mean; "
-    f"{first_college_coverage['has_completion_rate'].sum()}/"
-    f"{first_college_coverage.height} with IPEDS completion rate"
-)
-print(
-    "National student-weighted coarse values: "
-    f"k_mean 4yr={k_mean_4yr_coarse:.2f}, "
-    f"k_mean 2yr-or-less={k_mean_2yr_coarse:.2f}, "
-    f"completion 4yr={cmp_rate_4yr_coarse:.4f}, "
-    f"completion 2yr-or-less={cmp_rate_2yr_coarse:.4f}; "
-    "observed by-Y4 nonattender completion=0"
-)
-print(
-    "Sample-institution IPEDS-cohort-weighted coarse values: "
-    f"completion 4yr={cmp_rate_4yr_coarse_sample:.4f} "
-    f"({sample_4yr_institutions} institutions), "
-    f"completion 2yr-or-less={cmp_rate_2yr_coarse_sample:.4f} "
-    f"({sample_2yr_institutions} institutions)"
-)
-
-observable_y4 = nsc_outcomes.filter(pl.col("att_any_byY4").is_not_null())
-coarse_audit = observable_y4.select(
-    pl.len().alias("students"),
-    pl.col("k_mean_coarse").null_count().alias("k_mean_coarse_missing"),
-    pl.col("k_mean_coarse").min().alias("k_mean_coarse_min"),
-    pl.col("k_mean_coarse").max().alias("k_mean_coarse_max"),
-    pl.col("adj_cmp_rate_coarse")
-    .null_count()
-    .alias("adj_cmp_rate_coarse_missing"),
-    pl.col("adj_cmp_rate_coarse").min().alias("adj_cmp_rate_coarse_min"),
-    pl.col("adj_cmp_rate_coarse").max().alias("adj_cmp_rate_coarse_max"),
-    pl.col("adj_cmp_rate_coarse_sample")
-    .null_count()
-    .alias("adj_cmp_rate_coarse_sample_missing"),
-    pl.col("adj_cmp_rate_coarse_sample")
-    .min()
-    .alias("adj_cmp_rate_coarse_sample_min"),
-    pl.col("adj_cmp_rate_coarse_sample")
-    .max()
-    .alias("adj_cmp_rate_coarse_sample_max"),
-)
-
-if coarse_audit.item(0, "k_mean_coarse_missing") > 0:
-    raise ValueError("k_mean_coarse is unexpectedly missing for observable students.")
-# Attendees without a classified first college can have missing coarse rates.
-# Report those counts below rather than treating them as a build failure.
-print("Coarse outcome audit:")
-print(coarse_audit)
-
-attendee_completion_audit = (
-    observable_y4.filter(pl.col("att_any_byY4") == 1)
-    .select(
-        pl.len().alias("by_y4_attendees"),
-        pl.col("completion_rate_150pct_firstinst")
-        .is_null()
-        .sum()
-        .alias("missing_first_institution_rate"),
-        (
-            pl.col("completion_rate_150pct_firstinst").is_null()
-            & pl.col("completion_rate_150pct_ip").is_not_null()
-        )
-        .sum()
-        .alias("filled_by_tier_median"),
-        pl.col("adj_cmp_rate")
-        .is_null()
-        .sum()
-        .alias("missing_after_tier_median"),
-        pl.col("adj_cmp_rate")
-        .is_null()
-        .mean()
-        .alias("missing_after_tier_median_share"),
-        (
-            pl.col("adj_cmp_rate").is_null()
-            & pl.col("tier_firstinst").is_null()
-        )
-        .sum()
-        .alias("missing_after_tier_median_no_tier"),
-        (
-            pl.col("adj_cmp_rate").is_null()
-            & pl.col("tier_firstinst").is_not_null()
-        )
-        .sum()
-        .alias("missing_after_tier_median_with_tier"),
-    )
-)
-print("By-Y4 attendee completion-rate audit:")
-print(attendee_completion_audit)
-print(
-    nsc_outcomes.select("sid_cepr", "cohort_lottery", "recovered_nsc_outcome").sort(
-        "sid_cepr"
-    )
-)
-
-# Nonmissing year-eight completion identifies students with the full Y8 window.
-year8_attendance_audit = (
-    nsc_outcomes.filter(pl.col("cmp_BA_byY8").is_not_null())
-    .select(
-        pl.len().alias("N_year8_sample"),
-        (pl.col("att_any_byY4") == 0).sum().alias("N_no_attendance_byY4"),
-        (
-            (pl.col("att_any_byY4") == 0)
-            & (pl.col("cmp_BA_byY8") == 1)
-        ).sum().alias("N_no_attendance_byY4_BA_byY8"),
-    )
-)
-print("Year-eight sample attendance and late BA completion check:")
-print(year8_attendance_audit)