ltc-code 0.2.13__tar.gz → 0.2.14__tar.gz

{ltc_code-0.2.13 → ltc_code-0.2.14}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.3
 Name: ltc-code
-Version: 0.2.13
+Version: 0.2.14
 Summary: Add your description here
 Requires-Dist: fastexcel>=0.20.2
 Requires-Dist: polars>=1.42.1
 Requires-Dist: polars-readstat>=0.20.2
-Requires-Python: >=3.10
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 

{ltc_code-0.2.13 → ltc_code-0.2.14}/pyproject.toml

@@ -1,9 +1,9 @@
 [project]
 name = "ltc-code"
-version = "0.2.13"
+version = "0.2.14"
 description = "Add your description here"
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.9"
 dependencies = [
     "fastexcel>=0.20.2",
     "polars>=1.42.1",

{ltc_code-0.2.13 → ltc_code-0.2.14}/src/ltc_code/nsc/build_nsc_outcomes.py

@@ -1,5 +1,6 @@
 # --- Import necessary packages ---
 
+from datetime import date
 from pathlib import Path
 
 import polars as pl
@@ -31,6 +32,17 @@ DIRECTORY = RAW_NSC / "directory.dta"
 CREDENTIAL_LOOKUP = RAW_NSC / "CREDENTIAL_LEVEL_LOOKUP_TABLE.xlsx"
 # NSC credential-title lookup used before regex-based degree classification.
 
+CHETTY_MRC_TABLE2_CANDIDATES = [
+    RAW_NSC / "chetty" / "mrc_table2.dta",
+    RAW_NSC / "mrc_table2.dta",
+    PACKAGE_NSC.parents[3] / "College" / "chetty" / "mrc_table2.dta",
+]
+CHETTY_MRC_TABLE2 = next(
+    (path for path in CHETTY_MRC_TABLE2_CANDIDATES if path.exists()),
+    CHETTY_MRC_TABLE2_CANDIDATES[0],
+)
+# Chetty/Barron college tier lookup. Numeric tier 1/2 identifies elite colleges.
+
 INPUT_NSC = PACKAGE_NSC / "input"
 # Folder for secured Census inputs that are not packaged with the code.
 
@@ -40,6 +52,12 @@ APPS = INPUT_NSC / "apps.csv"
 NSC_RECORDS = INPUT_NSC / "nsc_records.csv"
 # Long NSC enrollment and degree records.
 
+INPUT_NSC_OLD = INPUT_NSC / "nsc_records_old.dta"
+# Older NSC pull, kept so the build can append old and new NSC versions.
+
+INPUT_NSC_NEW = INPUT_NSC / "nsc_records_new.dta"
+# Newer NSC pull with studyid-based rows.
+
 OUTPUT_NSC = PACKAGE_NSC / "output"
 # Package-local output folder for NSC outcome files.
 
@@ -53,9 +71,10 @@ OUTCOMES = OUTPUT_NSC / "nsc_outcomes.parquet"
 
 N_YEARS_OUT = 8
 NSC_OBSERVATION_YEAR = 2026
+NSC_CUTOFF_DATE = date(2026, 12, 31)
 RANDOM_SEED = 3852804
 STEM_CIP_FAMILIES = {11, 14, 15, 26, 27, 40, 41}
-COLLEGE_TYPES = ["any", "4yr", "2yr"]
+COLLEGE_TYPES = ["any", "4yr", "2yr", "elite"]
 
 
 ###########################################################
@@ -191,6 +210,26 @@ directory = (
     .select("unitid", "college_name_directory", "years_directory", "ownership_directory")
 )
 
+# Chetty/Barron tiers are keyed at the super-OPE level. The NSC college
+# crosswalk carries OPE IDs, so joining tier onto college_ref makes the tier
+# available on each enrollment spell.
+chetty_tiers = (
+    scan_readstat(
+        CHETTY_MRC_TABLE2,
+        missing_string_as_null=True,
+        value_labels_as_strings=False,
+    )
+    .select("super_opeid", "tier")
+    .collect()
+    .with_columns(
+        pl.col("super_opeid").cast(pl.Int64, strict=False).alias("ID_OPE"),
+        pl.col("tier").cast(pl.Int8, strict=False),
+    )
+    .select("ID_OPE", "tier")
+    .drop_nulls("ID_OPE")
+    .unique("ID_OPE", keep="first")
+)
+
 
 ###########################################################
 # Build college reference data
@@ -201,6 +240,7 @@ college_ref = (
     college_crosswalk.join(ipeds_2013, on="unitid", how="left")
     .join(ipeds_manual, on="unitid", how="left")
     .join(directory, on="unitid", how="left")
+    .join(chetty_tiers, on="ID_OPE", how="left", validate="m:1")
     .with_columns(
         pl.coalesce(
             [
@@ -228,6 +268,7 @@ college_ref = (
     .with_columns(
         (pl.col("college_years") == 4).cast(pl.Int8).alias("college_4yr"),
         (pl.col("college_years") == 2).cast(pl.Int8).alias("college_2yr"),
+        pl.col("tier").is_in([1, 2]).cast(pl.Int8).alias("college_elite"),
     )
     .with_columns(
         (pl.col("college_4yr") + pl.col("college_2yr") > 0)
@@ -244,6 +285,8 @@ college_ref = (
         "college_any",
         "college_4yr",
         "college_2yr",
+        "tier",
+        "college_elite",
     )
 )
 
@@ -354,22 +397,41 @@ fsc_branch_fixes = [
 
 # The real NSC file uses raw NSC names; Sarah renames these before building
 # outcomes. This block makes those same names explicit in Polars.
-if str(NSC_RECORDS).endswith(".dta"):
-    nsc = scan_readstat(
-        NSC_RECORDS, missing_string_as_null=True, value_labels_as_strings=True
-    ).collect()
-else:
-    nsc = pl.read_csv(NSC_RECORDS, infer_schema_length=0)
+nsc_old = (
+    scan_readstat(INPUT_NSC_OLD, missing_string_as_null=True, value_labels_as_strings=True)
+    .with_columns(
+        pl.lit(2023).alias("NSCdatayear"),
 
-if "sid_cepr" not in nsc.columns and "studyid" in nsc.columns:
-    nsc = (
-        nsc.with_row_index("origsrt")
-        .with_columns(
-            pl.col("studyid").cast(pl.String).str.split("_").alias("sid_cepr")
-        )
-        .explode("sid_cepr")
-        .filter(pl.col("sid_cepr") != "")
+        pl.col("searchdate")
+        .str.slice(0, 4)
+        .cast(pl.Int32)
+        .alias("searchbeginyear")
     )
+    .collect()
+)
+
+nsc_new = (
+    scan_readstat(INPUT_NSC_NEW, missing_string_as_null=True, value_labels_as_strings=True)
+    .with_row_index("origsrt", offset=1)
+    .filter(pl.col("studyid").is_not_null())
+    .with_columns(
+        pl.col("studyid").cast(pl.String).str.replace_all("_", "").alias("sid_cepr")
+    )
+    .explode("sid_cepr")
+    .filter(pl.col("sid_cepr") != "")
+    .with_columns(
+        pl.col("sid_cepr").cast(pl.String).str.replace_all("_", "").cast(pl.Int64, strict=False)
+    )
+    .filter(
+        pl.col("sid_cepr").is_not_null()
+    )
+    .drop("studyid", "studentageatstartofterm")
+    .with_columns(
+        pl.lit(2026).alias("NSCdatayear"),
+    )
+    .collect()
+)
+
 
 nsc_renames = {
     "collegecodebranch": "ID_FSC",
@@ -380,28 +442,28 @@ nsc_renames = {
     "enrollmentend": "term_end_date",
     "graduationdate": "graduated_date",
     "degreetitle": "degree_title",
-    "searchsubmit": "NSCdatayear",
 }
 
-nsc = nsc.rename(
+nsc_old = nsc_old.rename(
     {
         source: target
         for source, target in nsc_renames.items()
-        if source in nsc.columns and target not in nsc.columns
+        if source in nsc_old.columns and target not in nsc_old.columns
     }
 )
 
-for column in ["degreecip1", "degreecip2", "degreecip3", "degreecip4"]:
-    if column not in nsc.columns:
-        nsc = nsc.with_columns(pl.lit(None).cast(pl.String).alias(column))
-
-for column in ["degreemajor1", "degreemajor2", "degreemajor3", "degreemajor4"]:
-    if column not in nsc.columns:
-        nsc = nsc.with_columns(pl.lit(None).cast(pl.String).alias(column))
+nsc_new = nsc_new.rename(
+    {
+        source: target
+        for source, target in nsc_renames.items()
+        if source in nsc_new.columns and target not in nsc_new.columns
+    }
+)
 
-for column in ["NSCdatayear", "searchbeginyear", "collegesequence"]:
-    if column not in nsc.columns:
-        nsc = nsc.with_columns(pl.lit(None).cast(pl.Int64).alias(column))
+# Append
+nsc = (
+    pl.concat([nsc_old, nsc_new], how="diagonal_relaxed")
+)
 
 required_nsc_columns = [
     "sid_cepr",
@@ -493,10 +555,19 @@ enrollment_status = pl.col("enrollment").str.to_uppercase().fill_null("")
 enroll = (
     nsc.filter(pl.col("graduated") == "N")
     .filter(pl.col("term_start_date").is_not_null() & pl.col("term_end_date").is_not_null())
+    .filter(pl.col("term_start_date") <= NSC_CUTOFF_DATE)
+    .with_columns(
+        pl.min_horizontal(
+            pl.col("term_end_date"),
+            pl.lit(NSC_CUTOFF_DATE),
+        ).alias("term_end_date")
+    )
     .join(apps, on="sid_cepr", how="inner")
-    .join(college_ref, on="ID_FSC", how="left")
+    .join(college_ref, on="ID_FSC", how="left", validate="m:1")
     .with_columns(
-        pl.when(enrollment_status.str.contains("FULL|\\bF\\b", literal=False))
+        pl.when(enrollment_status == "") # unknown
+        .then(9)
+        .when(enrollment_status.str.contains("FULL|\\bF\\b", literal=False))
         .then(5)
         .when(enrollment_status.str.contains("PART", literal=False))
         .then(4)
@@ -514,15 +585,7 @@ enroll = (
         ((pl.col("term_start_date").cast(pl.Int64) // 7).cast(pl.Int64)).alias(
             "week_start"
         ),
-        ((pl.col("term_end_date").cast(pl.Int64) // 7).cast(pl.Int64)).alias("week_end"),
-        pl.coalesce([pl.col("NSCdatayear"), pl.lit(-1)]).alias("_NSCdatayear_sort"),
-        pl.coalesce([pl.col("searchbeginyear"), pl.lit(-1)]).alias(
-            "_searchbeginyear_sort"
-        ),
-        pl.coalesce([pl.col("collegesequence"), pl.lit(999999)]).alias(
-            "_collegesequence_sort"
-        ),
-        pl.coalesce([pl.col("unitid"), pl.lit(999999999)]).alias("_unitid_sort"),
+        ((pl.col("term_end_date").cast(pl.Int64) // 7).cast(pl.Int64)).alias("week_end"),       
     )
     .filter(pl.col("week_end") >= pl.col("week_start"))
     .unique(keep="first")
@@ -532,12 +595,13 @@ enroll = (
             "college",
             "term_start_date",
             "term_end_date",
-            "_NSCdatayear_sort",
-            "_searchbeginyear_sort",
-            "_collegesequence_sort",
+            "NSCdatayear",
+            "searchbeginyear",
+            "collegesequence",
             "cohort_18",
             "_enrollment_score",
-        ]
+        ],
+        nulls_last=True
     )
     .unique(
         [
@@ -545,9 +609,9 @@ enroll = (
             "college",
             "term_start_date",
             "term_end_date",
-            "_NSCdatayear_sort",
-            "_searchbeginyear_sort",
-            "_collegesequence_sort",
+            "NSCdatayear",
+            "searchbeginyear",
+            "collegesequence",
             "cohort_18",
         ],
         keep="first",
@@ -558,9 +622,9 @@ enroll = (
             "college",
             "term_start_date",
             "term_end_date",
-            "_NSCdatayear_sort",
-            "_searchbeginyear_sort",
-            "_collegesequence_sort",
+            "NSCdatayear",
+            "searchbeginyear",
+            "collegesequence",
         ],
         keep="first",
     )
@@ -570,16 +634,16 @@ enroll = (
             "college",
             "term_start_date",
             "term_end_date",
-            "_NSCdatayear_sort",
-            "_searchbeginyear_sort",
+            "NSCdatayear",
+            "searchbeginyear",
         ],
         keep="first",
     )
     .unique(
-        ["sid_cepr", "college", "term_start_date", "term_end_date", "_NSCdatayear_sort"],
+        ["sid_cepr", "college", "term_start_date", "term_end_date", "NSCdatayear"],
         keep="first",
     )
-    .sort(["sid_cepr", "college", "term_start_date", "term_end_date", "_NSCdatayear_sort"])
+    .sort(["sid_cepr", "college", "term_start_date", "term_end_date", "NSCdatayear"])
     .unique(["sid_cepr", "college", "term_start_date", "term_end_date"], keep="last")
 )
 
@@ -597,16 +661,18 @@ enroll_weeks = (
         "college_any",
         "college_4yr",
         "college_2yr",
+        "tier",
+        "college_elite",
         "term_start_date",
         "term_end_date",
         "week_start",
         "week_end",
         "_term_att",
         "_enrollment_score",
-        "_NSCdatayear_sort",
-        "_searchbeginyear_sort",
-        "_collegesequence_sort",
-        "_unitid_sort",
+        "NSCdatayear",
+        "searchbeginyear",
+        "collegesequence",
+        "unitid",
     )
     .with_columns(
         pl.int_ranges("week_start", pl.col("week_end") + 1).alias("week")
@@ -617,13 +683,14 @@ enroll_weeks = (
             "sid_cepr",
             "week",
             "_enrollment_score",
-            "_NSCdatayear_sort",
+            "NSCdatayear",
             "college_4yr",
-            "_collegesequence_sort",
+            "collegesequence",
             "term_start_date",
-            "_unitid_sort",
+            "unitid",
         ],
         descending=[False, False, True, True, True, False, False, False],
+        nulls_last=True
     )
     .unique(["sid_cepr", "week"], keep="first")
     .sort(["sid_cepr", "ID_FSC", "week"])
@@ -651,9 +718,12 @@ enroll = (
         pl.col("college_any").max(),
         pl.col("college_4yr").max(),
         pl.col("college_2yr").max(),
+        pl.col("tier").drop_nulls().min(),
+        pl.col("college_elite").max(),
         pl.col("_term_att").max(),
         pl.col("week").min().alias("week_start"),
         pl.col("week").max().alias("week_end"),
+        pl.col("_enrollment_score").max(),
     )
     .with_columns(
         (pl.col("week_start") * 7).cast(pl.Date).alias("term_start_date"),
@@ -1141,21 +1211,60 @@ nsc_outcomes = (
 )
 
 for year in range(1, N_YEARS_OUT + 1):
-    observable = pl.col("cohort_18") <= NSC_OBSERVATION_YEAR - year
-    observed_columns = [
+    in_start = pl.date(pl.col("cohort_18") + year - 1, 7, 1)
+    by_start = pl.date(pl.col("cohort_18"), 7, 1)
+    by_end = pl.date(pl.col("cohort_18") + year, 6, 30)
+
+    att_in_columns = [
         column
         for column in nsc_outcomes.columns
-        if f"_byY{year}" in column or f"_inY{year}" in column
+        if column.startswith("att_") and f"_inY{year}" in column
+    ]
+
+    att_by_columns = [
+        column
+        for column in nsc_outcomes.columns
+        if column.startswith("att_") and f"_byY{year}" in column
+    ]
+
+    cmp_by_columns = [
+        column
+        for column in nsc_outcomes.columns
+        if column.startswith("cmp_") and f"_byY{year}" in column
     ]
 
     nsc_outcomes = nsc_outcomes.with_columns(
         [
-            pl.when(pl.col(column).is_null() & observable)
+            pl.when(in_start > NSC_CUTOFF_DATE)
+            .then(None)
+            .when(pl.col(column).is_null())
+            .then(0)
+            .otherwise(pl.col(column))
+            .cast(pl.Int8)
+            .alias(column)
+            for column in att_in_columns
+        ]
+        +
+        [
+            pl.when(by_start > NSC_CUTOFF_DATE)
+            .then(None)
+            .when(pl.col(column).is_null())
+            .then(0)
+            .otherwise(pl.col(column))
+            .cast(pl.Int8)
+            .alias(column)
+            for column in att_by_columns
+        ]
+        +
+        [
+            pl.when(by_end > NSC_CUTOFF_DATE)
+            .then(None)
+            .when(pl.col(column).is_null())
             .then(0)
             .otherwise(pl.col(column))
             .cast(pl.Int8)
             .alias(column)
-            for column in observed_columns
+            for column in cmp_by_columns
         ]
     )
 
@@ -1165,9 +1274,21 @@ for year in range(1, N_YEARS_OUT + 1):
 ###########################################################
 
 outcome_columns = [
-    column
-    for column in nsc_outcomes.columns
-    if column.startswith("att_") or column.startswith("cmp_")
+    "att_any_inY1", "att_any_inY2", "att_any_inY3", "att_any_inY4",
+    "att_4yr_inY1", "att_4yr_inY2", "att_4yr_inY3", "att_4yr_inY4",
+    "att_2yr_inY1", "att_2yr_inY2", "att_2yr_inY3", "att_2yr_inY4",
+    "att_elite_inY1", "att_elite_inY2", "att_elite_inY3", "att_elite_inY4",
+
+    "att_any_byY2", "att_any_byY3", "att_any_byY4",
+    "att_4yr_byY2", "att_4yr_byY3", "att_4yr_byY4",
+    "att_2yr_byY2", "att_2yr_byY3", "att_2yr_byY4", 
+    "att_elite_byY2", "att_elite_byY3", "att_elite_byY4",
+
+    "cmp_any_byY4", "cmp_any_byY5", "cmp_any_byY6", "cmp_any_byY7", "cmp_any_byY8",
+    "cmp_AA_byY4", "cmp_AA_byY5", "cmp_AA_byY6", "cmp_AA_byY7", "cmp_AA_byY8",
+    "cmp_BA_byY4", "cmp_BA_byY5", "cmp_BA_byY6", "cmp_BA_byY7", "cmp_BA_byY8",
+
+
 ]
 
 nsc_outcomes = nsc_outcomes.with_columns(
@@ -1176,6 +1297,12 @@ nsc_outcomes = nsc_outcomes.with_columns(
     .alias("recovered_nsc_outcome")
 )
 
+keep_columns = [
+    "sid_cepr",
+] + outcome_columns
+
+nsc_outcomes_final = nsc_outcomes.select(keep_columns)
+
 OUTCOMES.parent.mkdir(parents=True, exist_ok=True)
 nsc_outcomes.write_parquet(OUTCOMES)
 nsc_outcomes.with_columns(