ltc-code 0.2.24__tar.gz → 0.2.25__tar.gz

{ltc_code-0.2.24 → ltc_code-0.2.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ltc-code
-Version: 0.2.24
+Version: 0.2.25
 Summary: Add your description here
 Requires-Dist: fastexcel>=0.16,<0.20
 Requires-Dist: polars>=1.36.1,<1.42

{ltc_code-0.2.24 → ltc_code-0.2.25}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.2.24"
+version = "0.2.25"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"

{ltc_code-0.2.24 → ltc_code-0.2.25}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.2.24"
+version = "0.2.25"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"

{ltc_code-0.2.24 → ltc_code-0.2.25}/src/ltc_code/nsc/build_nsc_outcomes.py

@@ -76,6 +76,12 @@ OUTCOMES = OUTPUT_NSC / "nsc_outcomes.parquet"
 # Final student-level NSC outcomes file.
 
 
+DHS_STEM_CIP_ADDITIONS_FILE = (
+    PACKAGE_NSC / "dhs_stem" / "dhs_stem_cip_additions_2024.csv"
+)
+# Exact six-digit additions in the fixed July 22, 2024 DHS STEM list.
+
+
 ###########################################################
 # Define run settings
 ###########################################################
@@ -84,11 +90,128 @@ N_YEARS_OUT = 8
 NSC_OBSERVATION_YEAR = 2026
 NSC_CUTOFF_DATE = date(2026, 12, 31)
 RANDOM_SEED = 3852804
-STEM_CIP_FAMILIES = {11, 14, 15, 26, 27, 40, 41}
+SARAH_STEM_CIP_FAMILIES = {"11", "14", "15", "26", "27", "40", "41"}
+DHS_STEM_CIP_FAMILIES = {"14", "26", "27", "40"}
+STEM_DEFINITIONS = ["sarah", "dhs"]
 COLLEGE_TYPES = ["any", "4yr", "2yr", "elite"]
 AGE_ATTENDANCE_TYPES = ["any", "4yr", "2yr"]
 AGE_ATTENDANCE_RANGE = range(18, 27)
 
+# These ordered rules exactly reproduce Sarah's Stata replacements. Every rule
+# checks the original CIP slot, so the last matching rule wins when names overlap.
+MAJOR_TO_CIP_RULES = [
+    ("ECONOMICS", "4506"),
+    ("BUSINESS|FINANCE", "52"),
+    ("OPERATIONS MANAG", "52"),
+    ("MARKETING", "5214"),
+    ("MANAGEMENT", "5213"),
+    ("ANTHRO|ARCHEO|CRIMINO|DEMOGRA|POLITIC|SOCIOLO|HUMANI|GOVERN|GEOGRA", "45"),
+    ("VISUAL|PERFORM|ART|DANCE|PHOTO|FILM|MUSIC|MEDIA|ENTERTAIN|DRAMA|CRAFT|THEATRE|THEATER", "50"),
+    ("HISTORY", "54"),
+    ("PSYCHOLOGY", "42"),
+    ("PHILOSO", "38"),
+    ("LIBERAL|GENERAL STUDIES", "24"),
+    ("ENGLISH|ENGLISH LANGUAGE", "23"),
+    ("ENGLISH LIERATURE|WRITING", "23"),
+    ("LEGAL|LAW", "22"),
+    ("FOREIGN LANGUAGE|LANGUAGES|SPAN|JAPAN|CHIN|COMPARATIVE LING|FRENCH", "16"),
+    ("EDUCATION", "13"),
+    ("COMMUNI", "09"),
+    ("ETHNIC|CULTURAL|GENDER|WOMEN", "05"),
+    ("ENGI", "14"),
+    ("COMPUT|INFORMATION TECH", "11"),
+    ("EDUC|CURRIC|EARLY CHI", "13"),
+    ("SCHOOL COUN|STUDENT COUN", "13"),
+    ("LIBRA", "25"),
+    ("BIOLOG|NEURO|BIOTECH", "26"),
+    ("MATH", "27"),
+    ("LIBRA", "25"),
+    ("RELIG", "38"),
+    ("THEOLOG", "39"),
+    ("ASTRON|ATMOS|GEOLOG|CHEMI|PHYSIC|METEO", "40"),
+    ("PUBLIC ADMIN|PUBLIC POL|SOCIAL WORK|COMMUNITY ORG|SOCIAL SERVICE|HUMAN SERV", "44"),
+    ("BIOCHEM|BIOPHY|PHYSIO|BOTANY|ANATOMY|ZOO|GENETIC|GENOM|PHARMA|TOXICO|ECOLOG|EPIDEM|WILDLIFE", "26"),
+    ("VETERINARY|DENT|HEALTH SCI", "51"),
+    ("SPORT MANAGEMENT|SPORTS MANAGEMENT|KINESIO|EXER", "31"),
+    ("PUBLIC RELA|ADVERTIS|RADIO|TELEV|JOURNA", "09"),
+    ("PUBLIC HEALTH|REHAB|PHYSICAL THER|PREMED|PRE-MED|PRE-VET|NUR|MEDIC|HEALTH CARE ADM", "51"),
+    ("PLANT|SOIL|AGRI|TURF|HORTIC|FOOD SCI|ANIMAL", "01"),
+    ("ARCHI", "04"),
+    ("DESIGN", "50"),
+    ("HUMAN RESO|HOSPITALITY|HOTEL", "52"),
+    ("FOREST|ENVIRONMENTAL STUD|ENVIRONMENTAL SCI", "03"),
+    ("HUMAN RESO|HOSPITALITY|HOTEL|FASHION MERCH", "52"),
+    ("FOODS|NUTRITION", "19"),
+    ("CRIMINAL JUS|CRIME & JUST", "43"),
+    ("CLASSICS", "16"),
+    ("ACCOUNTING", "5203"),
+    ("AUTOMOTIVE TECH|AUTOMOB", "47"),
+    ("CLINICAL LAB", "51"),
+    ("HUMAN DEV", "19"),
+]
+
+DHS_STEM_CIP_ADDITIONS = set(
+    pl.read_csv(
+        DHS_STEM_CIP_ADDITIONS_FILE,
+        schema_overrides={"cip6": pl.String},
+    )["cip6"].to_list()
+)
+
+if len(DHS_STEM_CIP_ADDITIONS) != 251:
+    raise ValueError(
+        "Expected 251 exact six-digit additions in the 2024 DHS STEM lookup, "
+        f"found {len(DHS_STEM_CIP_ADDITIONS)}"
+    )
+
+
+def impute_cip_from_major(cip_column, major_column):
+    """Fill a missing CIP with Sarah's ordered major-name replacement rules."""
+    cip = pl.col(cip_column).cast(pl.String).str.strip_chars()
+    major = pl.col(major_column).cast(pl.String).str.to_uppercase().fill_null("")
+    imputed = pl.lit(None, dtype=pl.String)
+
+    for pattern, cip_value in MAJOR_TO_CIP_RULES:
+        imputed = (
+            pl.when(major.str.contains(pattern, literal=False))
+            .then(pl.lit(cip_value))
+            .otherwise(imputed)
+        )
+
+    return (
+        pl.when(cip.is_null() | cip.is_in(["", "."]))
+        .then(imputed)
+        .otherwise(cip)
+        .alias(cip_column)
+    )
+
+
+def stem_flag_from_cips(cip_columns, definition):
+    """Classify a row as STEM when any of its four imputed CIP slots qualify."""
+    cip_digits = [
+        pl.col(column)
+        .cast(pl.String)
+        .str.strip_chars()
+        .str.replace(r"\.0$", "")
+        .str.replace_all(r"\D", "")
+        for column in cip_columns
+    ]
+
+    if definition == "sarah":
+        conditions = [
+            cip.str.slice(0, 2).is_in(SARAH_STEM_CIP_FAMILIES)
+            for cip in cip_digits
+        ]
+    elif definition == "dhs":
+        conditions = [
+            cip.str.slice(0, 2).is_in(DHS_STEM_CIP_FAMILIES)
+            | ((cip.str.len_chars() == 6) & cip.is_in(DHS_STEM_CIP_ADDITIONS))
+            for cip in cip_digits
+        ]
+    else:
+        raise ValueError(f"Unknown STEM definition: {definition}")
+
+    return pl.any_horizontal(conditions).fill_null(False).cast(pl.Int8)
+
 
 ###########################################################
 # Read college crosswalk
@@ -621,6 +744,10 @@ nsc_new = (
         "Enrollment CIP 1": "enrollmentcip1",
         "Enrollment Major 2": "enrollmentmajor2",
         "Enrollment CIP 2": "enrollmentcip2",
+        "Enrollment Major 3": "enrollmentmajor3",
+        "Enrollment CIP 3": "enrollmentcip3",
+        "Enrollment Major 4": "enrollmentmajor4",
+        "Enrollment CIP 4": "enrollmentcip4",
         "Graduated?": "graduated",
         "Graduation Date": "graduationdate",
         "Degree Title": "degreetitle",
@@ -635,7 +762,7 @@ nsc_new = (
         "College Sequence": "collegesequence",
         "Student Age at start of term": "studentageatstartofterm",
         
-    })
+    }, strict=False)
     .with_row_index("origsrt", offset=1)
     .filter(pl.col("studyid").is_not_null())
     .with_columns(
@@ -689,6 +816,15 @@ nsc = (
     pl.concat([nsc_old, nsc_new], how="diagonal_relaxed")
 )
 
+# Older NSC extracts may omit some enrollment-major slots entirely. Add those
+# slots as nulls so enrollment and degree majors always retain distinct names.
+for column in [
+    *[f"enrollmentcip{index}" for index in range(1, 5)],
+    *[f"enrollmentmajor{index}" for index in range(1, 5)],
+]:
+    if column not in nsc.columns:
+        nsc = nsc.with_columns(pl.lit(None).cast(pl.String).alias(column))
+
 required_nsc_columns = [
     "sid_cepr",
     "ID_FSC",
@@ -708,6 +844,14 @@ required_nsc_columns = [
     "degreemajor2",
     "degreemajor3",
     "degreemajor4",
+    "enrollmentcip1",
+    "enrollmentcip2",
+    "enrollmentcip3",
+    "enrollmentcip4",
+    "enrollmentmajor1",
+    "enrollmentmajor2",
+    "enrollmentmajor3",
+    "enrollmentmajor4",
     "NSCdatayear",
     "searchbeginyear",
     "collegesequence",
@@ -768,6 +912,32 @@ nsc = (
     )
 )
 
+# Impute every missing enrollment and degree CIP slot independently from its
+# corresponding major name, using the same ordered rules Sarah uses for degree 1.
+for cip_prefix, major_prefix in [
+    ("enrollmentcip", "enrollmentmajor"),
+    ("degreecip", "degreemajor"),
+]:
+    nsc = nsc.with_columns(
+        [
+            impute_cip_from_major(
+                f"{cip_prefix}{index}",
+                f"{major_prefix}{index}",
+            )
+            for index in range(1, 5)
+        ]
+    )
+
+# A row is STEM when any of its four imputed enrollment-major CIPs qualify.
+enrollment_cip_columns = [f"enrollmentcip{index}" for index in range(1, 5)]
+nsc = nsc.with_columns(
+    [
+        stem_flag_from_cips(enrollment_cip_columns, definition)
+        .alias(f"_enrollment_stem_{definition}")
+        for definition in STEM_DEFINITIONS
+    ]
+)
+
 
 ###########################################################
 # Build enrollment outcomes
@@ -895,6 +1065,8 @@ enroll_weeks = (
         "week_end",
         "_term_att",
         "_enrollment_score",
+        "_enrollment_stem_sarah",
+        "_enrollment_stem_dhs",
         "NSCdatayear",
         "searchbeginyear",
         "collegesequence",
@@ -925,6 +1097,14 @@ enroll_weeks = (
             (pl.col("week") != pl.col("week").shift(1) + 1)
             | (pl.col("sid_cepr") != pl.col("sid_cepr").shift(1))
             | (pl.col("ID_FSC") != pl.col("ID_FSC").shift(1))
+            | (
+                pl.col("_enrollment_stem_sarah")
+                != pl.col("_enrollment_stem_sarah").shift(1)
+            )
+            | (
+                pl.col("_enrollment_stem_dhs")
+                != pl.col("_enrollment_stem_dhs").shift(1)
+            )
         )
         .fill_null(True)
         .cast(pl.Int64)
@@ -949,6 +1129,8 @@ enroll = (
         pl.col("k_mean").drop_nulls().first(),
         pl.col("completion_rate_150pct_ip").drop_nulls().first(),
         pl.col("_term_att").max(),
+        pl.col("_enrollment_stem_sarah").max(),
+        pl.col("_enrollment_stem_dhs").max(),
         pl.col("week").min().alias("week_start"),
         pl.col("week").max().alias("week_end"),
         pl.col("_enrollment_score").max(),
@@ -1019,6 +1201,32 @@ for year in range(1, N_YEARS_OUT + 1):
                         .alias(f"att_{college_type}_{label}")
                         for college_type in COLLEGE_TYPES
                     ]
+                    + [
+                        expression
+                        for definition in STEM_DEFINITIONS
+                        for expression in [
+                            (
+                                pl.col("_overlaps")
+                                * pl.col(f"_enrollment_stem_{definition}")
+                            )
+                            .max()
+                            .alias(f"att_stem_{definition}_{label}"),
+                            (
+                                pl.col("_overlaps")
+                                * pl.col("college_4yr").fill_null(0).cast(pl.Int8)
+                                * pl.col(f"_enrollment_stem_{definition}")
+                            )
+                            .max()
+                            .alias(f"att_4yr_stem_{definition}_{label}"),
+                            (
+                                pl.col("_overlaps")
+                                * pl.col("college_2yr").fill_null(0).cast(pl.Int8)
+                                * pl.col(f"_enrollment_stem_{definition}")
+                            )
+                            .max()
+                            .alias(f"att_2yr_stem_{definition}_{label}"),
+                        ]
+                    ]
                 )
             )
 
@@ -1053,6 +1261,32 @@ for age in AGE_ATTENDANCE_RANGE:
                 .alias(f"att_{college_type}_{age}")
                 for college_type in AGE_ATTENDANCE_TYPES
             ]
+            + [
+                expression
+                for definition in STEM_DEFINITIONS
+                for expression in [
+                    (
+                        pl.col("_overlaps")
+                        * pl.col(f"_enrollment_stem_{definition}")
+                    )
+                    .max()
+                    .alias(f"att_stem_{definition}_{age}"),
+                    (
+                        pl.col("_overlaps")
+                        * pl.col("college_4yr").fill_null(0).cast(pl.Int8)
+                        * pl.col(f"_enrollment_stem_{definition}")
+                    )
+                    .max()
+                    .alias(f"att_4yr_stem_{definition}_{age}"),
+                    (
+                        pl.col("_overlaps")
+                        * pl.col("college_2yr").fill_null(0).cast(pl.Int8)
+                        * pl.col(f"_enrollment_stem_{definition}")
+                    )
+                    .max()
+                    .alias(f"att_2yr_stem_{definition}_{age}"),
+                ]
+            ]
         )
     )
 
@@ -1065,7 +1299,17 @@ for piece in enrollment_pieces[1:]:
 # Sarah makes these categories disjoint: a 4-year enrollment wins over 2-year.
 for column in enrollment_outcomes.columns:
     if column.startswith("att_2yr_"):
-        four_year_column = column.replace("att_2yr_", "att_4yr_")
+        if "_stem_" in column:
+            definition = next(
+                item for item in STEM_DEFINITIONS if f"_stem_{item}_" in column
+            )
+            four_year_column = column.replace(
+                f"att_2yr_stem_{definition}_",
+                "att_4yr_",
+            )
+        else:
+            four_year_column = column.replace("att_2yr_", "att_4yr_")
+
         enrollment_outcomes = enrollment_outcomes.with_columns(
             pl.when(pl.col(four_year_column) == 1)
             .then(0)
@@ -1211,211 +1455,21 @@ degrees = (
         .otherwise(pl.col("_degree"))
         .alias("_degree")
     )
-    .with_columns(
-        pl.col("degreemajor1").cast(pl.String).str.to_uppercase().alias("_major1_upper"),
-        pl.col("degreecip1").cast(pl.String).str.strip_chars().alias("_degreecip1_clean"),
-    )
-    .with_columns(
-        pl.when(
-            pl.col("_degreecip1_clean").is_null()
-            | (pl.col("_degreecip1_clean") == "")
-            | (pl.col("_degreecip1_clean") == ".")
-        )
-        .then(
-            pl.when(pl.col("_major1_upper").str.contains("ECONOMICS", literal=False))
-            .then(pl.lit("4506"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "BUSINESS|FINANCE|OPERATIONS MANAG", literal=False
-                )
-            )
-            .then(pl.lit("52"))
-            .when(pl.col("_major1_upper").str.contains("MARKETING", literal=False))
-            .then(pl.lit("5214"))
-            .when(pl.col("_major1_upper").str.contains("MANAGEMENT", literal=False))
-            .then(pl.lit("5213"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "ANTHRO|ARCHEO|CRIMINO|DEMOGRA|POLITIC|SOCIOLO|HUMANI|GOVERN|GEOGRA",
-                    literal=False,
-                )
-            )
-            .then(pl.lit("45"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "VISUAL|PERFORM|ART|DANCE|PHOTO|FILM|MUSIC|MEDIA|ENTERTAIN|DRAMA|CRAFT|THEATRE|THEATER|DESIGN",
-                    literal=False,
-                )
-            )
-            .then(pl.lit("50"))
-            .when(pl.col("_major1_upper").str.contains("HISTORY", literal=False))
-            .then(pl.lit("54"))
-            .when(pl.col("_major1_upper").str.contains("PSYCHOLOGY", literal=False))
-            .then(pl.lit("42"))
-            .when(pl.col("_major1_upper").str.contains("PHILOSO|RELIG", literal=False))
-            .then(pl.lit("38"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "LIBERAL|GENERAL STUDIES", literal=False
-                )
-            )
-            .then(pl.lit("24"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "ENGLISH|ENGLISH LANGUAGE|ENGLISH LIERATURE|WRITING",
-                    literal=False,
-                )
-            )
-            .then(pl.lit("23"))
-            .when(pl.col("_major1_upper").str.contains("LEGAL|LAW", literal=False))
-            .then(pl.lit("22"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "FOREIGN LANGUAGE|LANGUAGES|SPAN|JAPAN|CHIN|COMPARATIVE LING|FRENCH|CLASSICS",
-                    literal=False,
-                )
-            )
-            .then(pl.lit("16"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "EDUCATION|EDUC|CURRIC|EARLY CHI|SCHOOL COUN|STUDENT COUN",
-                    literal=False,
-                )
-            )
-            .then(pl.lit("13"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "COMMUNI|PUBLIC RELA|ADVERTIS|RADIO|TELEV|JOURNA",
-                    literal=False,
-                )
-            )
-            .then(pl.lit("09"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "ETHNIC|CULTURAL|GENDER|WOMEN", literal=False
-                )
-            )
-            .then(pl.lit("05"))
-            .when(pl.col("_major1_upper").str.contains("ENGI", literal=False))
-            .then(pl.lit("14"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "COMPUT|INFORMATION TECH", literal=False
-                )
-            )
-            .then(pl.lit("11"))
-            .when(pl.col("_major1_upper").str.contains("LIBRA", literal=False))
-            .then(pl.lit("25"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "BIOLOG|NEURO|BIOTECH|BIOCHEM|BIOPHY|PHYSIO|BOTANY|ANATOMY|ZOO|GENETIC|GENOM|PHARMA|TOXICO|ECOLOG|EPIDEM|WILDLIFE",
-                    literal=False,
-                )
-            )
-            .then(pl.lit("26"))
-            .when(pl.col("_major1_upper").str.contains("MATH", literal=False))
-            .then(pl.lit("27"))
-            .when(pl.col("_major1_upper").str.contains("THEOLOG", literal=False))
-            .then(pl.lit("39"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "ASTRON|ATMOS|GEOLOG|CHEMI|PHYSIC|METEO", literal=False
-                )
-            )
-            .then(pl.lit("40"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "PUBLIC ADMIN|PUBLIC POL|SOCIAL WORK|COMMUNITY ORG|SOCIAL SERVICE|HUMAN SERV",
-                    literal=False,
-                )
-            )
-            .then(pl.lit("44"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "VETERINARY|DENT|HEALTH SCI|PUBLIC HEALTH|REHAB|PHYSICAL THER|PREMED|PRE-MED|PRE-VET|NUR|MEDIC|HEALTH CARE ADM|CLINICAL LAB",
-                    literal=False,
-                )
-            )
-            .then(pl.lit("51"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "SPORT MANAGEMENT|SPORTS MANAGEMENT|KINESIO|EXER", literal=False
-                )
-            )
-            .then(pl.lit("31"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "PLANT|SOIL|AGRI|TURF|HORTIC|FOOD SCI|ANIMAL", literal=False
-                )
-            )
-            .then(pl.lit("01"))
-            .when(pl.col("_major1_upper").str.contains("ARCHI", literal=False))
-            .then(pl.lit("04"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "HUMAN RESO|HOSPITALITY|HOTEL|FASHION MERCH", literal=False
-                )
-            )
-            .then(pl.lit("52"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "FOREST|ENVIRONMENTAL STUD|ENVIRONMENTAL SCI", literal=False
-                )
-            )
-            .then(pl.lit("03"))
-            .when(
-                pl.col("_major1_upper").str.contains("FOODS|NUTRITION|HUMAN DEV", literal=False)
-            )
-            .then(pl.lit("19"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "CRIMINAL JUS|CRIME & JUST", literal=False
-                )
-            )
-            .then(pl.lit("43"))
-            .when(pl.col("_major1_upper").str.contains("ACCOUNTING", literal=False))
-            .then(pl.lit("5203"))
-            .when(
-                pl.col("_major1_upper").str.contains(
-                    "AUTOMOTIVE TECH|AUTOMOB", literal=False
-                )
-            )
-            .then(pl.lit("47"))
-            .otherwise(pl.col("_degreecip1_clean"))
-        )
-        .otherwise(pl.col("_degreecip1_clean"))
-        .alias("degreecip1_imputed")
-    )
     .with_columns(
         [
-            pl.when(i == 1)
-            .then(pl.col("degreecip1_imputed"))
-            .otherwise(pl.col(f"degreecip{i}"))
-            .cast(pl.String)
-            .str.replace_all(r"\D", "")
-            .str.slice(0, 2)
-            .cast(pl.Int64, strict=False)
-            .alias(f"_cip_family{i}")
-            for i in range(1, 5)
+            stem_flag_from_cips(
+                [f"degreecip{index}" for index in range(1, 5)],
+                definition,
+            ).alias(f"degree_stem_{definition}")
+            for definition in STEM_DEFINITIONS
         ]
     )
     .with_columns(
-        pl.any_horizontal(
-            [pl.col(f"_cip_family{i}").is_in(STEM_CIP_FAMILIES) for i in range(1, 5)]
-        )
-        .fill_null(False)
-        .cast(pl.Int8)
-        .alias("degree_stem"),
         pl.coalesce(
-            [pl.col("degreecip1_imputed")]
-            + [pl.col(f"degreecip{i}") for i in range(2, 5)]
+            [pl.col(f"degreecip{index}") for index in range(1, 5)]
         ).alias("degreecip"),
         pl.coalesce(
-            [
-                pl.col(column)
-                for column in nsc.columns
-                if column.startswith("degreemajor")
-            ]
+            [pl.col(f"degreemajor{index}") for index in range(1, 5)]
         ).alias("degreemajor"),
     )
 )
@@ -1452,6 +1506,33 @@ for year in range(1, N_YEARS_OUT + 1):
             )
             .max()
             .alias(f"cmp_elite_byY{year}"),
+            *[
+                expression
+                for definition in STEM_DEFINITIONS
+                for expression in [
+                    (
+                        pl.col("_by_end")
+                        * pl.col("_degree").is_in(["AA", "BA"]).cast(pl.Int8)
+                        * pl.col(f"degree_stem_{definition}")
+                    )
+                    .max()
+                    .alias(f"cmp_stem_{definition}_byY{year}"),
+                    (
+                        pl.col("_by_end")
+                        * (pl.col("_degree") == "AA").cast(pl.Int8)
+                        * pl.col(f"degree_stem_{definition}")
+                    )
+                    .max()
+                    .alias(f"cmp_AA_stem_{definition}_byY{year}"),
+                    (
+                        pl.col("_by_end")
+                        * (pl.col("_degree") == "BA").cast(pl.Int8)
+                        * pl.col(f"degree_stem_{definition}")
+                    )
+                    .max()
+                    .alias(f"cmp_BA_stem_{definition}_byY{year}"),
+                ]
+            ],
         )
     )
 
@@ -1494,7 +1575,8 @@ degree_details = (
         pl.col("graduated_date").first().alias("degree_date"),
         pl.col("degreecip").first().alias("degreecip"),
         pl.col("degreemajor").first().alias("degreemajor"),
-        pl.col("degree_stem").max().alias("degree_stem"),
+        pl.col("degree_stem_sarah").max().alias("degree_stem_sarah"),
+        pl.col("degree_stem_dhs").max().alias("degree_stem_dhs"),
         pl.col("ID_FSC").first().alias("ID_FSC_degree"),
         pl.col("college_name").first().alias("college_name_degree"),
         pl.col("college_years").first().alias("college_years_degree"),
@@ -1727,6 +1809,39 @@ outcome_columns = [
     "cmp_elite_byY4", "cmp_elite_byY5", "cmp_elite_byY6", "cmp_elite_byY7", "cmp_elite_byY8",
 ]
 
+stem_outcome_columns = []
+for year in range(1, N_YEARS_OUT + 1):
+    for prefix in ["in", "by"]:
+        for term in ["", "fall", "spring"]:
+            label = f"{prefix}Y{year}{term}"
+            for definition in STEM_DEFINITIONS:
+                stem_outcome_columns.extend(
+                    [
+                        f"att_stem_{definition}_{label}",
+                        f"att_4yr_stem_{definition}_{label}",
+                        f"att_2yr_stem_{definition}_{label}",
+                    ]
+                )
+
+    for definition in STEM_DEFINITIONS:
+        stem_outcome_columns.extend(
+            [
+                f"cmp_stem_{definition}_byY{year}",
+                f"cmp_AA_stem_{definition}_byY{year}",
+                f"cmp_BA_stem_{definition}_byY{year}",
+            ]
+        )
+
+for age in AGE_ATTENDANCE_RANGE:
+    for definition in STEM_DEFINITIONS:
+        stem_outcome_columns.extend(
+            [
+                f"att_stem_{definition}_{age}",
+                f"att_4yr_stem_{definition}_{age}",
+                f"att_2yr_stem_{definition}_{age}",
+            ]
+        )
+
 nsc_outcomes = nsc_outcomes.with_columns(
     pl.any_horizontal([pl.col(column).fill_null(0) == 1 for column in outcome_columns])
     .cast(pl.Int8)
@@ -1747,7 +1862,7 @@ keep_columns = [
     "college_name_firstinst",
     "tier_firstinst",
     "completion_rate_150pct_firstinst",
-] + outcome_columns
+] + outcome_columns + stem_outcome_columns
 
 nsc_outcomes_final = nsc_outcomes.select(keep_columns)
 

ltc_code-0.2.25/src/ltc_code/nsc/dhs_stem/dhs_stem_cip_additions_2024.csv

@@ -0,0 +1,252 @@
+cip6
+010308
+010901
+010902
+010903
+010904
+010905
+010906
+010907
+010999
+011001
+011002
+011099
+011101
+011102
+011103
+011104
+011105
+011106
+011199
+011201
+011202
+011203
+011299
+018105
+018106
+018107
+018108
+018109
+018110
+018111
+030101
+030103
+030104
+030199
+030204
+030205
+030210
+030501
+030502
+030508
+030509
+030510
+030601
+040601
+040902
+090702
+100304
+110101
+110102
+110103
+110104
+110105
+110199
+110201
+110202
+110203
+110299
+110301
+110401
+110501
+110701
+110801
+110802
+110803
+110804
+110899
+110901
+110902
+111001
+111002
+111003
+111004
+111005
+111006
+111099
+130501
+130601
+130603
+130608
+150000
+150101
+150201
+150303
+150304
+150305
+150306
+150399
+150401
+150403
+150404
+150405
+150406
+150407
+150499
+150501
+150506
+150507
+150508
+150599
+150607
+150611
+150612
+150613
+150614
+150615
+150616
+150617
+150699
+150701
+150702
+150703
+150704
+150799
+150801
+150803
+150805
+150899
+150901
+150903
+150999
+151001
+151102
+151103
+151199
+151201
+151202
+151203
+151204
+151299
+151301
+151302
+151303
+151304
+151305
+151306
+151399
+151401
+151501
+151502
+151503
+151599
+151601
+151701
+151703
+159999
+280501
+280502
+280505
+290201
+290202
+290203
+290204
+290205
+290206
+290207
+290299
+290301
+290302
+290303
+290304
+290305
+290306
+290307
+290399
+290401
+290402
+290403
+290404
+290405
+290406
+290407
+290408
+290409
+290499
+299999
+300101
+300601
+300801
+301001
+301701
+301801
+301901
+302501
+302701
+303001
+303101
+303201
+303301
+303401
+303501
+303801
+303901
+304101
+304301
+304401
+304801
+304901
+305001
+307001
+307101
+307102
+307103
+307104
+307199
+410000
+410101
+410204
+410205
+410299
+410301
+410303
+410399
+419999
+422701
+422702
+422703
+422704
+422705
+422706
+422707
+422708
+422709
+422710
+422799
+422804
+430403
+430406
+430407
+450102
+450301
+450501
+450603
+450702
+490101
+511002
+511005
+511401
+512003
+512004
+512005
+512006
+512007
+512009
+512010
+512202
+512205
+512706
+521301
+521302
+521304
+521399

ltc_code-0.2.25/src/ltc_code/nsc/dhs_stem/extract_dhs_stem_cips.R

@@ -0,0 +1,50 @@
+# Rebuild the fixed DHS six-digit additions lookup from the July 22, 2024 PDF.
+# Run from this directory with: Rscript extract_dhs_stem_cips.R
+
+# Source: https://www.ice.gov/doclib/sevis/pdf/stemList2024.pdf
+
+args <- commandArgs(trailingOnly = FALSE)
+file_arg <- grep("^--file=", args, value = TRUE)
+script_dir <- if (length(file_arg) == 1) {
+  dirname(normalizePath(sub("^--file=", "", file_arg)))
+} else {
+  getwd()
+}
+
+pdf_path <- file.path(script_dir, "stemList2024.pdf")
+output_path <- file.path(script_dir, "dhs_stem_cip_additions_2024.csv")
+
+pdftotext <- Sys.which("pdftotext")
+if (pdftotext == "") {
+  stop("Install Poppler so the pdftotext executable is available on PATH.")
+}
+
+text_path <- tempfile(fileext = ".txt")
+on.exit(unlink(text_path), add = TRUE)
+status <- system2(pdftotext, c("-layout", shQuote(pdf_path), shQuote(text_path)))
+if (status != 0) {
+  stop("pdftotext could not extract the DHS PDF.")
+}
+
+pdf_text <- paste(readLines(text_path, warn = FALSE), collapse = "\n")
+cip_codes <- unique(regmatches(
+  pdf_text,
+  gregexpr("(?<![0-9])[0-9]{2}\\.[0-9]{4}(?![0-9])", pdf_text, perl = TRUE)
+)[[1]])
+
+core_families <- c("14", "26", "27", "40")
+cip_codes <- sort(cip_codes[!substr(cip_codes, 1, 2) %in% core_families])
+cip_codes <- gsub("\\.", "", cip_codes)
+
+stopifnot(length(cip_codes) == 251)
+stopifnot(!any(substr(cip_codes, 1, 2) %in% core_families))
+stopifnot(all(grepl("^[0-9]{6}$", cip_codes)))
+
+write.csv(
+  data.frame(cip6 = cip_codes),
+  output_path,
+  row.names = FALSE,
+  quote = FALSE
+)
+
+message("Wrote ", length(cip_codes), " DHS six-digit additions to ", output_path)