ltc-code 0.2.16__tar.gz → 0.2.17__tar.gz

{ltc_code-0.2.16 → ltc_code-0.2.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ltc-code
-Version: 0.2.16
+Version: 0.2.17
 Summary: Add your description here
 Requires-Dist: fastexcel>=0.16,<0.20
 Requires-Dist: polars>=1.36.1,<1.42

{ltc_code-0.2.16 → ltc_code-0.2.17}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.2.16"
+version = "0.2.17"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"

{ltc_code-0.2.16 → ltc_code-0.2.17}/src/ltc_code/nsc/build_nsc_outcomes.py

@@ -54,25 +54,21 @@ CHETTY_MRC_TABLE11 = next(
 )
 # Chetty OPE-to-super-OPE crosswalk used to attach tier to NSC schools.
 
-INPUT_NSC = PACKAGE_NSC / "input"
-# Folder for secured Census inputs that are not packaged with the code.
 
 APPS = INPUT_NSC / "apps.csv"
 # Application/student records; source for unique sid_cepr/cohort_lottery.
 
-NSC_RECORDS = INPUT_NSC / "nsc_records.csv"
-# Long NSC enrollment and degree records.
+INPUT_NSC = PACKAGE_NSC / "input"
+# Folder for secured Census inputs that are not packaged with the code.
+
 
 INPUT_NSC_OLD = INPUT_NSC / "nsc_records_old.dta"
 # Older NSC pull, kept so the build can append old and new NSC versions.
 
-INPUT_NSC_NEW = INPUT_NSC / "nsc_records_new.dta"
+INPUT_NSC_NEW = INPUT_NSC / "nsc_records_new.csv"
 # Newer NSC pull with studyid-based rows.
 
-OUTPUT_NSC = PACKAGE_NSC / "output"
-# Package-local output folder for NSC outcome files.
-
-OUTCOMES = OUTPUT_NSC / "nsc_outcomes.parquet"
+OUTCOMES_OUTPUT = OUTPUT_NSC / "nsc_outcomes.parquet"
 # Final student-level NSC outcomes file.
 
 
@@ -99,17 +95,13 @@ college_crosswalk = (
         columns=["ID_IPEDS", "ID_OPE", "ID_FSC", "ID_FSC_long", "name"],
     )
     .with_columns(
-        pl.when(
-            pl.col("ID_FSC_long").is_not_null()
-            & (pl.col("ID_FSC_long").cast(pl.String).str.strip_chars() != "")
-        )
-        .then(pl.col("ID_FSC_long").cast(pl.String).str.strip_chars())
-        .when(pl.col("ID_FSC").is_not_null())
-        .then(
-            pl.col("ID_FSC").cast(pl.String).str.strip_chars().str.replace(r"\.0$", "")
-            + pl.lit("-00")
+        pl.coalesce(
+            [
+                pl.col("ID_FSC_long").cast(pl.String).str.strip_chars(),
+                pl.col("ID_FSC").cast(pl.String).str.strip_chars() + pl.lit("-00")
+            ]
         )
-        .otherwise(None)
+        .str.replace(r"\.0$", "")
         .alias("ID_FSC"),
         pl.col("ID_IPEDS").cast(pl.Int64, strict=False).alias("unitid"),
         pl.col("ID_OPE").cast(pl.Int64, strict=False),
@@ -318,8 +310,6 @@ college_ref = (
         "ID_FSC",
         "unitid",
         "ID_OPE",
-        "opeid",
-        "super_opeid",
         "college_name",
         "college_years",
         "college_ownership",
@@ -327,8 +317,6 @@ college_ref = (
         "college_4yr",
         "college_2yr",
         "tier",
-        "tier_name",
-        "superopeid_name",
         "college_elite",
     )
 )
@@ -340,22 +328,25 @@ college_ref = (
 
 # Start from unique students. cohort_lottery is the birth cohort; cohort_18 is
 # the year the student turns 18 and anchors Sarah's Y1/Y2/... outcome windows.
-if str(APPS).endswith(".dta"):
-    apps_raw = scan_readstat(
-        APPS, missing_string_as_null=True, value_labels_as_strings=True
-    ).collect()
-else:
-    apps_raw = pl.read_csv(APPS)
+
 
 apps = (
-    apps_raw.select("sid_cepr", "cohort_lottery")
+    scan_readstat(
+        APPS, missing_string_as_null=True, value_labels_as_strings=True
+    )
+    .select("sid_cepr", "cohort_lottery")
     .drop_nulls(["sid_cepr", "cohort_lottery"])
     .with_columns(
         pl.col("sid_cepr").cast(pl.Int64),
         pl.col("cohort_lottery").cast(pl.Int64),
     )
-    .unique(["sid_cepr", "cohort_lottery"])
+    .filter(
+        pl.col("cohort_lottery").is_between(1990, 2005)
+    )
+    .sort("cohort_lottery")
+    .unique(["sid_cepr"])
     .with_columns((pl.col("cohort_lottery") + 18).alias("cohort_18"))
+    .collect()
 )
 
 # These branch-code fixes mirror the Sarah code path that maps known bad FSC
@@ -441,7 +432,11 @@ fsc_branch_fixes = [
 # The real NSC file uses raw NSC names; Sarah renames these before building
 # outcomes. This block makes those same names explicit in Polars.
 nsc_old = (
-    scan_readstat(INPUT_NSC_OLD, missing_string_as_null=True, value_labels_as_strings=True)
+    scan_readstat(INPUT_NSC_OLD, missing_string_as_null=True, value_labels_as_strings=True, schema_overrides={
+            "searchdate": pl.String
+        }
+    )
+    .collect()
     .with_columns(
         pl.lit(2023).alias("NSCdatayear"),
 
@@ -450,11 +445,41 @@ nsc_old = (
         .cast(pl.Int32)
         .alias("searchbeginyear")
     )
-    .collect()
 )
 
 nsc_new = (
-    scan_readstat(INPUT_NSC_NEW, missing_string_as_null=True, value_labels_as_strings=True)
+    pl.scan_csv(INPUT_NSC_NEW, infer_schema=False, null_values=[""], try_parse_dates=False, ignore_errors=False,)
+    .rename({
+        "Study ID": "studyid",
+        "Record Found Y/N": "recordfoundyn",
+        "College Code/Branch": "collegecodebranch",
+        "College Name": "collegename",
+        "College State": "collegestate",
+        "2-year / 4-year": "2year4year",
+        "Public / Private": "publicprivate",
+        "Enrollment Begin": "enrollmentbegin",
+        "Enrollment End": "enrollmentend",
+        "Enrollment Status": "enrollmentstatus",
+        "Class Level": "classlevel",
+        "Enrollment Major 1": "enrollmentmajor1",
+        "Enrollment CIP 1": "enrollmentcip1",
+        "Enrollment Major 2": "enrollmentmajor2",
+        "Enrollment CIP 2": "enrollmentcip2",
+        "Graduated?": "graduated",
+        "Graduation Date": "graduationdate",
+        "Degree Title": "degreetitle",
+        "Degree Major 1": "degreemajor1",
+        "Degree CIP 1": "degreecip1",
+        "Degree Major 2": "degreemajor2",
+        "Degree CIP 2": "degreecip2",
+        "Degree Major 3": "degreemajor3",
+        "Degree CIP 3": "degreecip3",
+        "Degree Major 4": "degreemajor4",
+        "Degree CIP 4": "degreecip4",
+        "College Sequence": "collegesequence",
+        "Student Age at start of term": "studentageatstartofterm",
+        
+    })
     .with_row_index("origsrt", offset=1)
     .filter(pl.col("studyid").is_not_null())
     .with_columns(
@@ -463,7 +488,7 @@ nsc_new = (
     .explode("sid_cepr")
     .filter(pl.col("sid_cepr") != "")
     .with_columns(
-        pl.col("sid_cepr").cast(pl.Int64, strict=False)
+        pl.col("sid_cepr").cast(pl.String).str.replace_all("_", "").cast(pl.Int64, strict=False)
     )
     .filter(
         pl.col("sid_cepr").is_not_null()
@@ -920,6 +945,7 @@ degree_title = pl.col("_degree_title_clean").fill_null("")
 degrees = (
     nsc.filter(pl.col("graduated") == "Y")
     .filter(pl.col("graduated_date").is_not_null())
+    .filter(pl.col("graduated_date") <= NSC_CUTOFF_DATE)
     .with_columns(
         pl.col("degree_title")
         .cast(pl.String)
@@ -1198,6 +1224,13 @@ for year in range(1, N_YEARS_OUT + 1):
             (pl.col("_by_end") * pl.col("_degree").is_in(["AA", "BA"]).cast(pl.Int8))
             .max()
             .alias(f"cmp_any_byY{year}"),
+            (
+                pl.col("_by_end")
+                * pl.col("_degree").is_in(["AA", "BA"]).cast(pl.Int8)
+                * pl.col("college_elite").fill_null(0).cast(pl.Int8)
+            )
+            .max()
+            .alias(f"cmp_elite_byY{year}"),
         )
     )
 
@@ -1256,20 +1289,14 @@ degree_outcomes = degree_outcomes.join(degree_details, on="sid_cepr", how="left"
 
 # Merge enrollment and degree outcomes back to the unique application universe.
 nsc_outcomes = (
-    apps.join(enrollment_outcomes, on="sid_cepr", how="left")
-    .join(degree_outcomes, on="sid_cepr", how="left")
+    apps.join(enrollment_outcomes, on="sid_cepr", how="left", validate="m:1")
+    .join(degree_outcomes, on="sid_cepr", how="left", validate="m:1")
 )
 
 for year in range(1, N_YEARS_OUT + 1):
-    nsc_outcomes = nsc_outcomes.with_columns(
-        (pl.col("cohort_18") + year - 1).alias("_in_start_year"),
-        pl.col("cohort_18").alias("_by_start_year"),
-        (pl.col("cohort_18") + year).alias("_by_end_year"),
-    ).with_columns(
-        pl.date(pl.col("_in_start_year"), 7, 1).alias("_in_start"),
-        pl.date(pl.col("_by_start_year"), 7, 1).alias("_by_start"),
-        pl.date(pl.col("_by_end_year"), 6, 30).alias("_by_end"),
-    )
+    in_start = pl.date(pl.col("cohort_18") + year - 1, 7, 1)
+    by_start = pl.date(pl.col("cohort_18"), 7, 1)
+    by_end = pl.date(pl.col("cohort_18") + year, 6, 30)
 
     att_in_columns = [
         column
@@ -1291,7 +1318,7 @@ for year in range(1, N_YEARS_OUT + 1):
 
     nsc_outcomes = nsc_outcomes.with_columns(
         [
-            pl.when(pl.col("_in_start") > NSC_CUTOFF_DATE)
+            pl.when(in_start > NSC_CUTOFF_DATE)
             .then(None)
             .when(pl.col(column).is_null())
             .then(0)
@@ -1302,7 +1329,7 @@ for year in range(1, N_YEARS_OUT + 1):
         ]
         +
         [
-            pl.when(pl.col("_by_start") > NSC_CUTOFF_DATE)
+            pl.when(by_start > NSC_CUTOFF_DATE)
             .then(None)
             .when(pl.col(column).is_null())
             .then(0)
@@ -1313,7 +1340,7 @@ for year in range(1, N_YEARS_OUT + 1):
         ]
         +
         [
-            pl.when(pl.col("_by_end") > NSC_CUTOFF_DATE)
+            pl.when(by_end > NSC_CUTOFF_DATE)
             .then(None)
             .when(pl.col(column).is_null())
             .then(0)
@@ -1330,20 +1357,22 @@ for year in range(1, N_YEARS_OUT + 1):
 ###########################################################
 
 outcome_columns = [
-    "att_any_inY1", "att_any_inY2", "att_any_inY3", "att_any_inY4",
-    "att_4yr_inY1", "att_4yr_inY2", "att_4yr_inY3", "att_4yr_inY4",
+    "att_any_inY1", "att_any_inY2", "att_any_inY3", "att_any_inY4", "att_any_inY5", "att_any_inY6", "att_any_inY7", "att_any_inY8",
+    "att_4yr_inY1", "att_4yr_inY2", "att_4yr_inY3", "att_4yr_inY4", "att_4yr_inY5", "att_4yr_inY6", "att_4yr_inY7", "att_4yr_inY8",
     "att_2yr_inY1", "att_2yr_inY2", "att_2yr_inY3", "att_2yr_inY4",
     "att_elite_inY1", "att_elite_inY2", "att_elite_inY3", "att_elite_inY4",
 
     "att_any_byY2", "att_any_byY3", "att_any_byY4",
     "att_4yr_byY2", "att_4yr_byY3", "att_4yr_byY4",
     "att_2yr_byY2", "att_2yr_byY3", "att_2yr_byY4", 
-    "att_elite_byY2", "att_elite_byY3", "att_elite_byY4",
+    "att_elite_byY2", "att_elite_byY3", "att_elite_byY4", "att_elite_byY5", "att_elite_byY6"
 
     "cmp_any_byY4", "cmp_any_byY5", "cmp_any_byY6", "cmp_any_byY7", "cmp_any_byY8",
     "cmp_AA_byY4", "cmp_AA_byY5", "cmp_AA_byY6", "cmp_AA_byY7", "cmp_AA_byY8",
     "cmp_BA_byY4", "cmp_BA_byY5", "cmp_BA_byY6", "cmp_BA_byY7", "cmp_BA_byY8",
 
+    "cmp_elite_byY4", "cmp_elite_byY5", "cmp_elite_byY6", "cmp_elite_byY7", "cmp_elite_byY8"
+
 
 ]