ltc-code 0.2.15__tar.gz → 0.2.17__tar.gz

{ltc_code-0.2.15 → ltc_code-0.2.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ltc-code
-Version: 0.2.15
+Version: 0.2.17
 Summary: Add your description here
 Requires-Dist: fastexcel>=0.16,<0.20
 Requires-Dist: polars>=1.36.1,<1.42

{ltc_code-0.2.15 → ltc_code-0.2.17}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.2.15"
+version = "0.2.17"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"

{ltc_code-0.2.15 → ltc_code-0.2.17}/src/ltc_code/nsc/build_nsc_outcomes.py

@@ -43,25 +43,32 @@ CHETTY_MRC_TABLE2 = next(
 )
 # Chetty/Barron college tier lookup. Numeric tier 1/2 identifies elite colleges.
 
-INPUT_NSC = PACKAGE_NSC / "input"
-# Folder for secured Census inputs that are not packaged with the code.
+CHETTY_MRC_TABLE11_CANDIDATES = [
+    RAW_NSC / "chetty" / "mrc_table11.dta",
+    RAW_NSC / "mrc_table11.dta",
+    PACKAGE_NSC.parents[3] / "College" / "chetty" / "mrc_table11.dta",
+]
+CHETTY_MRC_TABLE11 = next(
+    (path for path in CHETTY_MRC_TABLE11_CANDIDATES if path.exists()),
+    CHETTY_MRC_TABLE11_CANDIDATES[0],
+)
+# Chetty OPE-to-super-OPE crosswalk used to attach tier to NSC schools.
+
 
 APPS = INPUT_NSC / "apps.csv"
 # Application/student records; source for unique sid_cepr/cohort_lottery.
 
-NSC_RECORDS = INPUT_NSC / "nsc_records.csv"
-# Long NSC enrollment and degree records.
+INPUT_NSC = PACKAGE_NSC / "input"
+# Folder for secured Census inputs that are not packaged with the code.
+
 
 INPUT_NSC_OLD = INPUT_NSC / "nsc_records_old.dta"
 # Older NSC pull, kept so the build can append old and new NSC versions.
 
-INPUT_NSC_NEW = INPUT_NSC / "nsc_records_new.dta"
+INPUT_NSC_NEW = INPUT_NSC / "nsc_records_new.csv"
 # Newer NSC pull with studyid-based rows.
 
-OUTPUT_NSC = PACKAGE_NSC / "output"
-# Package-local output folder for NSC outcome files.
-
-OUTCOMES = OUTPUT_NSC / "nsc_outcomes.parquet"
+OUTCOMES_OUTPUT = OUTPUT_NSC / "nsc_outcomes.parquet"
 # Final student-level NSC outcomes file.
 
 
@@ -85,12 +92,15 @@ COLLEGE_TYPES = ["any", "4yr", "2yr", "elite"]
 college_crosswalk = (
     pl.read_excel(
         COLLEGE_CROSSWALK,
-        columns=["ID_IPEDS", "ID_OPE", "ID_FSC_long", "name"],
+        columns=["ID_IPEDS", "ID_OPE", "ID_FSC", "ID_FSC_long", "name"],
     )
     .with_columns(
-        pl.col("ID_FSC_long")
-        .cast(pl.String)
-        .str.strip_chars()
+        pl.coalesce(
+            [
+                pl.col("ID_FSC_long").cast(pl.String).str.strip_chars(),
+                pl.col("ID_FSC").cast(pl.String).str.strip_chars() + pl.lit("-00")
+            ]
+        )
         .str.replace(r"\.0$", "")
         .alias("ID_FSC"),
         pl.col("ID_IPEDS").cast(pl.Int64, strict=False).alias("unitid"),
@@ -98,8 +108,11 @@ college_crosswalk = (
     )
     .select("ID_FSC", "unitid", "ID_OPE", "name")
     .rename({"name": "college_name_crosswalk"})
-    .drop_nulls("ID_FSC")
+    .drop_nulls(["ID_FSC", "unitid"])
     .unique("ID_FSC", keep="first")
+    .with_columns(
+        pl.col("ID_FSC").str.slice(0, 6).cast(pl.Int64, strict=False).alias("opeid")
+    )
 )
 
 
@@ -210,24 +223,42 @@ directory = (
     .select("unitid", "college_name_directory", "years_directory", "ownership_directory")
 )
 
-# Chetty/Barron tiers are keyed at the super-OPE level. The NSC college
-# crosswalk carries OPE IDs, so joining tier onto college_ref makes the tier
-# available on each enrollment spell.
+# Chetty/Barron tiers are keyed at super-OPE level. Table 11 maps school OPE IDs
+# to those super-OPE groups, which is the same route Sarah's Stata code uses.
 chetty_tiers = (
     scan_readstat(
         CHETTY_MRC_TABLE2,
         missing_string_as_null=True,
         value_labels_as_strings=False,
     )
-    .select("super_opeid", "tier")
+    .select("super_opeid", "tier", "tier_name")
     .collect()
     .with_columns(
-        pl.col("super_opeid").cast(pl.Int64, strict=False).alias("ID_OPE"),
+        pl.col("super_opeid").cast(pl.Int64, strict=False),
         pl.col("tier").cast(pl.Int8, strict=False),
     )
-    .select("ID_OPE", "tier")
-    .drop_nulls("ID_OPE")
-    .unique("ID_OPE", keep="first")
+    .drop_nulls("super_opeid")
+    .unique("super_opeid", keep="first")
+)
+
+chetty_ope_crosswalk = (
+    scan_readstat(
+        CHETTY_MRC_TABLE11,
+        missing_string_as_null=True,
+        value_labels_as_strings=False,
+    )
+    .select("opeid", "super_opeid", "superopeid_name")
+    .collect()
+    .with_columns(
+        pl.col("opeid").cast(pl.Int64, strict=False),
+        pl.col("super_opeid").cast(pl.Int64, strict=False),
+    )
+    .drop_nulls("opeid")
+)
+
+chetty_by_opeid = (
+    chetty_ope_crosswalk.join(chetty_tiers, on="super_opeid", how="left")
+    .unique("opeid", keep="first")
 )
 
 
@@ -240,7 +271,7 @@ college_ref = (
     college_crosswalk.join(ipeds_2013, on="unitid", how="left")
     .join(ipeds_manual, on="unitid", how="left")
     .join(directory, on="unitid", how="left")
-    .join(chetty_tiers, on="ID_OPE", how="left", validate="m:1")
+    .join(chetty_by_opeid, on="opeid", how="left", validate="m:1")
     .with_columns(
         pl.coalesce(
             [
@@ -297,22 +328,25 @@ college_ref = (
 
 # Start from unique students. cohort_lottery is the birth cohort; cohort_18 is
 # the year the student turns 18 and anchors Sarah's Y1/Y2/... outcome windows.
-if str(APPS).endswith(".dta"):
-    apps_raw = scan_readstat(
-        APPS, missing_string_as_null=True, value_labels_as_strings=True
-    ).collect()
-else:
-    apps_raw = pl.read_csv(APPS)
+
 
 apps = (
-    apps_raw.select("sid_cepr", "cohort_lottery")
+    scan_readstat(
+        APPS, missing_string_as_null=True, value_labels_as_strings=True
+    )
+    .select("sid_cepr", "cohort_lottery")
     .drop_nulls(["sid_cepr", "cohort_lottery"])
     .with_columns(
         pl.col("sid_cepr").cast(pl.Int64),
         pl.col("cohort_lottery").cast(pl.Int64),
     )
-    .unique(["sid_cepr", "cohort_lottery"])
+    .filter(
+        pl.col("cohort_lottery").is_between(1990, 2005)
+    )
+    .sort("cohort_lottery")
+    .unique(["sid_cepr"])
     .with_columns((pl.col("cohort_lottery") + 18).alias("cohort_18"))
+    .collect()
 )
 
 # These branch-code fixes mirror the Sarah code path that maps known bad FSC
@@ -398,7 +432,11 @@ fsc_branch_fixes = [
 # The real NSC file uses raw NSC names; Sarah renames these before building
 # outcomes. This block makes those same names explicit in Polars.
 nsc_old = (
-    scan_readstat(INPUT_NSC_OLD, missing_string_as_null=True, value_labels_as_strings=True)
+    scan_readstat(INPUT_NSC_OLD, missing_string_as_null=True, value_labels_as_strings=True, schema_overrides={
+            "searchdate": pl.String
+        }
+    )
+    .collect()
     .with_columns(
         pl.lit(2023).alias("NSCdatayear"),
 
@@ -407,15 +445,45 @@ nsc_old = (
         .cast(pl.Int32)
         .alias("searchbeginyear")
     )
-    .collect()
 )
 
 nsc_new = (
-    scan_readstat(INPUT_NSC_NEW, missing_string_as_null=True, value_labels_as_strings=True)
+    pl.scan_csv(INPUT_NSC_NEW, infer_schema=False, null_values=[""], try_parse_dates=False, ignore_errors=False,)
+    .rename({
+        "Study ID": "studyid",
+        "Record Found Y/N": "recordfoundyn",
+        "College Code/Branch": "collegecodebranch",
+        "College Name": "collegename",
+        "College State": "collegestate",
+        "2-year / 4-year": "2year4year",
+        "Public / Private": "publicprivate",
+        "Enrollment Begin": "enrollmentbegin",
+        "Enrollment End": "enrollmentend",
+        "Enrollment Status": "enrollmentstatus",
+        "Class Level": "classlevel",
+        "Enrollment Major 1": "enrollmentmajor1",
+        "Enrollment CIP 1": "enrollmentcip1",
+        "Enrollment Major 2": "enrollmentmajor2",
+        "Enrollment CIP 2": "enrollmentcip2",
+        "Graduated?": "graduated",
+        "Graduation Date": "graduationdate",
+        "Degree Title": "degreetitle",
+        "Degree Major 1": "degreemajor1",
+        "Degree CIP 1": "degreecip1",
+        "Degree Major 2": "degreemajor2",
+        "Degree CIP 2": "degreecip2",
+        "Degree Major 3": "degreemajor3",
+        "Degree CIP 3": "degreecip3",
+        "Degree Major 4": "degreemajor4",
+        "Degree CIP 4": "degreecip4",
+        "College Sequence": "collegesequence",
+        "Student Age at start of term": "studentageatstartofterm",
+        
+    })
     .with_row_index("origsrt", offset=1)
     .filter(pl.col("studyid").is_not_null())
     .with_columns(
-        pl.col("studyid").cast(pl.String).str.replace_all("_", "").alias("sid_cepr")
+        pl.col("studyid").cast(pl.String).str.split("_").alias("sid_cepr")
     )
     .explode("sid_cepr")
     .filter(pl.col("sid_cepr") != "")
@@ -759,8 +827,15 @@ for year in range(1, N_YEARS_OUT + 1):
                 end_month, end_day = 6, 30
 
             bounded = enroll.with_columns(
-                pl.date(start_year, start_month, start_day).alias("_window_start"),
-                pl.date(end_year, end_month, end_day).alias("_window_end"),
+                start_year.alias("_window_start_year"),
+                end_year.alias("_window_end_year"),
+            ).with_columns(
+                pl.date(pl.col("_window_start_year"), start_month, start_day).alias(
+                    "_window_start"
+                ),
+                pl.date(pl.col("_window_end_year"), end_month, end_day).alias(
+                    "_window_end"
+                ),
             ).with_columns(
                 (
                     (pl.col("term_start_date") <= pl.col("_window_end"))
@@ -870,6 +945,7 @@ degree_title = pl.col("_degree_title_clean").fill_null("")
 degrees = (
     nsc.filter(pl.col("graduated") == "Y")
     .filter(pl.col("graduated_date").is_not_null())
+    .filter(pl.col("graduated_date") <= NSC_CUTOFF_DATE)
     .with_columns(
         pl.col("degree_title")
         .cast(pl.String)
@@ -1148,6 +1224,13 @@ for year in range(1, N_YEARS_OUT + 1):
             (pl.col("_by_end") * pl.col("_degree").is_in(["AA", "BA"]).cast(pl.Int8))
             .max()
             .alias(f"cmp_any_byY{year}"),
+            (
+                pl.col("_by_end")
+                * pl.col("_degree").is_in(["AA", "BA"]).cast(pl.Int8)
+                * pl.col("college_elite").fill_null(0).cast(pl.Int8)
+            )
+            .max()
+            .alias(f"cmp_elite_byY{year}"),
         )
     )
 
@@ -1206,8 +1289,8 @@ degree_outcomes = degree_outcomes.join(degree_details, on="sid_cepr", how="left"
 
 # Merge enrollment and degree outcomes back to the unique application universe.
 nsc_outcomes = (
-    apps.join(enrollment_outcomes, on="sid_cepr", how="left")
-    .join(degree_outcomes, on="sid_cepr", how="left")
+    apps.join(enrollment_outcomes, on="sid_cepr", how="left", validate="m:1")
+    .join(degree_outcomes, on="sid_cepr", how="left", validate="m:1")
 )
 
 for year in range(1, N_YEARS_OUT + 1):
@@ -1274,20 +1357,22 @@ for year in range(1, N_YEARS_OUT + 1):
 ###########################################################
 
 outcome_columns = [
-    "att_any_inY1", "att_any_inY2", "att_any_inY3", "att_any_inY4",
-    "att_4yr_inY1", "att_4yr_inY2", "att_4yr_inY3", "att_4yr_inY4",
+    "att_any_inY1", "att_any_inY2", "att_any_inY3", "att_any_inY4", "att_any_inY5", "att_any_inY6", "att_any_inY7", "att_any_inY8",
+    "att_4yr_inY1", "att_4yr_inY2", "att_4yr_inY3", "att_4yr_inY4", "att_4yr_inY5", "att_4yr_inY6", "att_4yr_inY7", "att_4yr_inY8",
     "att_2yr_inY1", "att_2yr_inY2", "att_2yr_inY3", "att_2yr_inY4",
     "att_elite_inY1", "att_elite_inY2", "att_elite_inY3", "att_elite_inY4",
 
     "att_any_byY2", "att_any_byY3", "att_any_byY4",
     "att_4yr_byY2", "att_4yr_byY3", "att_4yr_byY4",
     "att_2yr_byY2", "att_2yr_byY3", "att_2yr_byY4", 
-    "att_elite_byY2", "att_elite_byY3", "att_elite_byY4",
+    "att_elite_byY2", "att_elite_byY3", "att_elite_byY4", "att_elite_byY5", "att_elite_byY6"
 
     "cmp_any_byY4", "cmp_any_byY5", "cmp_any_byY6", "cmp_any_byY7", "cmp_any_byY8",
     "cmp_AA_byY4", "cmp_AA_byY5", "cmp_AA_byY6", "cmp_AA_byY7", "cmp_AA_byY8",
     "cmp_BA_byY4", "cmp_BA_byY5", "cmp_BA_byY6", "cmp_BA_byY7", "cmp_BA_byY8",
 
+    "cmp_elite_byY4", "cmp_elite_byY5", "cmp_elite_byY6", "cmp_elite_byY7", "cmp_elite_byY8"
+
 
 ]