ltc-code 0.1.85__tar.gz → 0.1.87__tar.gz

{ltc_code-0.1.85 → ltc_code-0.1.87}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ltc-code
-Version: 0.1.85
+Version: 0.1.87
 Summary: Add your description here
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown

{ltc_code-0.1.85 → ltc_code-0.1.87}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.1.85"
+version = "0.1.87"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"

{ltc_code-0.1.85 → ltc_code-0.1.87}/src/ltc_code/june2.py

@@ -20,6 +20,14 @@ def _second_word_expr(col: str) -> pl.Expr:
     return pl.col(col).cast(pl.String).str.split(" ").list.get(1, null_on_oob=True)
 
 
+def _compact_expr(expr: pl.Expr) -> pl.Expr:
+    return (
+        expr.cast(pl.String)
+        .str.to_uppercase()
+        .str.replace_all(r"[^A-Z]", "")
+    )
+
+
 def _build_lookup(
     census: pl.DataFrame,
     *,
@@ -27,7 +35,13 @@ def _build_lookup(
     lname_expr: pl.Expr,
     dob_col: str,
     label: str,
+    compact: bool = False,
 ) -> pl.DataFrame:
+
+    if compact:
+        fname_expr = _compact_expr(fname_expr)
+        lname_expr = _compact_expr(lname_expr)
+
     lookup = (
         census
         .select(
@@ -37,6 +51,10 @@ def _build_lookup(
             pl.col("sid_cepr"),
         )
         .drop_nulls(["_fname_key", "_lname_key", "_dob_key", "sid_cepr"])
+        .filter(
+            pl.col("_fname_key").cast(pl.String).str.len_chars().gt(0)
+            & pl.col("_lname_key").cast(pl.String).str.len_chars().gt(0)
+        )
         .group_by(["_fname_key", "_lname_key", "_dob_key"])
         .agg(pl.col("sid_cepr").unique().alias("_sids"))
         .select(
@@ -57,13 +75,16 @@ def _add_name_lookups(
     *,
     prefix: str,
     dob_col: str,
+    compact: bool = False,
 ) -> None:
+
     lookups[f"{prefix}exact"] = _build_lookup(
         census,
         fname_expr=pl.col("fname"),
         lname_expr=pl.col("lname"),
         dob_col=dob_col,
         label=f"{prefix}exact",
+        compact=compact,
     )
 
     lookups[f"{prefix}mname"] = _build_lookup(
@@ -72,6 +93,7 @@ def _add_name_lookups(
         lname_expr=pl.col("lname"),
         dob_col=dob_col,
         label=f"{prefix}right fname + mname",
+        compact=compact,
     )
 
     lookups[f"{prefix}mname_lname"] = _build_lookup(
@@ -80,6 +102,7 @@ def _add_name_lookups(
         lname_expr=pl.concat_str([pl.col("mname"), pl.col("lname")], separator=" "),
         dob_col=dob_col,
         label=f"{prefix}right mname + lname",
+        compact=compact,
     )
 
     lookups[f"{prefix}mname_nospace"] = _build_lookup(
@@ -88,6 +111,7 @@ def _add_name_lookups(
         lname_expr=pl.col("lname"),
         dob_col=dob_col,
         label=f"{prefix}right fname + mname no space",
+        compact=compact,
     )
 
     lookups[f"{prefix}suffix"] = _build_lookup(
@@ -96,6 +120,7 @@ def _add_name_lookups(
         lname_expr=pl.concat_str([pl.col("lname"), pl.col("suffix")], separator=" "),
         dob_col=dob_col,
         label=f"{prefix}right lname + suffix",
+        compact=compact,
     )
 
     lookups[f"{prefix}suffix_fname"] = _build_lookup(
@@ -104,6 +129,7 @@ def _add_name_lookups(
         lname_expr=pl.col("lname"),
         dob_col=dob_col,
         label=f"{prefix}right fname + suffix",
+        compact=compact,
     )
 
     lookups[f"{prefix}suffix_fname_nospace"] = _build_lookup(
@@ -112,6 +138,7 @@ def _add_name_lookups(
         lname_expr=pl.col("lname"),
         dob_col=dob_col,
         label=f"{prefix}right fname + suffix no space",
+        compact=compact,
     )
 
     lookups[f"{prefix}suffix_lname_nospace"] = _build_lookup(
@@ -120,6 +147,7 @@ def _add_name_lookups(
         lname_expr=pl.concat_str([pl.col("lname"), pl.col("suffix")], separator=""),
         dob_col=dob_col,
         label=f"{prefix}right lname + suffix no space",
+        compact=compact,
     )
 
     lookups[f"{prefix}fname_first_word"] = _build_lookup(
@@ -128,6 +156,7 @@ def _add_name_lookups(
         lname_expr=pl.col("lname"),
         dob_col=dob_col,
         label=f"{prefix}right fname first word",
+        compact=compact,
     )
 
     lookups[f"{prefix}lname_first_word"] = _build_lookup(
@@ -136,6 +165,7 @@ def _add_name_lookups(
         lname_expr=_first_word_expr("lname"),
         dob_col=dob_col,
         label=f"{prefix}right lname first word",
+        compact=compact,
     )
 
     lookups[f"{prefix}lname_second_word"] = _build_lookup(
@@ -144,6 +174,7 @@ def _add_name_lookups(
         lname_expr=_second_word_expr("lname"),
         dob_col=dob_col,
         label=f"{prefix}right lname second word",
+        compact=compact,
     )
 
 
@@ -223,26 +254,23 @@ def build_census_lookups(*, cmo_name: str) -> dict[str, pl.DataFrame]:
 
     lookups = {}
 
-    _add_name_lookups(
-        lookups,
-        census,
-        prefix="",
-        dob_col="dob",
-    )
+    # Normal full-DOB lookups.
+    _add_name_lookups(lookups, census, prefix="", dob_col="dob", compact=False)
 
-    _add_name_lookups(
-        lookups,
-        census,
-        prefix="md_",
-        dob_col="dob_md",
-    )
+    # Compact full-DOB lookups.
+    _add_name_lookups(lookups, census, prefix="compact_", dob_col="dob", compact=True)
 
-    _add_name_lookups(
-        lookups,
-        census,
-        prefix="mdi_",
-        dob_col="dob_imp_md",
-    )
+    # Normal month-day lookups using dob.
+    _add_name_lookups(lookups, census, prefix="md_", dob_col="dob_md", compact=False)
+
+    # Compact month-day lookups using dob.
+    _add_name_lookups(lookups, census, prefix="compact_md_", dob_col="dob_md", compact=True)
+
+    # Normal month-day lookups using dob_imp.
+    _add_name_lookups(lookups, census, prefix="mdi_", dob_col="dob_imp_md", compact=False)
+
+    # Compact month-day lookups using dob_imp.
+    _add_name_lookups(lookups, census, prefix="compact_mdi_", dob_col="dob_imp_md", compact=True)
 
     lookups["dob_imp"] = _build_lookup(
         census,
@@ -252,6 +280,15 @@ def build_census_lookups(*, cmo_name: str) -> dict[str, pl.DataFrame]:
         label="dob_imp",
     )
 
+    lookups["compact_dob_imp"] = _build_lookup(
+        census,
+        fname_expr=pl.col("fname"),
+        lname_expr=pl.col("lname"),
+        dob_col="dob_imp",
+        label="compact dob_imp",
+        compact=True,
+    )
+
     for offset, key in [
         ("-1y", "dob_imp_minus_1"),
         ("1y", "dob_imp_plus_1"),
@@ -266,6 +303,15 @@ def build_census_lookups(*, cmo_name: str) -> dict[str, pl.DataFrame]:
             label=key,
         )
 
+        lookups[f"compact_{key}"] = _build_lookup(
+            census.with_columns(pl.col("dob_imp").dt.offset_by(offset).alias(key)),
+            fname_expr=pl.col("fname"),
+            lname_expr=pl.col("lname"),
+            dob_col=key,
+            label=f"compact {key}",
+            compact=True,
+        )
+
     return lookups
 
 
@@ -277,9 +323,15 @@ def _run_match_stage(
     lname_expr: pl.Expr,
     dob_expr: pl.Expr,
     label: str,
+    compact: bool = False,
 ) -> tuple[pl.DataFrame, pl.DataFrame]:
+
     before = len(unmatched)
 
+    if compact:
+        fname_expr = _compact_expr(fname_expr)
+        lname_expr = _compact_expr(lname_expr)
+
     stage = (
         unmatched
         .with_columns(
@@ -312,7 +364,9 @@ def _run_right_side_name_stages(
     prefix: str,
     dob_expr: pl.Expr,
     label_prefix: str,
+    compact: bool = False,
 ) -> pl.DataFrame:
+
     for label, key in [
         ("left exact -> right fname + mname", f"{prefix}mname"),
         ("left exact -> right mname + lname", f"{prefix}mname_lname"),
@@ -332,6 +386,7 @@ def _run_right_side_name_stages(
             lname_expr=pl.col(f"{cols['lname']}_clean"),
             dob_expr=dob_expr,
             label=f"{label_prefix}{label}",
+            compact=compact,
         )
         matched_frames.append(matched)
 
@@ -347,7 +402,9 @@ def _run_left_side_name_stages(
     exact_key: str,
     dob_expr: pl.Expr,
     label_prefix: str,
+    compact: bool = False,
 ) -> pl.DataFrame:
+
     for label, fname_expr, lname_expr in [
         (
             "left fname first word -> right exact",
@@ -372,6 +429,7 @@ def _run_left_side_name_stages(
             lname_expr=lname_expr,
             dob_expr=dob_expr,
             label=f"{label_prefix}{label}",
+            compact=compact,
         )
         matched_frames.append(matched)
 
@@ -411,6 +469,7 @@ def _run_left_side_name_stages(
                 lname_expr=lname_expr,
                 dob_expr=dob_expr,
                 label=f"{label_prefix}{label}",
+                compact=compact,
             )
             matched_frames.append(matched)
 
@@ -458,6 +517,7 @@ def _run_left_side_name_stages(
                 lname_expr=lname_expr,
                 dob_expr=dob_expr,
                 label=f"{label_prefix}{label}",
+                compact=compact,
             )
             matched_frames.append(matched)
 
@@ -496,6 +556,7 @@ def lookup_sid_cepr(
     matched_frames = []
     unmatched = current
 
+    # Exact full DOB.
     matched, unmatched = _run_match_stage(
         unmatched,
         lookup=lookups["exact"],
@@ -506,6 +567,18 @@ def lookup_sid_cepr(
     )
     matched_frames.append(matched)
 
+    # Compact exact full DOB. Handles SALDAN A vs SALDANA, apostrophes, hyphens, backticks, etc.
+    matched, unmatched = _run_match_stage(
+        unmatched,
+        lookup=lookups["compact_exact"],
+        fname_expr=pl.col(f"{cols['fname']}_clean"),
+        lname_expr=pl.col(f"{cols['lname']}_clean"),
+        dob_expr=pl.col(f"{cols['dob']}_clean"),
+        label="compact exact",
+        compact=True,
+    )
+    matched_frames.append(matched)
+
     unmatched = _run_right_side_name_stages(
         unmatched,
         matched_frames,
@@ -514,6 +587,7 @@ def lookup_sid_cepr(
         prefix="",
         dob_expr=pl.col(f"{cols['dob']}_clean"),
         label_prefix="",
+        compact=False,
     )
 
     unmatched = _run_left_side_name_stages(
@@ -524,6 +598,29 @@ def lookup_sid_cepr(
         exact_key="exact",
         dob_expr=pl.col(f"{cols['dob']}_clean"),
         label_prefix="",
+        compact=False,
+    )
+
+    unmatched = _run_right_side_name_stages(
+        unmatched,
+        matched_frames,
+        cols=cols,
+        lookups=lookups,
+        prefix="compact_",
+        dob_expr=pl.col(f"{cols['dob']}_clean"),
+        label_prefix="compact ",
+        compact=True,
+    )
+
+    unmatched = _run_left_side_name_stages(
+        unmatched,
+        matched_frames,
+        cols=cols,
+        lookups=lookups,
+        exact_key="compact_exact",
+        dob_expr=pl.col(f"{cols['dob']}_clean"),
+        label_prefix="compact ",
+        compact=True,
     )
 
     for key in [
@@ -532,6 +629,11 @@ def lookup_sid_cepr(
         "dob_imp_plus_1",
         "dob_imp_minus_2",
         "dob_imp_plus_2",
+        "compact_dob_imp",
+        "compact_dob_imp_minus_1",
+        "compact_dob_imp_plus_1",
+        "compact_dob_imp_minus_2",
+        "compact_dob_imp_plus_2",
     ]:
         matched, unmatched = _run_match_stage(
             unmatched,
@@ -540,9 +642,11 @@ def lookup_sid_cepr(
             lname_expr=pl.col(f"{cols['lname']}_clean"),
             dob_expr=pl.col(f"{cols['dob']}_clean"),
             label=key,
+            compact=key.startswith("compact_"),
         )
         matched_frames.append(matched)
 
+    # Month-day fallback on RHS dob.
     matched, unmatched = _run_match_stage(
         unmatched,
         lookup=lookups["md_exact"],
@@ -553,6 +657,17 @@ def lookup_sid_cepr(
     )
     matched_frames.append(matched)
 
+    matched, unmatched = _run_match_stage(
+        unmatched,
+        lookup=lookups["compact_md_exact"],
+        fname_expr=pl.col(f"{cols['fname']}_clean"),
+        lname_expr=pl.col(f"{cols['lname']}_clean"),
+        dob_expr=pl.col("_lhs_dob_md"),
+        label="compact month-day exact dob",
+        compact=True,
+    )
+    matched_frames.append(matched)
+
     unmatched = _run_right_side_name_stages(
         unmatched,
         matched_frames,
@@ -561,6 +676,7 @@ def lookup_sid_cepr(
         prefix="md_",
         dob_expr=pl.col("_lhs_dob_md"),
         label_prefix="month-day dob: ",
+        compact=False,
     )
 
     unmatched = _run_left_side_name_stages(
@@ -571,8 +687,32 @@ def lookup_sid_cepr(
         exact_key="md_exact",
         dob_expr=pl.col("_lhs_dob_md"),
         label_prefix="month-day dob: ",
+        compact=False,
     )
 
+    unmatched = _run_right_side_name_stages(
+        unmatched,
+        matched_frames,
+        cols=cols,
+        lookups=lookups,
+        prefix="compact_md_",
+        dob_expr=pl.col("_lhs_dob_md"),
+        label_prefix="compact month-day dob: ",
+        compact=True,
+    )
+
+    unmatched = _run_left_side_name_stages(
+        unmatched,
+        matched_frames,
+        cols=cols,
+        lookups=lookups,
+        exact_key="compact_md_exact",
+        dob_expr=pl.col("_lhs_dob_md"),
+        label_prefix="compact month-day dob: ",
+        compact=True,
+    )
+
+    # Month-day fallback on RHS dob_imp.
     matched, unmatched = _run_match_stage(
         unmatched,
         lookup=lookups["mdi_exact"],
@@ -583,6 +723,17 @@ def lookup_sid_cepr(
     )
     matched_frames.append(matched)
 
+    matched, unmatched = _run_match_stage(
+        unmatched,
+        lookup=lookups["compact_mdi_exact"],
+        fname_expr=pl.col(f"{cols['fname']}_clean"),
+        lname_expr=pl.col(f"{cols['lname']}_clean"),
+        dob_expr=pl.col("_lhs_dob_md"),
+        label="compact month-day dob_imp exact",
+        compact=True,
+    )
+    matched_frames.append(matched)
+
     unmatched = _run_right_side_name_stages(
         unmatched,
         matched_frames,
@@ -591,6 +742,7 @@ def lookup_sid_cepr(
         prefix="mdi_",
         dob_expr=pl.col("_lhs_dob_md"),
         label_prefix="month-day dob_imp: ",
+        compact=False,
     )
 
     unmatched = _run_left_side_name_stages(
@@ -601,6 +753,29 @@ def lookup_sid_cepr(
         exact_key="mdi_exact",
         dob_expr=pl.col("_lhs_dob_md"),
         label_prefix="month-day dob_imp: ",
+        compact=False,
+    )
+
+    unmatched = _run_right_side_name_stages(
+        unmatched,
+        matched_frames,
+        cols=cols,
+        lookups=lookups,
+        prefix="compact_mdi_",
+        dob_expr=pl.col("_lhs_dob_md"),
+        label_prefix="compact month-day dob_imp: ",
+        compact=True,
+    )
+
+    unmatched = _run_left_side_name_stages(
+        unmatched,
+        matched_frames,
+        cols=cols,
+        lookups=lookups,
+        exact_key="compact_mdi_exact",
+        dob_expr=pl.col("_lhs_dob_md"),
+        label_prefix="compact month-day dob_imp: ",
+        compact=True,
     )
 
     result = (

ltc_code-0.1.87/src/ltc_code/schema_mapping.py

@@ -0,0 +1,63 @@
+from typing import Union, List
+import polars as pl
+
+
+FINAL_APPS_SCHEMA = [
+    "cmo_name",
+    "cmo_region",
+    "ren_num",
+    "sid_cepr",
+    "school_name",
+    "school_set",
+    "school_year",
+    "entry_grade_clean",
+    "school_accept",
+    "offer",
+    "initial_offer",
+    "waitlist_offer",
+    "priority_group",
+    "priority_group_name",
+    "risk_id",
+    "birth_cohort",
+    "age",
+    "enroll",
+    "offer_accepted",
+    "enroll_years",
+    "enroll_school",
+    "withdraw",
+    "application_cancel",
+    "waitlist",
+    "waitlist_number",
+    "lottery_number",
+    "late_app",
+    "offer_date",
+    "application_date",
+    "sibling",
+    "staff",
+    "zoned",
+]
+
+
+
+def keep_final_apps_schema(
+    df: Union[pl.DataFrame, pl.LazyFrame],
+) -> Union[pl.DataFrame, pl.LazyFrame]:
+
+    cols = (
+        df.columns
+        if isinstance(df, pl.DataFrame)
+        else df.collect_schema().names()
+    )
+
+    keep_cols = [
+        col for col in FINAL_APPS_SCHEMA
+        if col in cols
+    ]
+
+    return df.select(keep_cols)
+
+
+df = (
+    df
+    .pipe(keep_final_apps_schema)
+)