ltc-code 0.1.84__tar.gz → 0.1.85__tar.gz

{ltc_code-0.1.84 → ltc_code-0.1.85}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ltc-code
-Version: 0.1.84
+Version: 0.1.85
 Summary: Add your description here
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown

{ltc_code-0.1.84 → ltc_code-0.1.85}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.1.84"
+version = "0.1.85"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"

ltc_code-0.1.85/src/ltc_code/june2.py

@@ -0,0 +1,619 @@
+def _parse_dob_expr(col: str) -> pl.Expr:
+    return pl.coalesce(
+        [
+            pl.col(col).cast(pl.String).str.strptime(pl.Date, "%m/%d/%Y", strict=False),
+            pl.col(col).cast(pl.String).str.strptime(pl.Date, "%m/%d/%y", strict=False),
+            pl.col(col).cast(pl.String).str.strptime(pl.Date, "%Y-%m-%d", strict=False),
+        ]
+    )
+
+
+def _month_day_expr(col: str) -> pl.Expr:
+    return pl.col(col).dt.strftime("%m-%d")
+
+
+def _first_word_expr(col: str) -> pl.Expr:
+    return pl.col(col).cast(pl.String).str.split(" ").list.first()
+
+
+def _second_word_expr(col: str) -> pl.Expr:
+    return pl.col(col).cast(pl.String).str.split(" ").list.get(1, null_on_oob=True)
+
+
+def _build_lookup(
+    census: pl.DataFrame,
+    *,
+    fname_expr: pl.Expr,
+    lname_expr: pl.Expr,
+    dob_col: str,
+    label: str,
+) -> pl.DataFrame:
+    lookup = (
+        census
+        .select(
+            fname_expr.alias("_fname_key"),
+            lname_expr.alias("_lname_key"),
+            pl.col(dob_col).alias("_dob_key"),
+            pl.col("sid_cepr"),
+        )
+        .drop_nulls(["_fname_key", "_lname_key", "_dob_key", "sid_cepr"])
+        .group_by(["_fname_key", "_lname_key", "_dob_key"])
+        .agg(pl.col("sid_cepr").unique().alias("_sids"))
+        .select(
+            "_fname_key",
+            "_lname_key",
+            "_dob_key",
+            pl.col("_sids").list.sort().list.first().alias("sid_cepr"),
+        )
+    )
+
+    print(f"built lookup: {label} ({len(lookup):,} usable keys)")
+    return lookup
+
+
+def _add_name_lookups(
+    lookups: dict[str, pl.DataFrame],
+    census: pl.DataFrame,
+    *,
+    prefix: str,
+    dob_col: str,
+) -> None:
+    lookups[f"{prefix}exact"] = _build_lookup(
+        census,
+        fname_expr=pl.col("fname"),
+        lname_expr=pl.col("lname"),
+        dob_col=dob_col,
+        label=f"{prefix}exact",
+    )
+
+    lookups[f"{prefix}mname"] = _build_lookup(
+        census,
+        fname_expr=pl.concat_str([pl.col("fname"), pl.col("mname")], separator=" "),
+        lname_expr=pl.col("lname"),
+        dob_col=dob_col,
+        label=f"{prefix}right fname + mname",
+    )
+
+    lookups[f"{prefix}mname_lname"] = _build_lookup(
+        census,
+        fname_expr=pl.col("fname"),
+        lname_expr=pl.concat_str([pl.col("mname"), pl.col("lname")], separator=" "),
+        dob_col=dob_col,
+        label=f"{prefix}right mname + lname",
+    )
+
+    lookups[f"{prefix}mname_nospace"] = _build_lookup(
+        census,
+        fname_expr=pl.concat_str([pl.col("fname"), pl.col("mname")], separator=""),
+        lname_expr=pl.col("lname"),
+        dob_col=dob_col,
+        label=f"{prefix}right fname + mname no space",
+    )
+
+    lookups[f"{prefix}suffix"] = _build_lookup(
+        census,
+        fname_expr=pl.col("fname"),
+        lname_expr=pl.concat_str([pl.col("lname"), pl.col("suffix")], separator=" "),
+        dob_col=dob_col,
+        label=f"{prefix}right lname + suffix",
+    )
+
+    lookups[f"{prefix}suffix_fname"] = _build_lookup(
+        census,
+        fname_expr=pl.concat_str([pl.col("fname"), pl.col("suffix")], separator=" "),
+        lname_expr=pl.col("lname"),
+        dob_col=dob_col,
+        label=f"{prefix}right fname + suffix",
+    )
+
+    lookups[f"{prefix}suffix_fname_nospace"] = _build_lookup(
+        census,
+        fname_expr=pl.concat_str([pl.col("fname"), pl.col("suffix")], separator=""),
+        lname_expr=pl.col("lname"),
+        dob_col=dob_col,
+        label=f"{prefix}right fname + suffix no space",
+    )
+
+    lookups[f"{prefix}suffix_lname_nospace"] = _build_lookup(
+        census,
+        fname_expr=pl.col("fname"),
+        lname_expr=pl.concat_str([pl.col("lname"), pl.col("suffix")], separator=""),
+        dob_col=dob_col,
+        label=f"{prefix}right lname + suffix no space",
+    )
+
+    lookups[f"{prefix}fname_first_word"] = _build_lookup(
+        census,
+        fname_expr=_first_word_expr("fname"),
+        lname_expr=pl.col("lname"),
+        dob_col=dob_col,
+        label=f"{prefix}right fname first word",
+    )
+
+    lookups[f"{prefix}lname_first_word"] = _build_lookup(
+        census,
+        fname_expr=pl.col("fname"),
+        lname_expr=_first_word_expr("lname"),
+        dob_col=dob_col,
+        label=f"{prefix}right lname first word",
+    )
+
+    lookups[f"{prefix}lname_second_word"] = _build_lookup(
+        census,
+        fname_expr=pl.col("fname"),
+        lname_expr=_second_word_expr("lname"),
+        dob_col=dob_col,
+        label=f"{prefix}right lname second word",
+    )
+
+
+def build_census_lookups(*, cmo_name: str) -> dict[str, pl.DataFrame]:
+    annual_frames = []
+
+    for year in range(1994, 2023):
+        path = CENSUS_STUDENTS / f"census_student_{year}.csv"
+
+        annual = (
+            pl.scan_csv(
+                path,
+                infer_schema=False,
+                null_values=[],
+                try_parse_dates=False,
+                ignore_errors=False,
+            )
+            .select(
+                "cmo_code",
+                "sid_cepr",
+                "fname_clean",
+                "lname_clean",
+                "mname_clean",
+                "suff_clean",
+                "birthdate_clean",
+                "birthdate_imp",
+            )
+            .rename(
+                {
+                    "fname_clean": "fname",
+                    "lname_clean": "lname",
+                    "mname_clean": "mname",
+                    "suff_clean": "suffix",
+                    "birthdate_clean": "dob",
+                    "birthdate_imp": "dob_imp",
+                }
+            )
+            .with_columns(
+                pl.col("cmo_code").replace(cmo_map).alias("cmo_name")
+            )
+            .filter(pl.col("cmo_name") == cmo_name)
+            .with_columns(
+                *clean_name("fname"),
+                *clean_name("lname"),
+                *clean_other_name("mname"),
+                *clean_other_name("suffix"),
+            )
+            .with_columns(*clean_dob(col="dob"))
+            .with_columns(*clean_dob(col="dob_imp"))
+            .with_columns(
+                _parse_dob_expr("dob_clean").alias("dob"),
+                _parse_dob_expr("dob_imp_clean").alias("dob_imp"),
+            )
+            .select(
+                "sid_cepr",
+                pl.col("fname_clean").alias("fname"),
+                pl.col("lname_clean").alias("lname"),
+                pl.col("mname_clean").alias("mname"),
+                pl.col("suffix_clean").alias("suffix"),
+                "dob",
+                "dob_imp",
+            )
+        )
+
+        annual_frames.append(annual)
+
+    census = (
+        pl.concat(annual_frames, how="vertical_relaxed")
+        .collect()
+        .with_columns(
+            _month_day_expr("dob").alias("dob_md"),
+            _month_day_expr("dob_imp").alias("dob_imp_md"),
+        )
+    )
+
+    print(f"census rows after CMO filter: {len(census):,}")
+
+    lookups = {}
+
+    _add_name_lookups(
+        lookups,
+        census,
+        prefix="",
+        dob_col="dob",
+    )
+
+    _add_name_lookups(
+        lookups,
+        census,
+        prefix="md_",
+        dob_col="dob_md",
+    )
+
+    _add_name_lookups(
+        lookups,
+        census,
+        prefix="mdi_",
+        dob_col="dob_imp_md",
+    )
+
+    lookups["dob_imp"] = _build_lookup(
+        census,
+        fname_expr=pl.col("fname"),
+        lname_expr=pl.col("lname"),
+        dob_col="dob_imp",
+        label="dob_imp",
+    )
+
+    for offset, key in [
+        ("-1y", "dob_imp_minus_1"),
+        ("1y", "dob_imp_plus_1"),
+        ("-2y", "dob_imp_minus_2"),
+        ("2y", "dob_imp_plus_2"),
+    ]:
+        lookups[key] = _build_lookup(
+            census.with_columns(pl.col("dob_imp").dt.offset_by(offset).alias(key)),
+            fname_expr=pl.col("fname"),
+            lname_expr=pl.col("lname"),
+            dob_col=key,
+            label=key,
+        )
+
+    return lookups
+
+
+def _run_match_stage(
+    unmatched: pl.DataFrame,
+    *,
+    lookup: pl.DataFrame,
+    fname_expr: pl.Expr,
+    lname_expr: pl.Expr,
+    dob_expr: pl.Expr,
+    label: str,
+) -> tuple[pl.DataFrame, pl.DataFrame]:
+    before = len(unmatched)
+
+    stage = (
+        unmatched
+        .with_columns(
+            fname_expr.alias("_fname_key"),
+            lname_expr.alias("_lname_key"),
+            dob_expr.alias("_dob_key"),
+        )
+        .join(
+            lookup,
+            on=["_fname_key", "_lname_key", "_dob_key"],
+            how="left",
+            validate="m:1",
+        )
+        .drop(["_fname_key", "_lname_key", "_dob_key"])
+    )
+
+    matched = stage.filter(pl.col("sid_cepr").is_not_null())
+    unmatched = stage.filter(pl.col("sid_cepr").is_null()).drop("sid_cepr")
+
+    print(f"{label}: matched {len(matched):,}/{before:,}")
+    return matched, unmatched
+
+
+def _run_right_side_name_stages(
+    unmatched: pl.DataFrame,
+    matched_frames: list[pl.DataFrame],
+    *,
+    cols: Mapping[str, str],
+    lookups: dict[str, pl.DataFrame],
+    prefix: str,
+    dob_expr: pl.Expr,
+    label_prefix: str,
+) -> pl.DataFrame:
+    for label, key in [
+        ("left exact -> right fname + mname", f"{prefix}mname"),
+        ("left exact -> right mname + lname", f"{prefix}mname_lname"),
+        ("left exact -> right fname + mname no space", f"{prefix}mname_nospace"),
+        ("left exact -> right lname + suffix", f"{prefix}suffix"),
+        ("left exact -> right fname + suffix", f"{prefix}suffix_fname"),
+        ("left exact -> right fname + suffix no space", f"{prefix}suffix_fname_nospace"),
+        ("left exact -> right lname + suffix no space", f"{prefix}suffix_lname_nospace"),
+        ("left exact -> right fname first word", f"{prefix}fname_first_word"),
+        ("left exact -> right lname first word", f"{prefix}lname_first_word"),
+        ("left exact -> right lname second word", f"{prefix}lname_second_word"),
+    ]:
+        matched, unmatched = _run_match_stage(
+            unmatched,
+            lookup=lookups[key],
+            fname_expr=pl.col(f"{cols['fname']}_clean"),
+            lname_expr=pl.col(f"{cols['lname']}_clean"),
+            dob_expr=dob_expr,
+            label=f"{label_prefix}{label}",
+        )
+        matched_frames.append(matched)
+
+    return unmatched
+
+
+def _run_left_side_name_stages(
+    unmatched: pl.DataFrame,
+    matched_frames: list[pl.DataFrame],
+    *,
+    cols: Mapping[str, str],
+    lookups: dict[str, pl.DataFrame],
+    exact_key: str,
+    dob_expr: pl.Expr,
+    label_prefix: str,
+) -> pl.DataFrame:
+    for label, fname_expr, lname_expr in [
+        (
+            "left fname first word -> right exact",
+            _first_word_expr(f"{cols['fname']}_clean"),
+            pl.col(f"{cols['lname']}_clean"),
+        ),
+        (
+            "left lname first word -> right exact",
+            pl.col(f"{cols['fname']}_clean"),
+            _first_word_expr(f"{cols['lname']}_clean"),
+        ),
+        (
+            "left lname second word -> right exact",
+            pl.col(f"{cols['fname']}_clean"),
+            _second_word_expr(f"{cols['lname']}_clean"),
+        ),
+    ]:
+        matched, unmatched = _run_match_stage(
+            unmatched,
+            lookup=lookups[exact_key],
+            fname_expr=fname_expr,
+            lname_expr=lname_expr,
+            dob_expr=dob_expr,
+            label=f"{label_prefix}{label}",
+        )
+        matched_frames.append(matched)
+
+    if "mname" in cols:
+        unmatched = unmatched.with_columns(*clean_other_name(cols["mname"]))
+
+        for label, fname_expr, lname_expr in [
+            (
+                "left fname + mname -> right exact",
+                pl.concat_str(
+                    [pl.col(f"{cols['fname']}_clean"), pl.col(f"{cols['mname']}_clean")],
+                    separator=" ",
+                ),
+                pl.col(f"{cols['lname']}_clean"),
+            ),
+            (
+                "left mname + lname -> right exact",
+                pl.col(f"{cols['fname']}_clean"),
+                pl.concat_str(
+                    [pl.col(f"{cols['mname']}_clean"), pl.col(f"{cols['lname']}_clean")],
+                    separator=" ",
+                ),
+            ),
+            (
+                "left fname + mname no space -> right exact",
+                pl.concat_str(
+                    [pl.col(f"{cols['fname']}_clean"), pl.col(f"{cols['mname']}_clean")],
+                    separator="",
+                ),
+                pl.col(f"{cols['lname']}_clean"),
+            ),
+        ]:
+            matched, unmatched = _run_match_stage(
+                unmatched,
+                lookup=lookups[exact_key],
+                fname_expr=fname_expr,
+                lname_expr=lname_expr,
+                dob_expr=dob_expr,
+                label=f"{label_prefix}{label}",
+            )
+            matched_frames.append(matched)
+
+    if "suffix" in cols:
+        unmatched = unmatched.with_columns(*clean_other_name(cols["suffix"]))
+
+        for label, fname_expr, lname_expr in [
+            (
+                "left fname + suffix -> right exact",
+                pl.concat_str(
+                    [pl.col(f"{cols['fname']}_clean"), pl.col(f"{cols['suffix']}_clean")],
+                    separator=" ",
+                ),
+                pl.col(f"{cols['lname']}_clean"),
+            ),
+            (
+                "left fname + suffix no space -> right exact",
+                pl.concat_str(
+                    [pl.col(f"{cols['fname']}_clean"), pl.col(f"{cols['suffix']}_clean")],
+                    separator="",
+                ),
+                pl.col(f"{cols['lname']}_clean"),
+            ),
+            (
+                "left lname + suffix -> right exact",
+                pl.col(f"{cols['fname']}_clean"),
+                pl.concat_str(
+                    [pl.col(f"{cols['lname']}_clean"), pl.col(f"{cols['suffix']}_clean")],
+                    separator=" ",
+                ),
+            ),
+            (
+                "left lname + suffix no space -> right exact",
+                pl.col(f"{cols['fname']}_clean"),
+                pl.concat_str(
+                    [pl.col(f"{cols['lname']}_clean"), pl.col(f"{cols['suffix']}_clean")],
+                    separator="",
+                ),
+            ),
+        ]:
+            matched, unmatched = _run_match_stage(
+                unmatched,
+                lookup=lookups[exact_key],
+                fname_expr=fname_expr,
+                lname_expr=lname_expr,
+                dob_expr=dob_expr,
+                label=f"{label_prefix}{label}",
+            )
+            matched_frames.append(matched)
+
+    return unmatched
+
+
+def lookup_sid_cepr(
+    frame,
+    *,
+    cols: Mapping[str, str],
+    lookups: dict[str, pl.DataFrame],
+):
+    is_lazy = isinstance(frame, pl.LazyFrame)
+    current = frame.collect() if is_lazy else frame
+
+    input_columns = current.columns
+
+    current = current.with_row_index("_row_id")
+
+    current = current.with_columns(
+        *clean_name(cols["fname"]),
+        *clean_name(cols["lname"]),
+    )
+
+    current = (
+        current
+        .with_columns(*clean_dob(col=cols["dob"]))
+        .with_columns(
+            _parse_dob_expr(f"{cols['dob']}_clean").alias(f"{cols['dob']}_clean")
+        )
+        .with_columns(
+            _month_day_expr(f"{cols['dob']}_clean").alias("_lhs_dob_md")
+        )
+    )
+
+    matched_frames = []
+    unmatched = current
+
+    matched, unmatched = _run_match_stage(
+        unmatched,
+        lookup=lookups["exact"],
+        fname_expr=pl.col(f"{cols['fname']}_clean"),
+        lname_expr=pl.col(f"{cols['lname']}_clean"),
+        dob_expr=pl.col(f"{cols['dob']}_clean"),
+        label="exact",
+    )
+    matched_frames.append(matched)
+
+    unmatched = _run_right_side_name_stages(
+        unmatched,
+        matched_frames,
+        cols=cols,
+        lookups=lookups,
+        prefix="",
+        dob_expr=pl.col(f"{cols['dob']}_clean"),
+        label_prefix="",
+    )
+
+    unmatched = _run_left_side_name_stages(
+        unmatched,
+        matched_frames,
+        cols=cols,
+        lookups=lookups,
+        exact_key="exact",
+        dob_expr=pl.col(f"{cols['dob']}_clean"),
+        label_prefix="",
+    )
+
+    for key in [
+        "dob_imp",
+        "dob_imp_minus_1",
+        "dob_imp_plus_1",
+        "dob_imp_minus_2",
+        "dob_imp_plus_2",
+    ]:
+        matched, unmatched = _run_match_stage(
+            unmatched,
+            lookup=lookups[key],
+            fname_expr=pl.col(f"{cols['fname']}_clean"),
+            lname_expr=pl.col(f"{cols['lname']}_clean"),
+            dob_expr=pl.col(f"{cols['dob']}_clean"),
+            label=key,
+        )
+        matched_frames.append(matched)
+
+    matched, unmatched = _run_match_stage(
+        unmatched,
+        lookup=lookups["md_exact"],
+        fname_expr=pl.col(f"{cols['fname']}_clean"),
+        lname_expr=pl.col(f"{cols['lname']}_clean"),
+        dob_expr=pl.col("_lhs_dob_md"),
+        label="month-day exact dob",
+    )
+    matched_frames.append(matched)
+
+    unmatched = _run_right_side_name_stages(
+        unmatched,
+        matched_frames,
+        cols=cols,
+        lookups=lookups,
+        prefix="md_",
+        dob_expr=pl.col("_lhs_dob_md"),
+        label_prefix="month-day dob: ",
+    )
+
+    unmatched = _run_left_side_name_stages(
+        unmatched,
+        matched_frames,
+        cols=cols,
+        lookups=lookups,
+        exact_key="md_exact",
+        dob_expr=pl.col("_lhs_dob_md"),
+        label_prefix="month-day dob: ",
+    )
+
+    matched, unmatched = _run_match_stage(
+        unmatched,
+        lookup=lookups["mdi_exact"],
+        fname_expr=pl.col(f"{cols['fname']}_clean"),
+        lname_expr=pl.col(f"{cols['lname']}_clean"),
+        dob_expr=pl.col("_lhs_dob_md"),
+        label="month-day dob_imp exact",
+    )
+    matched_frames.append(matched)
+
+    unmatched = _run_right_side_name_stages(
+        unmatched,
+        matched_frames,
+        cols=cols,
+        lookups=lookups,
+        prefix="mdi_",
+        dob_expr=pl.col("_lhs_dob_md"),
+        label_prefix="month-day dob_imp: ",
+    )
+
+    unmatched = _run_left_side_name_stages(
+        unmatched,
+        matched_frames,
+        cols=cols,
+        lookups=lookups,
+        exact_key="mdi_exact",
+        dob_expr=pl.col("_lhs_dob_md"),
+        label_prefix="month-day dob_imp: ",
+    )
+
+    result = (
+        pl.concat(matched_frames + [unmatched], how="diagonal_relaxed")
+        .sort("_row_id")
+        .drop(["_row_id", "_lhs_dob_md"])
+    )
+
+    if "sid_cepr" not in input_columns:
+        input_columns = input_columns + ["sid_cepr"]
+
+    result = result.select(input_columns)
+
+    print(f"final matched: {result['sid_cepr'].is_not_null().sum():,}/{len(result):,}")
+
+    return result