ltc-code 0.1.87__tar.gz → 0.1.89__tar.gz

{ltc_code-0.1.87 → ltc_code-0.1.89}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ltc-code
-Version: 0.1.87
+Version: 0.1.89
 Summary: Add your description here
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown

{ltc_code-0.1.87 → ltc_code-0.1.89}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.1.87"
+version = "0.1.89"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"

ltc_code-0.1.89/src/ltc_code/june5.py

@@ -0,0 +1,205 @@
+DEDUP_KEYS = [
+    "sid_cepr",
+    "school_year",
+    "entry_grade_clean",
+    "school_name",
+]
+
+EVER_COLS = [
+    "offer",
+    "waitlist",
+    "withdraw",
+    "offer_accepted",
+    "enroll",
+    "sibling",
+    "staff",
+    "zoned",
+]
+
+INFO_COLS = [
+    "lottery_number",
+    "waitlist_number",
+    "priority_group",
+    "priority_group_name",
+    "school_accept",
+]
+
+
+
+
+from typing import List, Union
+
+import polars as pl
+
+
+def resolve_duplicate_applications(
+    df: Union[pl.DataFrame, pl.LazyFrame],
+    dedup_keys: List[str],
+    ever_cols: List[str],
+    info_cols: List[str],
+) -> Union[pl.DataFrame, pl.LazyFrame]:
+    columns = df.collect_schema().names() if isinstance(df, pl.LazyFrame) else df.columns
+
+    missing = [
+        c for c in dedup_keys + ever_cols + info_cols
+        if c not in columns
+    ]
+    if missing:
+        raise ValueError("Missing specified columns: %s" % missing)
+
+    has_late_app = "late_app" in columns
+    has_application_cancel = "application_cancel" in columns
+
+    # These are optional tie-breakers only. They are not required and they do
+    # not affect the collapsed event flags.
+    date_tiebreakers = [
+        c for c in ["application_date", "offer_date"]
+        if c in columns
+    ]
+
+    late_rank = (
+        pl.when(pl.col("late_app") == 1)
+        .then(2)
+        .when(pl.col("late_app").is_null())
+        .then(1)
+        .otherwise(0)
+        if has_late_app
+        else pl.lit(0)
+    )
+
+    cancel_rank = (
+        pl.when(pl.col("application_cancel") == 1)
+        .then(2)
+        .when(pl.col("application_cancel").is_null())
+        .then(1)
+        .otherwise(0)
+        if has_application_cancel
+        else pl.lit(0)
+    )
+
+    # Outcome ranking follows the order of ever_cols:
+    # earlier columns are treated as stronger representative-row evidence.
+    outcome_rank = pl.lit(len(ever_cols))
+    for i, col in reversed(list(enumerate(ever_cols))):
+        outcome_rank = (
+            pl.when(pl.col(col) == 1)
+            .then(i)
+            .otherwise(outcome_rank)
+        )
+
+    sort_cols = (
+        dedup_keys
+        + ["_late_rank", "_cancel_rank", "_outcome_rank", "_info_count"]
+        + date_tiebreakers
+    )
+
+    descending = (
+        [False] * len(dedup_keys)
+        + [
+            False,  # prefer not late, if late_app exists
+            False,  # prefer not cancelled, if application_cancel exists
+            False,  # prefer earlier/stronger event columns from ever_cols
+            True,   # prefer rows with more non-null info columns
+        ]
+        + [False] * len(date_tiebreakers)
+    )
+
+    ranked = (
+        df
+        .with_columns(pl.len().over(dedup_keys).alias("_dup_n"))
+        .with_columns(
+            _late_rank=late_rank,
+            _cancel_rank=cancel_rank,
+            _outcome_rank=outcome_rank,
+            _info_count=pl.sum_horizontal(
+                [pl.col(c).is_not_null().cast(pl.Int8) for c in info_cols]
+            ),
+        )
+    )
+
+    temp_cols = [
+        "_dup_n",
+        "_late_rank",
+        "_cancel_rank",
+        "_outcome_rank",
+        "_info_count",
+    ]
+
+    non_dupes = (
+        ranked
+        .filter(pl.col("_dup_n") == 1)
+        .drop(temp_cols, strict=False)
+    )
+
+    dupes = (
+        ranked
+        .filter(pl.col("_dup_n") > 1)
+        .sort(sort_cols, descending=descending, nulls_last=True)
+    )
+
+    rep_rows = dupes.group_by(dedup_keys).first()
+
+    valid_filter = pl.lit(True)
+
+    if has_late_app:
+        valid_filter = valid_filter & (pl.col("late_app") != 1).fill_null(True)
+
+    if has_application_cancel:
+        valid_filter = (
+            valid_filter
+            & (pl.col("application_cancel") != 1).fill_null(True)
+        )
+
+    valid_dupes = dupes.filter(valid_filter)
+
+    collapsed_flags = (
+        valid_dupes
+        .group_by(dedup_keys)
+        .agg([pl.col(c).max().alias("_%s_collapsed" % c) for c in ever_cols])
+    )
+
+    dupes_resolved = (
+        rep_rows
+        .join(collapsed_flags, on=dedup_keys, how="left")
+        .with_columns(
+            [
+                pl.coalesce(pl.col("_%s_collapsed" % c), pl.col(c)).alias(c)
+                for c in ever_cols
+            ] + [
+                pl.lit(1).alias("former_dupe")
+            ]
+        )
+        .drop(
+            temp_cols + ["_%s_collapsed" % c for c in ever_cols],
+            strict=False,
+        )
+    )
+
+    final = pl.concat(
+        [
+            non_dupes.select(columns + ["former_dupe"]), 
+            dupes_resolved.select(columns + ["former_dupe"])
+        ], 
+            how="vertical_relaxed",
+    )
+
+    if isinstance(final, pl.DataFrame):
+        duplicate_count = (
+            final.select(dedup_keys).height
+            - final.select(dedup_keys).unique().height
+        )
+        assert duplicate_count == 0, (
+            "Found %s unresolved duplicate rows." % duplicate_count
+        )
+    
+    n_rows = (
+        final.height
+        if isinstance(final, pl.DataFrame)
+        else final.select(pl.len()).collect().item()
+    )
+    print(f"Number of observations after de-duplicating: {n_rows}")
+
+    return final
+
+
+

ltc_code-0.1.89/src/ltc_code/june7.py

@@ -0,0 +1,412 @@
+def resolve_rank_choice_duplicate_applications(
+    df: Union[pl.DataFrame, pl.LazyFrame],
+    base_keys: Sequence[str],
+    school_choice_cols: Sequence[str],
+    ever_cols: Sequence[str],
+    info_cols: Sequence[str],
+    outcome_school_cols: Optional[Mapping[str, str]] = None,
+    late_col: str = "late_app",
+    cancel_col: str = "application_cancel",
+    date_tiebreakers: Sequence[str] = ("application_date", "offer_date"),
+) -> Union[pl.DataFrame, pl.LazyFrame]:
+    """Resolve duplicate rank-choice application rows.
+
+    Rank-choice files can put multiple school choices on one row. This helper
+    compares the set of school choices within each applicant context
+    (``base_keys``), collapses rows that look like the same underlying
+    application, and leaves distinct applications separate.
+
+    Rows are collapsed when they have the exact same school set, or when one
+    row's school set is a strict subset of another row's set and the outcomes
+    for overlapping schools agree. Collapsed rows keep one representative row
+    for record-specific fields and take maxes across ``ever_cols`` among valid
+    rows. Rows left separate are flagged as current duplicates if more than
+    one application remains for the same ``base_keys``.
+
+    If ``df`` is a LazyFrame, this helper collects it to perform pairwise
+    set/subset comparisons, then returns a LazyFrame built from the result.
+    """
+    if not isinstance(df, (pl.DataFrame, pl.LazyFrame)):
+        raise TypeError("df must be a polars DataFrame or LazyFrame.")
+
+    is_lazy = isinstance(df, pl.LazyFrame)
+    frame = df.collect() if is_lazy else df
+    columns = list(frame.columns)
+
+    base_keys = list(base_keys)
+    school_choice_cols = list(school_choice_cols)
+    ever_cols = list(ever_cols)
+    info_cols = list(info_cols)
+    outcome_school_cols = dict(outcome_school_cols or {})
+
+    requested = base_keys + school_choice_cols + ever_cols + info_cols
+    requested.extend(outcome_school_cols.values())
+    if late_col in columns:
+        requested.append(late_col)
+    if cancel_col in columns:
+        requested.append(cancel_col)
+
+    missing = [column for column in requested if column not in columns]
+    if missing:
+        raise ValueError("Missing specified columns: %s" % sorted(set(missing)))
+    if not base_keys:
+        raise ValueError("base_keys must contain at least one column.")
+    if not school_choice_cols:
+        raise ValueError("school_choice_cols must contain at least one column.")
+    if not ever_cols:
+        raise ValueError("ever_cols must contain at least one column.")
+
+    invalid_outcome_keys = [column for column in outcome_school_cols if column not in ever_cols]
+    if invalid_outcome_keys:
+        raise ValueError(
+            "outcome_school_cols keys must also appear in ever_cols: %s"
+            % sorted(invalid_outcome_keys)
+        )
+
+    def _norm_school(value: Any) -> Optional[str]:
+        if value is None:
+            return None
+        text = str(value).strip()
+        return text or None
+
+    def _is_one(value: Any) -> bool:
+        if value is None:
+            return False
+        return value == 1 or value is True
+
+    def _school_values(row: Mapping[str, Any]) -> Tuple[List[str], Tuple[str, ...]]:
+        ordered = []
+        seen = set()
+        for column in school_choice_cols:
+            school = _norm_school(row[column])
+            if school is None or school in seen:
+                continue
+            ordered.append(school)
+            seen.add(school)
+        return ordered, tuple(sorted(seen))
+
+    def _outcome_signature(row: Mapping[str, Any], schools: Iterable[str]) -> Tuple[Any, ...]:
+        overlap = sorted(set(schools))
+        mapped_outcomes = [
+            column for column in ever_cols if column in outcome_school_cols
+        ]
+
+        # Best case: an outcome flag has a paired school column, e.g.
+        # offer + offer_school. Then compare outcomes only for overlap schools.
+        if mapped_outcomes:
+            values = []
+            for school in overlap:
+                for outcome_col in mapped_outcomes:
+                    school_col = outcome_school_cols[outcome_col]
+                    outcome_school = _norm_school(row[school_col])
+                    values.append(
+                        (
+                            outcome_col,
+                            school,
+                            int(outcome_school == school and _is_one(row[outcome_col])),
+                        )
+                    )
+            return tuple(values)
+
+        # Fallback: without school-specific outcome columns, the helper can only
+        # compare row-level outcome flags. This is enough for exact school-set
+        # duplicates, but it is a weaker assumption for subset decisions.
+        return tuple((column, int(_is_one(row[column]))) for column in ever_cols)
+
+    def _same_overlap_outcomes(
+        left: Mapping[str, Any],
+        right: Mapping[str, Any],
+        overlap: Iterable[str],
+    ) -> bool:
+        return _outcome_signature(left, overlap) == _outcome_signature(right, overlap)
+
+    def _info_count(row: Mapping[str, Any]) -> int:
+        return sum(1 for column in info_cols if row[column] is not None)
+
+    def _component_name(base_values: Tuple[Any, ...], school_set: Tuple[str, ...]) -> str:
+        return "%r|schools=%r" % (base_values, school_set)
+
+    work = frame.with_row_index("_rc_row_id")
+    rows = work.to_dicts()
+
+    metadata = {}
+    for row in rows:
+        ordered_schools, school_set = _school_values(row)
+        row_id = row["_rc_row_id"]
+        metadata[row_id] = {
+            "base": tuple(row[column] for column in base_keys),
+            "ordered": ordered_schools,
+            "set": school_set,
+            "component": "%r|row=%s" % (tuple(row[column] for column in base_keys), row_id),
+            "case": 0,
+            "school_set_readable": " | ".join(school_set) or None,
+            "school_ordered_readable": " | ".join(ordered_schools) or None,
+        }
+
+    rows_by_base = {}
+    for row in rows:
+        rows_by_base.setdefault(metadata[row["_rc_row_id"]]["base"], []).append(row)
+
+    for base_values, group_rows in rows_by_base.items():
+        rows_by_set = {}
+        for row in group_rows:
+            rows_by_set.setdefault(metadata[row["_rc_row_id"]]["set"], []).append(row)
+
+        # Cases 1-2: exact same application school set. These collapse even
+        # when row-level outcomes differ, because max(ever_cols) keeps evidence.
+        for school_set, set_rows in rows_by_set.items():
+            if len(set_rows) <= 1:
+                continue
+            signatures = {
+                _outcome_signature(row, school_set)
+                for row in set_rows
+            }
+            case = 1 if len(signatures) == 1 else 2
+            component = _component_name(base_values, school_set)
+            for row in set_rows:
+                row_id = row["_rc_row_id"]
+                metadata[row_id]["component"] = component
+                metadata[row_id]["case"] = case
+
+        # Case 3: a strict subset row can collapse into a richer superset row
+        # when outcomes agree for the overlapping schools. If more than one
+        # superset qualifies, choose the largest/richest one deterministically.
+        for row in group_rows:
+            row_id = row["_rc_row_id"]
+            school_set = set(metadata[row_id]["set"])
+            if not school_set:
+                continue
+
+            candidates = []
+            for other in group_rows:
+                other_id = other["_rc_row_id"]
+                if row_id == other_id:
+                    continue
+                other_set = set(metadata[other_id]["set"])
+                if not school_set < other_set:
+                    continue
+                if not _same_overlap_outcomes(row, other, school_set):
+                    continue
+                candidates.append(other)
+
+            if not candidates:
+                continue
+
+            candidates.sort(
+                key=lambda candidate: (
+                    len(metadata[candidate["_rc_row_id"]]["set"]),
+                    _info_count(candidate),
+                    -candidate["_rc_row_id"],
+                ),
+                reverse=True,
+            )
+            target_set = metadata[candidates[0]["_rc_row_id"]]["set"]
+            component = _component_name(base_values, target_set)
+            metadata[row_id]["component"] = component
+            metadata[row_id]["case"] = 3
+            for target_row in rows_by_set[target_set]:
+                target_id = target_row["_rc_row_id"]
+                metadata[target_id]["component"] = component
+
+    assignment_rows = []
+    for row_id, values in metadata.items():
+        assignment_rows.append(
+            {
+                "_rc_row_id": row_id,
+                "_rc_component": values["component"],
+                "rank_choice_duplicate_case": values["case"],
+                "rank_choice_school_set": values["school_set_readable"],
+                "rank_choice_school_ordered": values["school_ordered_readable"],
+            }
+        )
+
+    assignments = pl.DataFrame(assignment_rows)
+    work = work.join(assignments, on="_rc_row_id", how="left")
+
+    component_sizes = (
+        work.group_by("_rc_component")
+        .agg(
+            [
+                pl.len().alias("_rc_component_n"),
+                pl.col("rank_choice_duplicate_case")
+                .max()
+                .alias("_rc_component_case"),
+            ]
+        )
+    )
+    work = (
+        work
+        .join(component_sizes, on="_rc_component", how="left")
+        .with_columns(
+            pl.col("_rc_component_case").alias("rank_choice_duplicate_case")
+        )
+    )
+
+    outcome_rank = pl.lit(len(ever_cols))
+    for index, column in reversed(list(enumerate(ever_cols))):
+        outcome_rank = pl.when(pl.col(column) == 1).then(index).otherwise(outcome_rank)
+
+    late_rank = (
+        pl.when(pl.col(late_col) == 1)
+        .then(2)
+        .when(pl.col(late_col).is_null())
+        .then(1)
+        .otherwise(0)
+        if late_col in columns
+        else pl.lit(0)
+    )
+    cancel_rank = (
+        pl.when(pl.col(cancel_col) == 1)
+        .then(2)
+        .when(pl.col(cancel_col).is_null())
+        .then(1)
+        .otherwise(0)
+        if cancel_col in columns
+        else pl.lit(0)
+    )
+    info_count = (
+        pl.sum_horizontal([pl.col(c).is_not_null().cast(pl.Int8) for c in info_cols])
+        if info_cols
+        else pl.lit(0)
+    )
+
+    existing_date_tiebreakers = [column for column in date_tiebreakers if column in columns]
+    sort_cols = [
+        "_rc_component",
+        "_late_rank",
+        "_cancel_rank",
+        "_outcome_rank",
+        "_info_count",
+    ] + existing_date_tiebreakers
+    descending = [False, False, False, False, True] + [
+        False for _ in existing_date_tiebreakers
+    ]
+
+    ranked = work.with_columns(
+        _late_rank=late_rank,
+        _cancel_rank=cancel_rank,
+        _outcome_rank=outcome_rank,
+        _info_count=info_count,
+    )
+
+    untouched = ranked.filter(pl.col("_rc_component_n") == 1)
+    collapsed = (
+        ranked
+        .filter(pl.col("_rc_component_n") > 1)
+        .sort(sort_cols, descending=descending, nulls_last=True)
+    )
+
+    rep_rows = collapsed.group_by("_rc_component").first()
+
+    valid_filter = pl.lit(True)
+    if late_col in columns:
+        valid_filter = valid_filter & (pl.col(late_col) != 1).fill_null(True)
+    if cancel_col in columns:
+        valid_filter = valid_filter & (pl.col(cancel_col) != 1).fill_null(True)
+
+    collapsed_flags = (
+        collapsed
+        .filter(valid_filter)
+        .group_by("_rc_component")
+        .agg([pl.col(c).max().alias("_%s_collapsed" % c) for c in ever_cols])
+    )
+
+    temp_cols = [
+        "_rc_row_id",
+        "_rc_component",
+        "_rc_component_n",
+        "_rc_component_case",
+        "_late_rank",
+        "_cancel_rank",
+        "_outcome_rank",
+        "_info_count",
+    ]
+    collapsed_flag_cols = ["_%s_collapsed" % column for column in ever_cols]
+
+    collapsed_resolved = (
+        rep_rows
+        .join(collapsed_flags, on="_rc_component", how="left")
+        .with_columns(
+            [
+                pl.coalesce(pl.col("_%s_collapsed" % column), pl.col(column)).alias(
+                    column
+                )
+                for column in ever_cols
+            ]
+            + [pl.lit(1, dtype=pl.Int8).alias("former_dupe")]
+        )
+        .drop(collapsed_flag_cols, strict=False)
+    )
+
+    untouched = untouched.with_columns(pl.lit(0, dtype=pl.Int8).alias("former_dupe"))
+    output_columns = [
+        column for column in columns
+        if column not in {"former_dupe", "current_dupe"}
+    ] + [
+        "rank_choice_school_set",
+        "rank_choice_school_ordered",
+        "rank_choice_duplicate_case",
+        "former_dupe",
+    ]
+
+    result = pl.concat(
+        [
+            untouched.select(output_columns),
+            collapsed_resolved.select(output_columns),
+        ],
+        how="vertical_relaxed",
+    )
+
+    current_dupe_flags = (
+        result
+        .group_by(base_keys)
+        .agg((pl.len() > 1).cast(pl.Int8).alias("current_dupe"))
+    )
+    result = result.join(current_dupe_flags, on=base_keys, how="left")
+
+    final_columns = output_columns + ["current_dupe"]
+    result = result.select(final_columns)
+
+    unresolved_collapsible = result.filter(
+        (pl.col("rank_choice_duplicate_case").is_in([1, 2, 3]))
+        & (pl.col("former_dupe") != 1)
+    ).height
+    assert unresolved_collapsible == 0, (
+        "Found %s rows marked as collapsible cases but not collapsed."
+        % unresolved_collapsible
+    )
+
+    current_dupe_errors = result.filter(
+        (pl.len().over(base_keys) > 1)
+        & (pl.col("current_dupe") != 1)
+    ).height
+    assert current_dupe_errors == 0, (
+        "Found %s rows remaining duplicated at base_keys without current_dupe=1."
+        % current_dupe_errors
+    )
+
+    return result.lazy() if is_lazy else result
+
+
+# Example call without school specific outcomes
+final = df.pipe(
+    resolve_rank_choice_duplicate_applications,
+    base_keys=["sid_cepr", "school_year", "entry_grade_clean"],
+    school_choice_cols=["first_choice", "second_choice", "third_choice"],
+    ever_cols=["offer", "waitlist", "withdraw", "offer_accepted", "enroll"],
+    info_cols=["lottery_number", "waitlist_number", "priority_group"],
+)
+
+# Example call with school specific outcomes
+final = df.pipe(
+    resolve_rank_choice_duplicate_applications,
+    base_keys=["sid_cepr", "school_year", "entry_grade_clean"],
+    school_choice_cols=["first_choice", "second_choice", "third_choice"],
+    ever_cols=["offer", "waitlist", "withdraw", "offer_accepted", "enroll"],
+    info_cols=["lottery_number", "waitlist_number", "priority_group"],
+    outcome_school_cols={
+        "offer": "offer_school",
+        "waitlist": "waitlist_school",
+        "withdraw": "withdraw_school",
+    },
+)