ltc-code 0.1.88__tar.gz → 0.1.91__tar.gz

{ltc_code-0.1.88 → ltc_code-0.1.91}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ltc-code
-Version: 0.1.88
+Version: 0.1.91
 Summary: Add your description here
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown

{ltc_code-0.1.88 → ltc_code-0.1.91}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.1.88"
+version = "0.1.91"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"

{ltc_code-0.1.88 → ltc_code-0.1.91}/src/ltc_code/june5.py

@@ -165,6 +165,8 @@ def resolve_duplicate_applications(
             [
                 pl.coalesce(pl.col("_%s_collapsed" % c), pl.col(c)).alias(c)
                 for c in ever_cols
+            ] + [
+                pl.lit(1).alias("former_dupe")
             ]
         )
         .drop(
@@ -173,7 +175,13 @@ def resolve_duplicate_applications(
         )
     )
 
-    final = pl.concat([non_dupes, dupes_resolved], how="vertical_relaxed")
+    final = pl.concat(
+        [
+            non_dupes.select(columns + ["former_dupe"]), 
+            dupes_resolved.select(columns + ["former_dupe"])
+        ], 
+            how="vertical_relaxed",
+    )
 
     if isinstance(final, pl.DataFrame):
         duplicate_count = (
@@ -183,5 +191,15 @@ def resolve_duplicate_applications(
         assert duplicate_count == 0, (
             "Found %s unresolved duplicate rows." % duplicate_count
         )
+    
+    n_rows = (
+        final.height
+        if isinstance(final, pl.DataFrame)
+        else final.select(pl.len()).collect().item()
+    )
+    print(f"Number of observations after de-duplicating: {n_rows}")
+
+    return final
+
+
 
-    return final

ltc_code-0.1.91/src/ltc_code/june7.py

@@ -0,0 +1,406 @@
+def resolve_rank_choice_duplicate_applications(
+    df: Union[pl.DataFrame, pl.LazyFrame],
+    base_keys: Sequence[str],
+    school_choice_cols: Sequence[str],
+    ever_cols: Sequence[str],
+    info_cols: Sequence[str],
+    outcome_school_cols: Optional[Mapping[str, str]] = None,
+    late_col: str = "late_app",
+    cancel_col: str = "application_cancel",
+    date_tiebreakers: Sequence[str] = ("application_date", "offer_date"),
+) -> Union[pl.DataFrame, pl.LazyFrame]:
+    """Resolve duplicate rank-choice application rows.
+
+    If df is a LazyFrame, this helper collects it to do pairwise set/subset
+    comparisons, then returns a LazyFrame built from the result.
+    """
+    if not isinstance(df, (pl.DataFrame, pl.LazyFrame)):
+        raise TypeError("df must be a polars DataFrame or LazyFrame.")
+
+    is_lazy = isinstance(df, pl.LazyFrame)
+    frame = df.collect() if is_lazy else df
+    columns = list(frame.columns)
+
+    base_keys = list(base_keys)
+    school_choice_cols = list(school_choice_cols)
+    ever_cols = list(ever_cols)
+    info_cols = list(info_cols)
+    outcome_school_cols = dict(outcome_school_cols or {})
+
+    requested = base_keys + school_choice_cols + ever_cols + info_cols
+    requested.extend(outcome_school_cols.values())
+
+    missing = [column for column in requested if column not in columns]
+    if missing:
+        raise ValueError("Missing specified columns: %s" % sorted(set(missing)))
+    if not base_keys:
+        raise ValueError("base_keys must contain at least one column.")
+    if not school_choice_cols:
+        raise ValueError("school_choice_cols must contain at least one column.")
+    if not ever_cols:
+        raise ValueError("ever_cols must contain at least one column.")
+
+    invalid_outcome_keys = [column for column in outcome_school_cols if column not in ever_cols]
+    if invalid_outcome_keys:
+        raise ValueError(
+            "outcome_school_cols keys must also appear in ever_cols: %s"
+            % sorted(invalid_outcome_keys)
+        )
+
+    def _norm_school(value: Any) -> Optional[str]:
+        if value is None:
+            return None
+        text = str(value).strip()
+        return text or None
+
+    def _is_one(value: Any) -> bool:
+        if value is None:
+            return False
+        return value == 1 or value is True
+
+    def _school_values(row: Mapping[str, Any]) -> Tuple[List[str], Tuple[str, ...]]:
+        ordered = []
+        seen = set()
+        for column in school_choice_cols:
+            school = _norm_school(row[column])
+            if school is None or school in seen:
+                continue
+            ordered.append(school)
+            seen.add(school)
+        return ordered, tuple(sorted(seen))
+
+    def _outcome_signature(row: Mapping[str, Any], schools: Iterable[str]) -> Tuple[Any, ...]:
+        overlap = sorted(set(schools))
+        mapped_outcomes = [column for column in ever_cols if column in outcome_school_cols]
+
+        if mapped_outcomes:
+            values = []
+            for school in overlap:
+                for outcome_col in mapped_outcomes:
+                    school_col = outcome_school_cols[outcome_col]
+                    outcome_school = _norm_school(row[school_col])
+                    values.append(
+                        (
+                            outcome_col,
+                            school,
+                            int(outcome_school == school and _is_one(row[outcome_col])),
+                        )
+                    )
+            return tuple(values)
+
+        return tuple((column, int(_is_one(row[column]))) for column in ever_cols)
+
+    def _same_overlap_outcomes(
+        left: Mapping[str, Any],
+        right: Mapping[str, Any],
+        overlap: Iterable[str],
+    ) -> bool:
+        return _outcome_signature(left, overlap) == _outcome_signature(right, overlap)
+
+    def _info_count(row: Mapping[str, Any]) -> int:
+        return sum(1 for column in info_cols if row[column] is not None)
+
+    def _component_name(base_values: Tuple[Any, ...], school_set: Tuple[str, ...]) -> str:
+        return "%r|schools=%r" % (base_values, school_set)
+
+    def _find(parent: Dict[int, int], row_id: int) -> int:
+        while parent[row_id] != row_id:
+            parent[row_id] = parent[parent[row_id]]
+            row_id = parent[row_id]
+        return row_id
+
+    def _union(parent: Dict[int, int], left_id: int, right_id: int) -> None:
+        left_root = _find(parent, left_id)
+        right_root = _find(parent, right_id)
+        if left_root == right_root:
+            return
+        parent[max(left_root, right_root)] = min(left_root, right_root)
+
+    work = frame.with_row_index("_rc_row_id")
+    rows = work.to_dicts()
+
+    metadata = {}
+    parent = {}
+    for row in rows:
+        ordered_schools, school_set = _school_values(row)
+        row_id = row["_rc_row_id"]
+        parent[row_id] = row_id
+        metadata[row_id] = {
+            "base": tuple(row[column] for column in base_keys),
+            "ordered": ordered_schools,
+            "set": school_set,
+            "case": 0,
+            "school_set_readable": " | ".join(school_set) or None,
+            "school_ordered_readable": " | ".join(ordered_schools) or None,
+        }
+
+    rows_by_base = {}
+    for row in rows:
+        rows_by_base.setdefault(metadata[row["_rc_row_id"]]["base"], []).append(row)
+
+    for base_values, group_rows in rows_by_base.items():
+        rows_by_set = {}
+        for row in group_rows:
+            rows_by_set.setdefault(metadata[row["_rc_row_id"]]["set"], []).append(row)
+
+        for school_set, set_rows in rows_by_set.items():
+            if len(set_rows) <= 1:
+                continue
+            signatures = {_outcome_signature(row, school_set) for row in set_rows}
+            case = 1 if len(signatures) == 1 else 2
+            anchor_id = set_rows[0]["_rc_row_id"]
+            for row in set_rows:
+                row_id = row["_rc_row_id"]
+                _union(parent, anchor_id, row_id)
+                metadata[row_id]["case"] = case
+
+        for row in group_rows:
+            row_id = row["_rc_row_id"]
+            school_set = set(metadata[row_id]["set"])
+            if not school_set:
+                continue
+
+            for other in group_rows:
+                other_id = other["_rc_row_id"]
+                if row_id == other_id:
+                    continue
+                other_set = set(metadata[other_id]["set"])
+                if not school_set < other_set:
+                    continue
+                if not _same_overlap_outcomes(row, other, school_set):
+                    continue
+                _union(parent, row_id, other_id)
+                if metadata[row_id]["case"] == 0:
+                    metadata[row_id]["case"] = 3
+                if metadata[other_id]["case"] == 0:
+                    metadata[other_id]["case"] = 3
+
+    component_rows = {}
+    for row in rows:
+        row_id = row["_rc_row_id"]
+        component_rows.setdefault(_find(parent, row_id), []).append(row_id)
+
+    component_labels = {}
+    for component_row_ids in component_rows.values():
+        representative_id = max(
+            component_row_ids,
+            key=lambda row_id: (
+                len(metadata[row_id]["set"]),
+                _info_count(rows[row_id]),
+                -row_id,
+            ),
+        )
+        component_school_set = metadata[representative_id]["set"]
+        component_base = metadata[representative_id]["base"]
+        if len(component_row_ids) == 1:
+            component = "%r|row=%s" % (component_base, representative_id)
+        else:
+            component = _component_name(component_base, component_school_set)
+
+        component_case = max(metadata[row_id]["case"] for row_id in component_row_ids)
+        for row_id in component_row_ids:
+            component_labels[row_id] = component
+            metadata[row_id]["case"] = component_case
+
+    assignment_rows = []
+    for row_id, values in metadata.items():
+        assignment_rows.append(
+            {
+                "_rc_row_id": row_id,
+                "_rc_component": component_labels[row_id],
+                "rank_choice_duplicate_case": values["case"],
+                "rank_choice_school_set": values["school_set_readable"],
+                "rank_choice_school_ordered": values["school_ordered_readable"],
+                "_rc_school_count": len(values["set"]),
+            }
+        )
+
+    assignments = pl.DataFrame(assignment_rows)
+    work = work.join(assignments, on="_rc_row_id", how="left")
+
+    component_sizes = (
+        work.group_by("_rc_component")
+        .agg(
+            [
+                pl.len().alias("_rc_component_n"),
+                pl.col("rank_choice_duplicate_case")
+                .max()
+                .alias("_rc_component_case"),
+            ]
+        )
+    )
+
+    work = (
+        work
+        .join(component_sizes, on="_rc_component", how="left")
+        .with_columns(pl.col("_rc_component_case").alias("rank_choice_duplicate_case"))
+    )
+
+    outcome_rank = pl.lit(len(ever_cols))
+    for index, column in reversed(list(enumerate(ever_cols))):
+        outcome_rank = pl.when(pl.col(column) == 1).then(index).otherwise(outcome_rank)
+
+    late_rank = (
+        pl.when(pl.col(late_col) == 1)
+        .then(2)
+        .when(pl.col(late_col).is_null())
+        .then(1)
+        .otherwise(0)
+        if late_col in columns
+        else pl.lit(0)
+    )
+
+    cancel_rank = (
+        pl.when(pl.col(cancel_col) == 1)
+        .then(2)
+        .when(pl.col(cancel_col).is_null())
+        .then(1)
+        .otherwise(0)
+        if cancel_col in columns
+        else pl.lit(0)
+    )
+
+    info_count = (
+        pl.sum_horizontal([pl.col(c).is_not_null().cast(pl.Int8) for c in info_cols])
+        if info_cols
+        else pl.lit(0)
+    )
+
+    existing_date_tiebreakers = [column for column in date_tiebreakers if column in columns]
+
+    sort_cols = [
+        "_rc_component",
+        "_late_rank",
+        "_cancel_rank",
+        "_outcome_rank",
+        "_rc_school_count",
+        "_info_count",
+    ] + existing_date_tiebreakers
+
+    descending = [False, False, False, False, True, True] + [
+        False for _ in existing_date_tiebreakers
+    ]
+
+    ranked = work.with_columns(
+        _late_rank=late_rank,
+        _cancel_rank=cancel_rank,
+        _outcome_rank=outcome_rank,
+        _info_count=info_count,
+    )
+
+    untouched = ranked.filter(pl.col("_rc_component_n") == 1)
+
+    collapsed = (
+        ranked
+        .filter(pl.col("_rc_component_n") > 1)
+        .sort(sort_cols, descending=descending, nulls_last=True)
+    )
+
+    rep_rows = collapsed.group_by("_rc_component").first()
+
+    valid_filter = pl.lit(True)
+    if late_col in columns:
+        valid_filter = valid_filter & (pl.col(late_col) != 1).fill_null(True)
+    if cancel_col in columns:
+        valid_filter = valid_filter & (pl.col(cancel_col) != 1).fill_null(True)
+
+    collapsed_flags = (
+        collapsed
+        .filter(valid_filter)
+        .group_by("_rc_component")
+        .agg([pl.col(c).max().alias("_%s_collapsed" % c) for c in ever_cols])
+    )
+
+    collapsed_flag_cols = ["_%s_collapsed" % column for column in ever_cols]
+
+    collapsed_resolved = (
+        rep_rows
+        .join(collapsed_flags, on="_rc_component", how="left")
+        .with_columns(
+            [
+                pl.coalesce(pl.col("_%s_collapsed" % column), pl.col(column)).alias(column)
+                for column in ever_cols
+            ]
+            + [pl.lit(1, dtype=pl.Int8).alias("former_dupe")]
+        )
+        .drop(collapsed_flag_cols, strict=False)
+    )
+
+    untouched = untouched.with_columns(pl.lit(0, dtype=pl.Int8).alias("former_dupe"))
+
+    output_columns = [
+        column for column in columns
+        if column not in {"former_dupe", "current_dupe"}
+    ] + [
+        "rank_choice_school_set",
+        "rank_choice_school_ordered",
+        "rank_choice_duplicate_case",
+        "former_dupe",
+    ]
+
+    result = pl.concat(
+        [
+            untouched.select(output_columns),
+            collapsed_resolved.select(output_columns),
+        ],
+        how="vertical_relaxed",
+    )
+
+    current_dupe_flags = (
+        result
+        .group_by(base_keys)
+        .agg((pl.len() > 1).cast(pl.Int8).alias("current_dupe"))
+    )
+
+    result = result.join(current_dupe_flags, on=base_keys, how="left")
+
+    final_columns = output_columns + ["current_dupe"]
+    result = result.select(final_columns)
+
+    unresolved_collapsible = result.filter(
+        (pl.col("rank_choice_duplicate_case").is_in([1, 2, 3]))
+        & (pl.col("former_dupe") != 1)
+    ).height
+
+    assert unresolved_collapsible == 0, (
+        "Found %s rows marked as collapsible cases but not collapsed."
+        % unresolved_collapsible
+    )
+
+    current_dupe_errors = result.filter(
+        (pl.len().over(base_keys) > 1)
+        & (pl.col("current_dupe") != 1)
+    ).height
+
+    assert current_dupe_errors == 0, (
+        "Found %s rows remaining duplicated at base_keys without current_dupe=1."
+        % current_dupe_errors
+    )
+
+    return result.lazy() if is_lazy else result
+
+
+
+
+# Example call without school specific outcomes
+final = df.pipe(
+    resolve_rank_choice_duplicate_applications,
+    base_keys=["sid_cepr", "school_year", "entry_grade_clean"],
+    school_choice_cols=["first_choice", "second_choice", "third_choice"],
+    ever_cols=["offer", "waitlist", "withdraw", "offer_accepted", "enroll"],
+    info_cols=["lottery_number", "waitlist_number", "priority_group"],
+)
+
+# Example call with school specific outcomes
+final = df.pipe(
+    resolve_rank_choice_duplicate_applications,
+    base_keys=["sid_cepr", "school_year", "entry_grade_clean"],
+    school_choice_cols=["first_choice", "second_choice", "third_choice"],
+    ever_cols=["offer", "waitlist", "withdraw", "offer_accepted", "enroll"],
+    info_cols=["lottery_number", "waitlist_number", "priority_group"],
+    outcome_school_cols={
+        "offer": "offer_school",
+        "waitlist": "waitlist_school",
+        "withdraw": "withdraw_school",
+    },
+)