ltc-code 0.1.1__tar.gz → 0.1.2__tar.gz

{ltc_code-0.1.1 → ltc_code-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ltc-code
-Version: 0.1.1
+Version: 0.1.2
 Summary: Add your description here
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown

{ltc_code-0.1.1 → ltc_code-0.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ltc-code"
-version = "0.1.1"
+version = "0.1.2"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.9"

ltc_code-0.1.2/src/ltc_code/may27.py

@@ -0,0 +1,974 @@
+"""Reusable helper functions for the LTC project."""
+
+from collections import Counter
+from dataclasses import dataclass
+from functools import lru_cache
+from itertools import combinations
+import math
+import re
+import warnings
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple, Union
+
+import polars as pl
+with warnings.catch_warnings():
+    warnings.filterwarnings("ignore", message="urllib3 v2 only supports OpenSSL")
+    from scourgify import normalize_address_record
+    from scourgify.exceptions import AddressNormalizationError
+import usaddress
+
+
+__all__ = [
+    "clean_addresses",
+    "clean_address_columns",
+    "ColumnConflictError",
+    "ConsolidationDiagnostics",
+    "GroupDiagnostics",
+    "consolidate_columns",
+    "select_lottery_columns",
+    "select_enr_columns",
+]
+
+
+###############################################################################
+# ADDRESS STANDARDIZATION AND CONFLICT REVIEW
+#
+# Public entry point:
+#     cleaned = frame.pipe(clean_addresses)
+#
+# This section reconciles one to three optional address-bearing columns
+# (`address`, `address1`, and `address2` by default) into standardized output
+# fields and an integer `address_conflict` review flag.
+#
+# Operational notes:
+# - Parsing uses usaddress-scourgify with usaddress fallback for fragments.
+# - Parsed text is standardized, not verified against an official address file.
+# - Repeated exact raw values are cached in memory only for the Python process.
+# - DataFrame inputs print diagnostics while returning a DataFrame.
+# - LazyFrame inputs also print diagnostics; doing so executes separate summary
+#   and example queries immediately, while the returned result remains lazy.
+###############################################################################
+
+DEFAULT_ADDRESS_COLUMNS = ("address", "address1", "address2")
+OUTPUT_DTYPE = pl.Struct(
+    {
+        "address1_clean": pl.String,
+        "address2_clean": pl.String,
+        "zipcode_clean": pl.String,
+        "city_clean": pl.String,
+        "state_clean": pl.String,
+        "address_conflict": pl.Int8,
+        "address_conflict_reason": pl.String,
+        "address_parse_ok": pl.Boolean,
+    }
+)
+
+STATE_ABBREVIATIONS = {
+    "ALABAMA": "AL",
+    "ALASKA": "AK",
+    "ARIZONA": "AZ",
+    "ARKANSAS": "AR",
+    "CALIFORNIA": "CA",
+    "COLORADO": "CO",
+    "CONNECTICUT": "CT",
+    "DELAWARE": "DE",
+    "FLORIDA": "FL",
+    "GEORGIA": "GA",
+    "HAWAII": "HI",
+    "IDAHO": "ID",
+    "ILLINOIS": "IL",
+    "INDIANA": "IN",
+    "IOWA": "IA",
+    "KANSAS": "KS",
+    "KENTUCKY": "KY",
+    "LOUISIANA": "LA",
+    "MAINE": "ME",
+    "MARYLAND": "MD",
+    "MASSACHUSETTS": "MA",
+    "MICHIGAN": "MI",
+    "MINNESOTA": "MN",
+    "MISSISSIPPI": "MS",
+    "MISSOURI": "MO",
+    "MONTANA": "MT",
+    "NEBRASKA": "NE",
+    "NEVADA": "NV",
+    "NEW HAMPSHIRE": "NH",
+    "NEW JERSEY": "NJ",
+    "NEW MEXICO": "NM",
+    "NEW YORK": "NY",
+    "NORTH CAROLINA": "NC",
+    "NORTH DAKOTA": "ND",
+    "OHIO": "OH",
+    "OKLAHOMA": "OK",
+    "OREGON": "OR",
+    "PENNSYLVANIA": "PA",
+    "RHODE ISLAND": "RI",
+    "SOUTH CAROLINA": "SC",
+    "SOUTH DAKOTA": "SD",
+    "TENNESSEE": "TN",
+    "TEXAS": "TX",
+    "UTAH": "UT",
+    "VERMONT": "VT",
+    "VIRGINIA": "VA",
+    "WASHINGTON": "WA",
+    "WEST VIRGINIA": "WV",
+    "WISCONSIN": "WI",
+    "WYOMING": "WY",
+}
+
+COMPONENT_KEYS = (
+    "address_line_1",
+    "address_line_2",
+    "city",
+    "state",
+    "postal_code",
+)
+
+
+def _clean_text(value: Any) -> Optional[str]:
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text or None
+
+
+def _prepare_address(value: str) -> str:
+    text = value.upper()
+    text = re.sub(r"\bZIP\s*:?\s*", "", text)
+    text = re.sub(r"\s*[|;/]\s*", ", ", text)
+    text = re.sub(r"\s*--+\s*", ", ", text)
+    text = re.sub(r"#\s*(APT|STE|SUITE|UNIT|FLOOR)\b", r"\1", text)
+    for state, abbreviation in STATE_ABBREVIATIONS.items():
+        text = re.sub(
+            r"\b%s\b(?=\s*,?\s*(?:\d{5}(?:-\d{4})?\b|$))" % re.escape(state),
+            abbreviation,
+            text,
+        )
+    text = re.sub(
+        r"^(.*?),\s*(.*?),\s*(\d{5}(?:-\d{4})?),\s*([A-Z]{2})$",
+        r"\1, \2, \4 \3",
+        text,
+    )
+    text = re.sub(
+        r"^(.*?),\s*(.*?)\s+([A-Z]{2})-(\d{5}(?:-\d{4})?)\s+(.+)$",
+        r"\1, \5, \2, \3 \4",
+        text,
+    )
+    text = re.sub(r"\s+", " ", text)
+    return text.strip(" ,")
+
+
+def _join_tag_values(tags: Dict[str, str], keys: Iterable[str]) -> Optional[str]:
+    values = [tags[key].upper() for key in keys if tags.get(key)]
+    return " ".join(values) if values else None
+
+
+def _from_usaddress(value: str) -> Dict[str, Optional[str]]:
+    try:
+        tags, _ = usaddress.tag(value)
+    except usaddress.RepeatedLabelError:
+        return {key: None for key in COMPONENT_KEYS}
+
+    line_1 = _join_tag_values(
+        tags,
+        (
+            "USPSBoxType",
+            "USPSBoxID",
+            "AddressNumber",
+            "StreetNamePreDirectional",
+            "StreetNamePreType",
+            "StreetName",
+            "StreetNamePostType",
+            "StreetNamePostDirectional",
+        ),
+    )
+    line_2 = _join_tag_values(
+        tags,
+        ("OccupancyType", "OccupancyIdentifier", "SubaddressType", "SubaddressIdentifier"),
+    )
+    state = tags.get("StateName")
+    return {
+        "address_line_1": line_1,
+        "address_line_2": line_2,
+        "city": tags.get("PlaceName", "").upper() or None,
+        "state": state.upper() if state else None,
+        "postal_code": tags.get("ZipCode", "").upper() or None,
+    }
+
+
+@lru_cache(maxsize=100000)
+def _parse_address(value: str) -> Dict[str, Optional[str]]:
+    prepared = _prepare_address(value)
+    try:
+        normalized = normalize_address_record(prepared)
+        return {key: normalized.get(key) for key in COMPONENT_KEYS}
+    except AddressNormalizationError:
+        return _from_usaddress(prepared)
+
+
+def _candidate_score(candidate: Dict[str, Optional[str]]) -> int:
+    return (
+        (4 if candidate["address_line_1"] else 0)
+        + (1 if candidate["address_line_2"] else 0)
+        + (1 if candidate["city"] else 0)
+        + (1 if candidate["state"] else 0)
+        + (1 if candidate["postal_code"] else 0)
+    )
+
+
+def _comparison_value(key: str, value: str) -> str:
+    text = re.sub(r"[^A-Z0-9# ]+", " ", value.upper())
+    text = re.sub(r"\s+", " ", text).strip()
+    if key == "address_line_2":
+        text = re.sub(r"^APT\s+", "APT ", text)
+        text = re.sub(r"^(FLOOR|FL)\s+", "FL ", text)
+        text = re.sub(r"^(SUITE|STE)\s+", "STE ", text)
+        text = re.sub(r"^(APARTMENT)\s+", "APT ", text)
+    return text
+
+
+def _resolve_component(
+    candidates: List[Dict[str, Optional[str]]], key: str
+) -> Tuple[Optional[str], List[str]]:
+    observed = [candidate[key] for candidate in candidates if candidate[key]]
+    if not observed:
+        return None, []
+    counts = Counter(_comparison_value(key, value) for value in observed)
+    best_count = max(counts.values())
+    most_common = {value for value, count in counts.items() if count == best_count}
+    chosen = next(
+        candidate[key]
+        for candidate in sorted(candidates, key=_candidate_score, reverse=True)
+        if candidate[key] and _comparison_value(key, candidate[key]) in most_common
+    )
+    distinct = []
+    seen = set()
+    for value in observed:
+        comparison = _comparison_value(key, value)
+        if comparison not in seen:
+            distinct.append(value)
+            seen.add(comparison)
+    return chosen, sorted(distinct)
+
+
+def _reconcile_row(row: Dict[str, Any], address_columns: Sequence[str]) -> Dict[str, Any]:
+    values = [_clean_text(row.get(column)) for column in address_columns]
+    candidates = [_parse_address(value) for value in values if value]
+    resolved = {}
+    conflicts = []
+    for key in COMPONENT_KEYS:
+        resolved[key], distinct = _resolve_component(candidates, key)
+        if len(distinct) > 1:
+            conflicts.append("%s: %s" % (key, " <> ".join(distinct)))
+
+    return {
+        "address1_clean": resolved["address_line_1"],
+        "address2_clean": resolved["address_line_2"],
+        "zipcode_clean": resolved["postal_code"],
+        "city_clean": resolved["city"],
+        "state_clean": resolved["state"],
+        "address_conflict": int(bool(conflicts)),
+        "address_conflict_reason": "; ".join(conflicts) or None,
+        "address_parse_ok": bool(resolved["address_line_1"] or resolved["postal_code"]),
+    }
+
+
+def _print_address_diagnostics(
+    frame: Union[pl.DataFrame, pl.LazyFrame],
+    address_columns: Sequence[str],
+    example_count: int = 3,
+) -> None:
+    nonblank_expressions = [
+        pl.col(column).fill_null("").str.strip_chars().ne("") for column in address_columns
+    ]
+    summary_expressions = [
+        pl.len().alias("rows"),
+        *[
+            expression.sum().alias("nonblank_%s" % column)
+            for column, expression in zip(address_columns, nonblank_expressions)
+        ],
+        pl.any_horizontal(nonblank_expressions).sum().alias("rows_with_address"),
+        pl.col("address_parse_ok").sum().alias("parsed_rows"),
+        pl.col("address_conflict").sum().alias("conflicting_rows"),
+    ]
+    summary = frame.select(summary_expressions)
+    if isinstance(summary, pl.LazyFrame):
+        summary = summary.collect()
+    totals = summary.row(0, named=True)
+    conflicting_rows = totals["conflicting_rows"]
+
+    print("Address cleaning diagnostics")
+    print("  rows: %s" % totals["rows"])
+    for column in address_columns:
+        print("  non-null %s: %s" % (column, totals["nonblank_%s" % column]))
+    print("  rows with any supplied address: %s" % totals["rows_with_address"])
+    print("  parsed successfully: %s" % totals["parsed_rows"])
+    print("  contradictions: %s" % conflicting_rows)
+
+    if conflicting_rows:
+        print("  contradiction examples:")
+        example_columns = list(address_columns) + [
+            "address1_clean",
+            "address2_clean",
+            "zipcode_clean",
+            "address_conflict",
+            "address_conflict_reason",
+        ]
+        examples = (
+            frame.filter(pl.col("address_conflict") == 1)
+            .select(example_columns)
+            .head(example_count)
+        )
+        if isinstance(examples, pl.LazyFrame):
+            examples = examples.collect()
+        for number, row in enumerate(examples.iter_rows(named=True), 1):
+            values = [
+                "%s=%s" % (column, row[column] if row[column] is not None else "NULL")
+                for column in address_columns
+            ]
+            print("    %s. %s" % (number, " | ".join(values)))
+            print(
+                "       address1_clean=%s | address2_clean=%s | zipcode_clean=%s"
+                % (
+                    row["address1_clean"] or "NULL",
+                    row["address2_clean"] or "NULL",
+                    row["zipcode_clean"] or "NULL",
+                )
+            )
+            print("       address_conflict=1 | reason=%s" % row["address_conflict_reason"])
+
+
+def clean_addresses(
+    frame: Union[pl.DataFrame, pl.LazyFrame],
+    address_columns: Optional[Union[str, Sequence[str]]] = None,
+) -> Union[pl.DataFrame, pl.LazyFrame]:
+    """Clean one to three address fields; designed for ``frame.pipe(...)``.
+
+    With no ``address_columns`` argument, any existing columns named
+    ``address``, ``address1``, and ``address2`` are used automatically.
+    Values from all chosen nonblank columns contribute to the resolved output.
+    Disagreements are retained in the conflict audit columns.
+    Calls print a brief diagnostic summary and conflict examples. For a
+    ``LazyFrame``, this evaluates separate diagnostic queries immediately but
+    returns the cleaned result as a still-lazy ``LazyFrame``.
+    """
+    existing = set(
+        frame.collect_schema().names() if isinstance(frame, pl.LazyFrame) else frame.columns
+    )
+    if address_columns is None:
+        columns = [column for column in DEFAULT_ADDRESS_COLUMNS if column in existing]
+    elif isinstance(address_columns, str):
+        columns = [address_columns]
+    else:
+        columns = list(address_columns)
+
+    if not 1 <= len(columns) <= 3:
+        raise ValueError("Provide one, two, or three address column names.")
+    if len(set(columns)) != len(columns):
+        raise ValueError("Address column names must be unique.")
+    missing = [column for column in columns if column not in existing]
+    if missing:
+        raise ValueError("Missing address columns: %s" % ", ".join(missing))
+
+    clean_struct = pl.struct(columns).map_elements(
+        lambda row: _reconcile_row(row, columns),
+        return_dtype=OUTPUT_DTYPE,
+    ).alias("_address_clean")
+    result = frame.with_columns(clean_struct).unnest("_address_clean")
+    _print_address_diagnostics(result, columns)
+    return result
+
+
+clean_address_columns = clean_addresses
+
+
+###############################################################################
+# HORIZONTAL COLUMN CONSOLIDATION AND CONFLICT REVIEW
+#
+# Public entry point:
+#     reviewed = frame.pipe(
+#         consolidate_columns,
+#         {"dob": ["dob_1", "dob_2"], "fname": ["fname_1", "fname_2"]},
+#     )
+#
+# This section merges repeated representations of the same field, such as
+# several DOB columns into `dob` and several first-name columns into `fname`.
+# Values are compared exactly as stored; this helper does not trim, parse,
+# standardize, or normalize them.
+#
+# Operational notes:
+# - This helper uses in-memory Polars processing only.  It makes no API calls
+#   and does not read from or write to a database.
+# - It accepts a DataFrame or LazyFrame and returns the same frame type.
+# - Printing diagnostics for a LazyFrame evaluates only the relevant source
+#   projection immediately; the returned transformed result remains lazy.
+# - On contradictions, the default is to retain source evidence and add
+#   integer `<output>_conflict` columns where `1` indicates review is needed.
+# - When all compared values agree, the input source columns are dropped.
+# - Rows are never dropped.
+###############################################################################
+
+Frame = Union[pl.DataFrame, pl.LazyFrame]
+ConsolidatedResult = Union[Frame, Tuple[Frame, "ConsolidationDiagnostics"]]
+
+
+@dataclass
+class GroupDiagnostics:
+    """Diagnostics for one requested consolidated output column."""
+
+    output_column: str
+    source_columns: List[str]
+    context_columns: List[str]
+    total_rows: int
+    rows_with_any_value: int
+    rows_with_multiple_values: int
+    conflicting_rows: int
+    conflicting_row_numbers: List[int]
+    conflict_column_pairs: pl.DataFrame
+    value_patterns: pl.DataFrame
+    example_conflicts: pl.DataFrame
+
+    @property
+    def conflict_rate(self) -> float:
+        if not self.total_rows:
+            return 0.0
+        return self.conflicting_rows / self.total_rows
+
+    @property
+    def rows_with_output(self) -> int:
+        """Rows for which the requested output can be safely populated."""
+        return self.rows_with_any_value - self.conflicting_rows
+
+    @property
+    def rows_without_value(self) -> int:
+        """Rows with no non-missing source value for this output."""
+        return self.total_rows - self.rows_with_any_value
+
+
+@dataclass
+class ConsolidationDiagnostics:
+    """Structured audit information produced while consolidating columns."""
+
+    groups: Dict[str, GroupDiagnostics]
+
+    @property
+    def has_conflicts(self) -> bool:
+        return any(group.conflicting_rows for group in self.groups.values())
+
+    @property
+    def total_conflicting_rows(self) -> int:
+        conflict_rows = set()
+        for group in self.groups.values():
+            conflict_rows.update(group.conflicting_row_numbers)
+        return len(conflict_rows)
+
+    def summary(self) -> pl.DataFrame:
+        """Return one audit-summary row for each consolidated output."""
+        return pl.DataFrame(
+            [
+                {
+                    "output_column": group.output_column,
+                    "source_columns": ", ".join(group.source_columns),
+                    "total_rows": group.total_rows,
+                    "rows_with_output": group.rows_with_output,
+                    "rows_without_value": group.rows_without_value,
+                    "conflicting_rows": group.conflicting_rows,
+                    "conflict_rate": group.conflict_rate,
+                }
+                for group in self.groups.values()
+            ]
+        )
+
+    def format_report(self, action: Optional[str] = None) -> str:
+        """Render a compact human-readable consolidation report."""
+        if self.has_conflicts:
+            headline = "Column consolidation found contradictions in %s unique rows." % (
+                self.total_conflicting_rows
+            )
+        else:
+            headline = "Column consolidation completed with no contradictions."
+        with pl.Config(tbl_cols=-1, tbl_width_chars=220):
+            lines = [headline, str(self.summary())]
+            for group in self.groups.values():
+                if not group.conflicting_rows:
+                    continue
+                lines.extend(
+                    [
+                        "",
+                        "Output %r: conflicting source-column pairs:" % group.output_column,
+                        str(group.conflict_column_pairs),
+                        "Most frequent contradictory value patterns:",
+                        str(group.value_patterns),
+                        "Example contradictory rows:",
+                        str(group.example_conflicts),
+                    ]
+                )
+        if action:
+            lines.extend(["", "Action: %s" % action])
+        return "\n".join(lines)
+
+
+class ColumnConflictError(ValueError):
+    """Raised when consolidation is configured to stop on contradictory values."""
+
+    def __init__(self, diagnostics: ConsolidationDiagnostics) -> None:
+        self.diagnostics = diagnostics
+        super().__init__(diagnostics.format_report())
+
+
+def _consolidation_non_missing(value: Any) -> Optional[Any]:
+    if value is None:
+        return None
+    if isinstance(value, float) and math.isnan(value):
+        return None
+    return value
+
+
+def _consolidation_comparable_value(value: Any) -> Any:
+    try:
+        hash(value)
+    except TypeError:
+        return repr(value)
+    return value
+
+
+def _consolidation_row_values(
+    row: Mapping[str, Any],
+    source_columns: Sequence[str],
+) -> Tuple[List[Tuple[str, Any, Any]], List[Any]]:
+    populated = []
+    distinct = []
+    for column in source_columns:
+        value = _consolidation_non_missing(row[column])
+        if value is None:
+            continue
+        comparable = _consolidation_comparable_value(value)
+        populated.append((column, value, comparable))
+        if comparable not in distinct:
+            distinct.append(comparable)
+    return populated, distinct
+
+
+def _analyze_consolidation_group(
+    frame: pl.DataFrame,
+    output_column: str,
+    source_columns: Sequence[str],
+    context_columns: Sequence[str],
+    max_examples: int,
+    row_number_column: str,
+) -> Tuple[List[Any], List[int], GroupDiagnostics]:
+    numbered = frame.with_row_index(row_number_column)
+    selected = numbered.select(
+        [row_number_column] + list(context_columns) + list(source_columns)
+    )
+    output_values = []
+    conflict_flags = []
+    rows_with_any_value = 0
+    rows_with_multiple_values = 0
+    conflicting_rows = []
+    conflicting_row_numbers = []
+    patterns = Counter()
+    pair_counts = Counter()
+
+    for row in selected.iter_rows(named=True):
+        populated, distinct = _consolidation_row_values(row, source_columns)
+        if populated:
+            rows_with_any_value += 1
+        if len(populated) > 1:
+            rows_with_multiple_values += 1
+        if len(distinct) <= 1:
+            output_values.append(populated[0][1] if populated else None)
+            conflict_flags.append(0)
+            continue
+
+        output_values.append(None)
+        conflict_flags.append(1)
+        conflicting_rows.append(row)
+        conflicting_row_numbers.append(row[row_number_column])
+        pattern = " <> ".join(repr(value) for value in distinct)
+        patterns[pattern] += 1
+        for left, right in combinations(populated, 2):
+            if left[2] != right[2]:
+                pair_counts[(left[0], right[0])] += 1
+
+    pair_rows = [
+        {"column_a": pair[0], "column_b": pair[1], "conflicting_rows": count}
+        for pair, count in pair_counts.most_common()
+    ]
+    pattern_rows = [
+        {"observed_values": pattern, "rows": count}
+        for pattern, count in patterns.most_common(max_examples)
+    ]
+    diagnostics = GroupDiagnostics(
+        output_column=output_column,
+        source_columns=list(source_columns),
+        context_columns=list(context_columns),
+        total_rows=frame.height,
+        rows_with_any_value=rows_with_any_value,
+        rows_with_multiple_values=rows_with_multiple_values,
+        conflicting_rows=len(conflicting_rows),
+        conflicting_row_numbers=conflicting_row_numbers,
+        conflict_column_pairs=pl.DataFrame(
+            pair_rows,
+            schema={"column_a": pl.String, "column_b": pl.String, "conflicting_rows": pl.Int64},
+        ),
+        value_patterns=pl.DataFrame(
+            pattern_rows, schema={"observed_values": pl.String, "rows": pl.Int64}
+        ),
+        example_conflicts=pl.DataFrame(
+            conflicting_rows[:max_examples],
+            schema=selected.schema,
+        ),
+    )
+    return output_values, conflict_flags, diagnostics
+
+
+def _temporary_consolidation_row_number(existing: Sequence[str]) -> str:
+    """Choose an internal diagnostic row-number column not present in the input."""
+    column = "_row_number"
+    while column in existing:
+        column = "_" + column
+    return column
+
+
+def consolidate_columns(
+    frame: Frame,
+    columns: Mapping[str, Sequence[str]],
+    *,
+    context_columns: Optional[Sequence[str]] = None,
+    on_conflict: str = "keep",
+    max_examples: int = 5,
+    return_diagnostics: bool = False,
+) -> ConsolidatedResult:
+    """Coalesce repeated exact-value columns and print conflict diagnostics.
+
+    ``columns`` maps each desired output column to its possible source
+    columns; any number of outputs may be consolidated in one call.  No
+    normalization is performed: values must match exactly to agree.
+
+    ``context_columns`` optionally supplies identifiers, such as
+    ``["person_id"]``, to display in conflict examples.  Without one, a
+    temporary row number is printed in examples but is not returned.
+
+    ``on_conflict="keep"`` is the safe default: input columns are retained and
+    integer ``<output>_conflict`` flags are added when contradictions exist.
+    ``"raise"`` prints diagnostics and stops, while ``"drop"`` discards source
+    evidence and leaves contradicted consolidated outputs null.
+
+    Accepts a Polars DataFrame or LazyFrame and returns the same frame type.
+    For a LazyFrame, diagnostic printing evaluates the relevant source
+    projection immediately, while the returned transformed frame stays lazy.
+    The function makes no API or database calls and never drops rows.
+    """
+    if not isinstance(frame, (pl.DataFrame, pl.LazyFrame)):
+        raise TypeError(
+            "consolidate_columns expects a polars.DataFrame or polars.LazyFrame."
+        )
+    if not columns:
+        raise ValueError("Provide at least one output-to-source column mapping.")
+    if on_conflict not in {"raise", "keep", "drop"}:
+        raise ValueError("on_conflict must be 'raise', 'keep', or 'drop'.")
+    if max_examples < 1:
+        raise ValueError("max_examples must be at least 1.")
+
+    context_columns = list(dict.fromkeys(context_columns or []))
+    is_lazy = isinstance(frame, pl.LazyFrame)
+    input_columns = frame.collect_schema().names() if is_lazy else frame.columns
+    existing = set(input_columns)
+    missing_context = [column for column in context_columns if column not in existing]
+    if missing_context:
+        raise ValueError(
+            "Missing diagnostic context columns: %s" % ", ".join(missing_context)
+        )
+
+    used_sources = set()
+    for output_column, source_columns in columns.items():
+        if not source_columns:
+            raise ValueError("Output %r has no source columns." % output_column)
+        if len(set(source_columns)) != len(source_columns):
+            raise ValueError("Output %r contains duplicate source columns." % output_column)
+        if output_column in source_columns:
+            raise ValueError("Output %r cannot also be one of its sources." % output_column)
+        if output_column in existing:
+            raise ValueError(
+                "Output %r already exists; choose a new output name or remove it first."
+                % output_column
+            )
+        if on_conflict == "keep" and "%s_conflict" % output_column in existing:
+            raise ValueError(
+                "Audit output %r already exists; remove it or choose a new output name."
+                % ("%s_conflict" % output_column)
+            )
+        missing = [column for column in source_columns if column not in existing]
+        if missing:
+            raise ValueError(
+                "Output %r has missing source columns: %s"
+                % (output_column, ", ".join(missing))
+            )
+        overlap = used_sources.intersection(source_columns)
+        if overlap:
+            raise ValueError(
+                "Source columns cannot feed more than one output: %s"
+                % ", ".join(sorted(overlap))
+            )
+        used_sources.update(source_columns)
+
+    diagnostic_columns = list(
+        dict.fromkeys(
+            list(context_columns)
+            + [
+                source_column
+                for source_columns in columns.values()
+                for source_column in source_columns
+            ]
+        )
+    )
+    diagnostic_source = frame.select(diagnostic_columns)
+    if isinstance(diagnostic_source, pl.LazyFrame):
+        diagnostic_source = diagnostic_source.collect()
+    row_number_column = _temporary_consolidation_row_number(input_columns)
+    outputs = {}
+    conflict_flags = {}
+    reports = {}
+    for output_column, source_columns in columns.items():
+        group_context = [
+            column for column in context_columns if column not in source_columns
+        ]
+        (
+            outputs[output_column],
+            conflict_flags[output_column],
+            reports[output_column],
+        ) = _analyze_consolidation_group(
+            diagnostic_source,
+            output_column,
+            source_columns,
+            group_context,
+            max_examples,
+            row_number_column,
+        )
+
+    diagnostics = ConsolidationDiagnostics(reports)
+    if diagnostics.has_conflicts and on_conflict == "raise":
+        action = "Source columns retained; no rows or columns changed because conflicts were detected."
+    elif diagnostics.has_conflicts and on_conflict == "keep":
+        action = (
+            "Source columns retained and 0/1 conflict flags added for audit; "
+            "no rows dropped."
+        )
+    elif diagnostics.has_conflicts:
+        action = (
+            "Source columns dropped as requested; conflicting output values "
+            "remain null; no rows dropped."
+        )
+    else:
+        action = "Source columns dropped after successful consolidation; no rows dropped."
+    print(diagnostics.format_report(action))
+    if diagnostics.has_conflicts and on_conflict == "raise":
+        raise ColumnConflictError(diagnostics)
+
+    update_values = dict(outputs)
+    if diagnostics.has_conflicts and on_conflict == "keep":
+        update_values.update(
+            {
+                "%s_conflict" % output_column: pl.Series(values, dtype=pl.Int8)
+                for output_column, values in conflict_flags.items()
+            }
+        )
+    if is_lazy:
+        update_values[row_number_column] = range(diagnostic_source.height)
+        updates = pl.DataFrame(update_values).lazy()
+        result = (
+            frame.with_row_index(row_number_column)
+            .join(updates, on=row_number_column, how="left")
+            .drop(row_number_column)
+        )
+    else:
+        result = frame.with_columns(
+            [pl.Series(output_column, values) for output_column, values in update_values.items()]
+        )
+    if not diagnostics.has_conflicts or on_conflict == "drop":
+        result = result.drop(list(used_sources))
+
+    if return_diagnostics:
+        return result, diagnostics
+    return result
+
+
+###############################################################################
+# DEMOCRACY PREP LOTTERY COLUMN SELECTION
+###############################################################################
+
+def select_lottery_columns(frame: Frame) -> Frame:
+    """Select the requested lottery fields and rename direct schema matches."""
+    return frame.select(
+        [
+            "origorder",
+            "year",
+            "applygrade",
+            "firstname",
+            "lastname",
+            "lottstatus",
+            "offered",
+            "address",
+            "apt",
+            "zip",
+            "pfirstname",
+            "plastname",
+            "dob_orig",
+            "waitlist",
+            "dob",
+            "sibling1name",
+            "sibling1status",
+            "sibling1school",
+            "sibling1grade",
+            "in_dist",
+            "number",
+            "oth_district",
+            "in_district",
+            "currentschool2",
+            "datereceived",
+            "siblingcurrent",
+            "offered_school",
+            "currentgrade",
+            "dob2",
+            "status",
+            "appno",
+            "app_hpcs",
+            "app_dpe",
+            "app_dph",
+            "app_dpcs",
+            "guardian2",
+            "sibling2name",
+            "sibling2status",
+            "sibling2school",
+            "sibling2grade",
+            "sibling3name",
+            "sibling3status",
+            "sibling3school",
+            "sibling3grade",
+            "actualdistrict",
+            "waitoffer",
+            "waitaccept",
+            "waitacceptdate",
+            "waitdeclinedate",
+            "waitacceptschool",
+            "appid",
+            "hpcs",
+            "dpe",
+            "dph",
+            "dpcs",
+            "acceptedschoolname",
+            "guardian1name",
+            "hpcsletter",
+            "dpeletter",
+            "dphletter",
+            "dpcsletter",
+            "anyletter",
+            "currentgrade_orig",
+            "siblingstatus",
+            "guardian1lastname",
+            "guardian1firstname",
+            "guardian1street",
+            "guardian1apt",
+            "guardian1city",
+            "guardian1zipcode",
+            "guardian2lastname",
+            "guardian2firstname",
+            "guardian2street",
+            "guardian2apt",
+            "guardian2city",
+            "guardian2zipcode",
+            "currentschoolpreference",
+            "applicationsource",
+            "waitlistinfo",
+            "offer_dpcs2",
+            "offer_dpe2",
+            "offer_dph2",
+            "offer_hpcs2",
+            "wait_dpcs2",
+            "wait_dpe2",
+            "wait_dph2",
+            "wait_hpcs2",
+            "whichschl",
+            "dob_orig1415",
+            "graderank_hpcs",
+            "graderank_dph",
+            "graderank_dpcs",
+            "graderank_dpe",
+            "prigroup",
+            "preferencesort_hpcs",
+            "preferencesort_dph",
+            "preferencesort_dpcs",
+            "preferencesort_dpe",
+            "waitlist_best",
+            "offered2",
+            "Sid2",
+            "sid_impute",
+            "geodist",
+        ]
+    ).rename(
+        {
+            "year": "school_year",
+            "applygrade": "entry_grade",
+            "firstname": "fname",
+            "lastname": "lname",
+            "apt": "address2",
+            "pfirstname": "p_fname",
+            "plastname": "p_lname",
+            "waitlist": "waitlist_number",
+            "datereceived": "application_date",
+            "whichschl": "school_name",
+            "guardian1name": "p_name",
+            "guardian1firstname": "p1_fname",
+            "guardian1lastname": "p1_lname",
+            "guardian2firstname": "p2_fname",
+            "guardian2lastname": "p2_lname",
+        }
+    )
+
+
+###############################################################################
+# ENR DATA COLUMN SELECTION
+###############################################################################
+
+def select_enr_columns(frame: Frame) -> Frame:
+    """Select ENR fields and rename direct centralized-schema matches."""
+    return frame.select(
+        [
+            "year",
+            "datasource",
+            "dbn_oct31",
+            "dbn_sis",
+            "first_name",
+            "middle_name",
+            "last_name",
+            "dob",
+            "dob_sis",
+            "mealcode",
+            "admitdate",
+            "home_lang",
+            "pob_code",
+            "grade_level",
+            "official_class",
+            "createdate",
+            "spec_ed_flag",
+            "school_level",
+            "building code",
+            "residencestreetaddress",
+            "residencezip",
+            "contact1name",
+            "contact1streetaddress",
+            "contact1zip",
+        ]
+    ).rename(
+        {
+            "year": "school_year",
+            "first_name": "fname",
+            "middle_name": "mname",
+            "last_name": "lname",
+            "grade_level": "enrollment_grade",
+            "residencestreetaddress": "address",
+            "residencezip": "zip",
+            "contact1name": "p_name",
+            "spec_ed_flag": "sped",
+        }
+    )