whitebox-dedup 0.1.0__py3-none-any.whl

whitebox_dedup/__init__.py

@@ -0,0 +1,27 @@
+"""
+whitebox-dedup: Interpretable entity deduplication with auditable MARS equations
+and Inductive Conformal Prediction.
+
+Developed by FutuX Agri-consult Limited.
+
+Key exports:
+- DeduplicationPipeline: end-to-end deduplication runner
+- EntitySchema: declares the semantic role of each column
+- MARSScorer: zero-dependency MARS equation evaluator
+- DeduplicationResult: structured output from pipeline.run()
+"""
+from .pipeline import DeduplicationPipeline, DeduplicationResult
+from .schema import EntitySchema
+from .mars_scorer import MARSScorer
+from .trainer import MARSTrainer
+
+__all__ = [
+    "DeduplicationPipeline",
+    "EntitySchema",
+    "MARSScorer",
+    "MARSTrainer",
+    "DeduplicationResult",
+]
+
+__version__ = "0.1.0"
+__author__ = "Daniel Olagunju, Temitope Olarewaju"

whitebox_dedup/__main__.py

@@ -0,0 +1,5 @@
+"""Package entry point for `python -m whitebox_dedup`."""
+from whitebox_dedup.cli import main
+
+if __name__ == "__main__":
+    main()

whitebox_dedup/blocking.py

@@ -0,0 +1,115 @@
+"""
+Blocking strategies to generate candidate duplicate pairs efficiently.
+
+Without blocking, comparing every pair in N records requires N*(N-1)/2 comparisons.
+Blocking groups records into buckets where duplicates are likely to be found,
+then only compares records within the same bucket.
+"""
+import math
+from typing import Iterator
+import pandas as pd
+from .schema import EntitySchema, FieldRole
+
+
+def generate_candidate_pairs(
+    df: pd.DataFrame,
+    schema: EntitySchema,
+    geo_radius_m: float = 50.0,
+) -> list[tuple[int, int]]:
+    """
+    Generate candidate duplicate pairs using available blocking strategies.
+    Returns list of (row_index_a, row_index_b) tuples.
+    Deduplicates pairs (always returns (i, j) where i < j).
+    """
+    pairs: set[tuple[int, int]] = set()
+
+    # Strategy 1: First-letter blocking on name fields
+    name_cols = schema.fields_by_role(FieldRole.NAME)
+    if name_cols:
+        pairs.update(_name_block(df, name_cols[0]))
+
+    # Strategy 2: Exact phone blocking
+    phone_cols = schema.fields_by_role(FieldRole.PHONE)
+    if phone_cols:
+        pairs.update(_exact_block(df, phone_cols[0]))
+
+    # Strategy 3: Exact ID blocking (to detect hard duplicates)
+    id_cols = schema.fields_by_role(FieldRole.ID)
+    if id_cols:
+        pairs.update(_exact_block(df, id_cols[0]))
+
+    # Strategy 4: Spatial blocking (haversine <= geo_radius_m)
+    lat_cols = schema.fields_by_role(FieldRole.LATITUDE)
+    lon_cols = schema.fields_by_role(FieldRole.LONGITUDE)
+    if lat_cols and lon_cols:
+        pairs.update(_geo_block(df, lat_cols[0], lon_cols[0], geo_radius_m))
+
+    return sorted(pairs)
+
+
+def _name_block(df: pd.DataFrame, col: str) -> set[tuple[int, int]]:
+    """Block by first 2 characters of name field."""
+    if col not in df.columns:
+        return set()
+    buckets: dict[str, list[int]] = {}
+    for idx, val in df[col].items():
+        key = str(val or "")[:2].lower()
+        if key:
+            buckets.setdefault(key, []).append(idx)
+    return _pairs_from_buckets(buckets)
+
+
+def _exact_block(df: pd.DataFrame, col: str) -> set[tuple[int, int]]:
+    """Block by exact value match (detects hard duplicates)."""
+    if col not in df.columns:
+        return set()
+    buckets: dict[str, list[int]] = {}
+    for idx, val in df[col].items():
+        key = str(val or "").strip().lower()
+        if key:
+            buckets.setdefault(key, []).append(idx)
+    # Only keep buckets with > 1 record (actual duplicates possible)
+    return _pairs_from_buckets({k: v for k, v in buckets.items() if len(v) > 1})
+
+
+def _geo_block(df: pd.DataFrame, lat_col: str, lon_col: str, radius_m: float) -> set[tuple[int, int]]:
+    """
+    Spatial blocking: pair records within radius_m metres of each other.
+    Uses a simple grid cell approach: approximately 0.001 degree = 111m resolution.
+    """
+    if lat_col not in df.columns or lon_col not in df.columns:
+        return set()
+
+    cell_deg = radius_m / 111_000.0
+    buckets: dict[tuple[int, int], list[int]] = {}
+    for idx, row in df[[lat_col, lon_col]].iterrows():
+        try:
+            lat, lon = float(row[lat_col]), float(row[lon_col])
+            cell = (int(lat / cell_deg), int(lon / cell_deg))
+            buckets.setdefault(cell, []).append(idx)
+        except (ValueError, TypeError):
+            continue
+
+    # Also check adjacent cells
+    pairs: set[tuple[int, int]] = set()
+    cells = list(buckets.keys())
+    for i, cell in enumerate(cells):
+        members = buckets[cell]
+        for di in [-1, 0, 1]:
+            for dj in [-1, 0, 1]:
+                neighbor = (cell[0] + di, cell[1] + dj)
+                if neighbor in buckets:
+                    for a in members:
+                        for b in buckets[neighbor]:
+                            if a < b:
+                                pairs.add((a, b))
+    return pairs
+
+
+def _pairs_from_buckets(buckets: dict) -> set[tuple[int, int]]:
+    pairs: set[tuple[int, int]] = set()
+    for members in buckets.values():
+        for i in range(len(members)):
+            for j in range(i + 1, len(members)):
+                pairs.add((min(members[i], members[j]), max(members[i], members[j])))
+    return pairs

whitebox_dedup/cli.py

@@ -0,0 +1,199 @@
+"""
+CLI for whitebox-dedup.
+
+Commands:
+  wdedup deduplicate --input FILE [--schema YAML] [--output JSON]
+  wdedup score-pair  [--schema YAML]
+  wdedup show-equation [--model MODEL_NAME]
+"""
+from __future__ import annotations
+import json
+import sys
+from pathlib import Path
+
+
+def _print_err(msg: str) -> None:
+    print(f"[error] {msg}", file=sys.stderr)
+
+
+def _deduplicate_cmd(args) -> int:
+    """Run full deduplication on a CSV file."""
+    from .pipeline import DeduplicationPipeline
+    from .schema import EntitySchema
+
+    input_path = Path(args.input)
+    if not input_path.exists():
+        _print_err(f"Input file not found: {input_path}")
+        return 1
+
+    schema = None
+    if args.schema:
+        schema_path = Path(args.schema)
+        if not schema_path.exists():
+            _print_err(f"Schema file not found: {schema_path}")
+            return 1
+        schema = EntitySchema.from_yaml(schema_path)
+
+    print(f"Loading data from: {input_path}")
+    pipeline = DeduplicationPipeline.from_csv(input_path, schema=schema)
+    print(f"Schema active features: {sorted(pipeline.schema.active_features())}")
+    print(f"Model: {pipeline.scorer.model_name}")
+    print("Running deduplication...")
+
+    result = pipeline.run()
+    summary = result.summary()
+
+    print("\nResults:")
+    print(f"  Records processed:       {summary['total_records']}")
+    print(f"  Candidate pairs:         {summary['candidate_pairs_evaluated']}")
+    print(f"  Tier 1 (auto-merge):     {summary['tier1_auto_merge']}")
+    print(f"  Tier 2 (human review):   {summary['tier2_human_review']}")
+    print(f"  Tier 3 (auto-distinct):  {summary['tier3_auto_distinct']}")
+
+    if args.output:
+        output_path = Path(args.output)
+        output_data = {
+            "summary": summary,
+            "tier1_auto_merge": result.tier1_auto_merge,
+            "tier2_human_review": result.tier2_human_review,
+            "tier3_auto_distinct": result.tier3_auto_distinct,
+        }
+        with open(output_path, "w") as f:
+            json.dump(output_data, f, indent=2, default=str)
+        print(f"\nFull results written to: {output_path}")
+
+    return 0
+
+
+def _score_pair_cmd(args) -> int:
+    """Interactively score a single pair of records."""
+    from .pipeline import DeduplicationPipeline
+    from .schema import EntitySchema
+
+    schema = None
+    if args.schema:
+        schema_path = Path(args.schema)
+        if not schema_path.exists():
+            _print_err(f"Schema file not found: {schema_path}")
+            return 1
+        schema = EntitySchema.from_yaml(schema_path)
+    else:
+        print("No schema provided. Using default agricultural schema.")
+        from .schema import EntityField, FieldRole
+        schema = EntitySchema(fields={
+            "full_name": EntityField("full_name", FieldRole.NAME),
+            "phone": EntityField("phone", FieldRole.PHONE),
+            "state": EntityField("state", FieldRole.ADMIN_1),
+        })
+
+    print("Enter record A as JSON (e.g. {\"full_name\": \"Aminu Bello\", \"phone\": \"08012345678\"}):")
+    try:
+        line_a = input("> ").strip()
+        row_a = json.loads(line_a)
+    except (json.JSONDecodeError, EOFError) as exc:
+        _print_err(f"Invalid JSON for record A: {exc}")
+        return 1
+
+    print("Enter record B as JSON:")
+    try:
+        line_b = input("> ").strip()
+        row_b = json.loads(line_b)
+    except (json.JSONDecodeError, EOFError) as exc:
+        _print_err(f"Invalid JSON for record B: {exc}")
+        return 1
+
+    from .mars_scorer import MARSScorer
+    from .router import DecisionRouter
+
+    scorer = DeduplicationPipeline._load_best_model(schema)
+    try:
+        from .conformal import ICPPredictor
+        icp = ICPPredictor.load_pretrained(scorer.model_name)
+    except FileNotFoundError:
+        icp = None
+
+    router = DecisionRouter(scorer, icp)
+    from .features import extract_all
+    feats = extract_all(row_a, row_b, schema)
+    routing = router.route(feats, schema)
+
+    print(f"\nTier:        {routing.tier.value}")
+    print(f"Probability: {routing.probability:.4f}")
+    print(f"Reason:      {routing.reason}")
+    print(f"Features:    {json.dumps({k: round(v, 4) if v is not None else None for k, v in feats.items()}, indent=2)}")
+
+    if routing.mars_trace:
+        print("\nMARS trace:")
+        for contrib in routing.mars_trace["hinge_contributions"]:
+            print(f"  {contrib['feature']} hinge at {contrib['knot']}: {contrib['hinge_output']:.4f}")
+
+    return 0
+
+
+def _show_equation_cmd(args) -> int:
+    """Print the MARS equation for a given model."""
+    from .mars_scorer import MARSScorer
+
+    model_name = args.model or "text_only_v1"
+    try:
+        scorer = MARSScorer.load_pretrained(model_name)
+    except FileNotFoundError as exc:
+        _print_err(str(exc))
+        return 1
+
+    print(f"Model: {scorer.model_name}")
+    if scorer.description:
+        print(f"Description: {scorer.description}")
+    print()
+    print(scorer.equation_text())
+    return 0
+
+
+def main() -> None:
+    """Entry point for the wdedup CLI."""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        prog="wdedup",
+        description="whitebox-dedup: interpretable entity deduplication",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    # deduplicate command
+    dedup_parser = subparsers.add_parser(
+        "deduplicate",
+        help="Deduplicate a CSV file",
+    )
+    dedup_parser.add_argument("--input", required=True, help="Path to input CSV file")
+    dedup_parser.add_argument("--schema", default=None, help="Path to schema YAML file (optional)")
+    dedup_parser.add_argument("--output", default=None, help="Path to output JSON file (optional)")
+
+    # score-pair command
+    score_parser = subparsers.add_parser(
+        "score-pair",
+        help="Interactively score a single pair of records",
+    )
+    score_parser.add_argument("--schema", default=None, help="Path to schema YAML file (optional)")
+
+    # show-equation command
+    eq_parser = subparsers.add_parser(
+        "show-equation",
+        help="Print the MARS equation for a model",
+    )
+    eq_parser.add_argument("--model", default="text_only_v1", help="Model name (default: text_only_v1)")
+
+    args = parser.parse_args()
+
+    if args.command == "deduplicate":
+        sys.exit(_deduplicate_cmd(args))
+    elif args.command == "score-pair":
+        sys.exit(_score_pair_cmd(args))
+    elif args.command == "show-equation":
+        sys.exit(_show_equation_cmd(args))
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

whitebox_dedup/conformal.py

@@ -0,0 +1,96 @@
+"""
+Inductive Conformal Prediction (ICP) for deduplication.
+
+ICP wraps the MARS scorer to produce a prediction SET {0}, {1}, or {0,1}
+at a user-specified coverage level (e.g., 95%).
+
+When the prediction set is {0,1} (ambiguous), the circuit breaker fires:
+the system cannot make an automated decision and diverts to human review.
+
+Coverage stratified by Jaro-Winkler similarity shows that coverage collapses
+in the ambiguous zone (JW >= 0.491), establishing an automated statutory trigger.
+"""
+from __future__ import annotations
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+
+@dataclass
+class ICPCalibration:
+    """Pre-computed nonconformity score thresholds."""
+    alpha: float                    # significance level (e.g., 0.05 for 95% coverage)
+    threshold_duplicate: float      # nonconformity threshold for class 1 (duplicate)
+    threshold_distinct: float       # nonconformity threshold for class 0 (distinct)
+    model_name: str = ""
+
+    @classmethod
+    def load(cls, path: str | Path) -> "ICPCalibration":
+        with open(path) as f:
+            data = json.load(f)
+        return cls(
+            alpha=data["alpha"],
+            threshold_duplicate=data["threshold_duplicate"],
+            threshold_distinct=data["threshold_distinct"],
+            model_name=data.get("model_name", ""),
+        )
+
+
+class ICPPredictor:
+    """
+    Produces conformal prediction sets given a MARS probability score.
+
+    A prediction set of {1} means "definitely duplicate at the stated coverage level".
+    A prediction set of {0} means "definitely distinct".
+    A prediction set of {0, 1} means "ambiguous: circuit breaker fires".
+    """
+
+    def __init__(self, calibration: ICPCalibration):
+        self.cal = calibration
+
+    @classmethod
+    def load_pretrained(cls, model_name: str) -> "ICPPredictor":
+        cal_dir = Path(__file__).parent.parent / "models" / "calibration"
+        path = cal_dir / f"{model_name}_icp.json"
+        if not path.exists():
+            raise FileNotFoundError(f"ICP calibration not found: {path}")
+        return cls(ICPCalibration.load(path))
+
+    def predict_set(self, probability: float) -> frozenset[int]:
+        """
+        Return the conformal prediction set for a given duplicate probability.
+
+        The nonconformity score for class 1 is (1 - p), for class 0 is p.
+        A label y is included in the prediction set if its nonconformity
+        score <= the calibration threshold.
+        """
+        nc_1 = 1.0 - probability     # nonconformity for "duplicate"
+        nc_0 = probability           # nonconformity for "distinct"
+
+        include_1 = nc_1 <= self.cal.threshold_duplicate
+        include_0 = nc_0 <= self.cal.threshold_distinct
+
+        result = set()
+        if include_1:
+            result.add(1)
+        if include_0:
+            result.add(0)
+        if not result:
+            result = {0, 1}   # always return non-empty set
+        return frozenset(result)
+
+    def is_ambiguous(self, probability: float) -> bool:
+        return self.predict_set(probability) == frozenset({0, 1})
+
+    def coverage_zone(self, jw_score: float) -> str:
+        """
+        Classify coverage zone from JW similarity.
+        Based on empirical finding: coverage collapses to 77.93% when JW >= 0.491.
+        """
+        if jw_score < 0.491:
+            return "high_coverage"       # >= 95% empirical coverage
+        elif jw_score >= 0.898:
+            return "high_coverage"       # very high similarity -> clear duplicate
+        else:
+            return "ambiguous_zone"      # coverage 77.93% -> circuit breaker

whitebox_dedup/features.py

@@ -0,0 +1,166 @@
+"""
+Pairwise feature extraction for entity deduplication.
+
+All features take two record dicts (row_a, row_b) and an EntitySchema,
+and return a float value. Missing columns return None (feature skipped).
+"""
+import math
+import re
+from typing import Optional
+import jellyfish
+
+
+def jw_name_full(row_a: dict, row_b: dict, name_cols: list[str]) -> Optional[float]:
+    """Jaro-Winkler similarity on concatenated name tokens."""
+    if not name_cols:
+        return None
+    a = " ".join(str(row_a.get(c, "") or "").strip() for c in name_cols).lower()
+    b = " ".join(str(row_b.get(c, "") or "").strip() for c in name_cols).lower()
+    if not a or not b:
+        return None
+    return jellyfish.jaro_winkler_similarity(a, b)
+
+
+def jw_name_first(row_a: dict, row_b: dict, name_cols: list[str]) -> Optional[float]:
+    """Jaro-Winkler on first token of primary name field."""
+    if not name_cols:
+        return None
+    a_tokens = str(row_a.get(name_cols[0], "") or "").split()
+    b_tokens = str(row_b.get(name_cols[0], "") or "").split()
+    if not a_tokens or not b_tokens:
+        return None
+    return jellyfish.jaro_winkler_similarity(a_tokens[0].lower(), b_tokens[0].lower())
+
+
+def jw_name_last(row_a: dict, row_b: dict, name_cols: list[str]) -> Optional[float]:
+    """Jaro-Winkler on last token of primary name field."""
+    if not name_cols:
+        return None
+    a_tokens = str(row_a.get(name_cols[0], "") or "").split()
+    b_tokens = str(row_b.get(name_cols[0], "") or "").split()
+    if not a_tokens or not b_tokens:
+        return None
+    return jellyfish.jaro_winkler_similarity(a_tokens[-1].lower(), b_tokens[-1].lower())
+
+
+def haversine_m(row_a: dict, row_b: dict, lat_col: str, lon_col: str) -> Optional[float]:
+    """Great-circle distance in metres between two GPS coordinates."""
+    try:
+        lat1, lon1 = float(row_a[lat_col]), float(row_a[lon_col])
+        lat2, lon2 = float(row_b[lat_col]), float(row_b[lon_col])
+    except (KeyError, TypeError, ValueError):
+        return None
+    R = 6_371_000.0
+    phi1, phi2 = math.radians(lat1), math.radians(lat2)
+    dphi = math.radians(lat2 - lat1)
+    dlambda = math.radians(lon2 - lon1)
+    a = math.sin(dphi / 2) ** 2 + math.cos(phi1) * math.cos(phi2) * math.sin(dlambda / 2) ** 2
+    return R * 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
+
+
+def _normalize_phone(phone: str) -> str:
+    return re.sub(r"[\s\-().+]", "", str(phone or ""))
+
+
+def exact_phone(row_a: dict, row_b: dict, phone_col: str) -> Optional[float]:
+    """1.0 if phone numbers match after normalisation, 0.0 otherwise."""
+    a = _normalize_phone(row_a.get(phone_col, ""))
+    b = _normalize_phone(row_b.get(phone_col, ""))
+    if not a or not b:
+        return None
+    return 1.0 if a == b else 0.0
+
+
+def exact_id(row_a: dict, row_b: dict, id_col: str) -> Optional[float]:
+    """1.0 if unique ID values match, 0.0 if both non-null and different."""
+    a = str(row_a.get(id_col, "") or "").strip()
+    b = str(row_b.get(id_col, "") or "").strip()
+    if not a or not b:
+        return None
+    return 1.0 if a.lower() == b.lower() else 0.0
+
+
+def same_admin(row_a: dict, row_b: dict, col: str) -> Optional[float]:
+    """1.0 if same administrative unit, 0.0 otherwise."""
+    a = str(row_a.get(col, "") or "").strip().lower()
+    b = str(row_b.get(col, "") or "").strip().lower()
+    if not a or not b:
+        return None
+    return 1.0 if a == b else 0.0
+
+
+def dob_match(row_a: dict, row_b: dict, dob_col: str) -> Optional[float]:
+    """1.0 if date of birth strings match exactly (after stripping)."""
+    a = str(row_a.get(dob_col, "") or "").strip()
+    b = str(row_b.get(dob_col, "") or "").strip()
+    if not a or not b:
+        return None
+    return 1.0 if a == b else 0.0
+
+
+def same_gender(row_a: dict, row_b: dict, gender_col: str) -> Optional[float]:
+    """1.0 if gender values match (normalised to first letter)."""
+    a = str(row_a.get(gender_col, "") or "")[:1].upper()
+    b = str(row_b.get(gender_col, "") or "")[:1].upper()
+    if not a or not b:
+        return None
+    return 1.0 if a == b else 0.0
+
+
+def same_enumerator(row_a: dict, row_b: dict, enum_col: str) -> Optional[float]:
+    """1.0 if records were collected by the same field agent."""
+    a = str(row_a.get(enum_col, "") or "").strip()
+    b = str(row_b.get(enum_col, "") or "").strip()
+    if not a or not b:
+        return None
+    return 1.0 if a == b else 0.0
+
+
+def extract_all(row_a: dict, row_b: dict, schema) -> dict[str, Optional[float]]:
+    """
+    Extract all available features for a pair given the EntitySchema.
+    Returns a dict of feature_name -> value (None if feature unavailable).
+    """
+    from .schema import FieldRole
+    result: dict[str, Optional[float]] = {}
+    active = schema.active_features()
+
+    name_cols = schema.fields_by_role(FieldRole.NAME)
+    if "jw_name_full"  in active: result["jw_name_full"]  = jw_name_full(row_a, row_b, name_cols)
+    if "jw_name_first" in active: result["jw_name_first"] = jw_name_first(row_a, row_b, name_cols)
+    if "jw_name_last"  in active: result["jw_name_last"]  = jw_name_last(row_a, row_b, name_cols)
+
+    id_cols = schema.fields_by_role(FieldRole.ID)
+    if "exact_id" in active and id_cols:
+        result["exact_id"] = exact_id(row_a, row_b, id_cols[0])
+
+    phone_cols = schema.fields_by_role(FieldRole.PHONE)
+    if "exact_phone" in active and phone_cols:
+        result["exact_phone"] = exact_phone(row_a, row_b, phone_cols[0])
+
+    lat_cols = schema.fields_by_role(FieldRole.LATITUDE)
+    lon_cols = schema.fields_by_role(FieldRole.LONGITUDE)
+    if "haversine_m" in active and lat_cols and lon_cols:
+        result["haversine_m"] = haversine_m(row_a, row_b, lat_cols[0], lon_cols[0])
+
+    a1_cols = schema.fields_by_role(FieldRole.ADMIN_1)
+    if "same_admin_1" in active and a1_cols:
+        result["same_admin_1"] = same_admin(row_a, row_b, a1_cols[0])
+
+    a2_cols = schema.fields_by_role(FieldRole.ADMIN_2)
+    if "same_admin_2" in active and a2_cols:
+        result["same_admin_2"] = same_admin(row_a, row_b, a2_cols[0])
+
+    dob_cols = schema.fields_by_role(FieldRole.DATE_OF_BIRTH)
+    if "dob_match" in active and dob_cols:
+        result["dob_match"] = dob_match(row_a, row_b, dob_cols[0])
+
+    gender_cols = schema.fields_by_role(FieldRole.GENDER)
+    if "same_gender" in active and gender_cols:
+        result["same_gender"] = same_gender(row_a, row_b, gender_cols[0])
+
+    enum_cols = schema.fields_by_role(FieldRole.ENUMERATOR)
+    if "same_enumerator" in active and enum_cols:
+        result["same_enumerator"] = same_enumerator(row_a, row_b, enum_cols[0])
+
+    return result