unitsleuth 0.1.0__py3-none-any.whl

unitsleuth/__init__.py

@@ -0,0 +1,19 @@
+"""unitsleuth: detect mixed measurement units in numeric columns."""
+
+from unitsleuth import catalog
+from unitsleuth.catalog import Conversion
+from unitsleuth.detect import UnivariateFit, detect, fit_conversion
+from unitsleuth.multivariate import RowFit, detect_rows, repair
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Conversion",
+    "RowFit",
+    "UnivariateFit",
+    "catalog",
+    "detect",
+    "detect_rows",
+    "fit_conversion",
+    "repair",
+]

unitsleuth/__main__.py

@@ -0,0 +1,3 @@
+from unitsleuth.cli import main
+
+raise SystemExit(main())

unitsleuth/_frame.py

@@ -0,0 +1,50 @@
+"""Dataframe adapters built on Narwhals (pandas, Polars, PyArrow, ...)."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+import narwhals.stable.v1 as nw
+import numpy as np
+from numpy.typing import ArrayLike, NDArray
+
+FloatArray = NDArray[np.float64]
+
+
+def is_frame(obj: Any) -> bool:
+    """True for native eager dataframes that Narwhals understands."""
+    if isinstance(obj, Mapping):
+        return False
+    try:
+        nw.from_native(obj, eager_only=True)
+    except TypeError:
+        return False
+    return True
+
+
+def columns_of(data: Any, names: list[str] | tuple[str, ...]) -> dict[str, ArrayLike]:
+    """Extract named columns as float arrays from a mapping or a dataframe."""
+    if isinstance(data, Mapping):
+        return {n: data[n] for n in names}
+    frame = nw.from_native(data, eager_only=True)
+    missing = [n for n in names if n not in frame.columns]
+    if missing:
+        raise KeyError(f"columns not in data: {missing}")
+    out: dict[str, ArrayLike] = {}
+    for n in names:
+        s = frame[n].cast(nw.Float64)
+        out[n] = np.array(s.to_list(), dtype=np.float64)
+        # to_list maps nulls to None; np.array(..., float64) turns None into nan
+    return out
+
+
+def replace_columns(data: Any, cols: Mapping[str, FloatArray]) -> Any:
+    """Return a native dataframe of the same type with ``cols`` replaced."""
+    frame = nw.from_native(data, eager_only=True)
+    ns = nw.get_native_namespace(frame)
+    new = [
+        nw.new_series(name, arr.tolist(), nw.Float64, native_namespace=ns)
+        for name, arr in cols.items()
+    ]
+    return frame.with_columns(*new).to_native()

unitsleuth/catalog.py

@@ -0,0 +1,117 @@
+"""Catalog of unit conversions that unitsleuth can test for.
+
+Every conversion is affine: ``x_to = scale * x_from + offset``. Multiplicative
+conversions (``offset == 0``) cover most unit pairs; temperature needs the offset.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+from numpy.typing import ArrayLike, NDArray
+
+FloatArray = NDArray[np.float64]
+
+
+@dataclass(frozen=True)
+class Conversion:
+    """An affine map between two units of the same quantity."""
+
+    quantity: str
+    from_unit: str
+    to_unit: str
+    scale: float
+    offset: float = 0.0
+
+    def __post_init__(self) -> None:
+        if not (np.isfinite(self.scale) and np.isfinite(self.offset)):
+            raise ValueError("scale and offset must be finite")
+        if self.scale == 0.0:
+            raise ValueError("scale must be non-zero")
+
+    @property
+    def name(self) -> str:
+        """Short label such as ``'kg->lb'``."""
+        return f"{self.from_unit}->{self.to_unit}"
+
+    @property
+    def is_multiplicative(self) -> bool:
+        """True when the conversion has no additive offset."""
+        return self.offset == 0.0
+
+    def forward(self, x: ArrayLike) -> FloatArray:
+        """Convert values expressed in ``from_unit`` to ``to_unit``."""
+        arr = np.asarray(x, dtype=np.float64)
+        return np.asarray(self.scale * arr + self.offset, dtype=np.float64)
+
+    def inverse(self, y: ArrayLike) -> FloatArray:
+        """Convert values expressed in ``to_unit`` back to ``from_unit``."""
+        arr = np.asarray(y, dtype=np.float64)
+        return np.asarray((arr - self.offset) / self.scale, dtype=np.float64)
+
+
+def _c(
+    quantity: str, from_unit: str, to_unit: str, scale: float, offset: float = 0.0
+) -> Conversion:
+    return Conversion(quantity, from_unit, to_unit, scale, offset)
+
+
+_CATALOG: tuple[Conversion, ...] = (
+    # length
+    _c("length", "m", "ft", 3.280839895013123),
+    _c("length", "cm", "in", 1.0 / 2.54),
+    _c("length", "km", "mi", 0.621371192237334),
+    _c("length", "m", "cm", 100.0),
+    _c("length", "ft", "in", 12.0),
+    # mass
+    _c("mass", "kg", "lb", 2.2046226218487757),
+    _c("mass", "kg", "g", 1000.0),
+    _c("mass", "lb", "oz", 16.0),
+    _c("mass", "g", "oz", 0.035273961949580414),
+    # temperature
+    _c("temperature", "degC", "degF", 1.8, 32.0),
+    _c("temperature", "degC", "K", 1.0, 273.15),
+    # volume
+    _c("volume", "L", "mL", 1000.0),
+    _c("volume", "L", "gal_us", 0.26417205235814845),
+    # pressure
+    _c("pressure", "kPa", "Pa", 1000.0),
+    _c("pressure", "kPa", "psi", 0.14503773773375),
+    _c("pressure", "mmHg", "kPa", 0.133322387415),
+    # speed
+    _c("speed", "km/h", "mph", 0.621371192237334),
+    _c("speed", "km/h", "m/s", 1.0 / 3.6),
+    # time
+    _c("time", "h", "min", 60.0),
+    _c("time", "min", "s", 60.0),
+    # proportions
+    _c("ratio", "fraction", "percent", 100.0),
+    # generic power-of-ten scale errors (classic keying/recording mistakes)
+    _c("generic", "x1", "x10", 10.0),
+    _c("generic", "x1", "x100", 100.0),
+    _c("generic", "x1", "x1000", 1000.0),
+)
+
+
+def conversions(quantity: str | None = None) -> tuple[Conversion, ...]:
+    """Return the built-in conversions, optionally filtered by quantity."""
+    if quantity is None:
+        return _CATALOG
+    selected = tuple(c for c in _CATALOG if c.quantity == quantity)
+    if not selected:
+        raise KeyError(f"unknown quantity {quantity!r}; known: {quantities()}")
+    return selected
+
+
+def quantities() -> tuple[str, ...]:
+    """Return the quantities covered by the catalog, in catalog order."""
+    return tuple(dict.fromkeys(c.quantity for c in _CATALOG))
+
+
+def get(from_unit: str, to_unit: str) -> Conversion:
+    """Look up a built-in conversion by its unit names."""
+    for conv in _CATALOG:
+        if conv.from_unit == from_unit and conv.to_unit == to_unit:
+            return conv
+    raise KeyError(f"no built-in conversion {from_unit}->{to_unit}")

unitsleuth/cli.py

@@ -0,0 +1,104 @@
+"""Command-line interface: ``unitsleuth data.csv --col height:cm,in``."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import sys
+from collections.abc import Sequence
+
+import numpy as np
+
+from unitsleuth import catalog
+from unitsleuth.catalog import Conversion
+from unitsleuth.multivariate import detect_rows, repair
+
+
+def _resolve(spec: str) -> tuple[str, Conversion]:
+    try:
+        name, units = spec.rsplit(":", 1)
+        a, b = (u.strip() for u in units.split(","))
+    except ValueError:
+        raise argparse.ArgumentTypeError(
+            f"bad --col {spec!r}; expected NAME:FROM,TO such as height:cm,in"
+        ) from None
+    try:
+        return name, catalog.get(a, b)
+    except KeyError:
+        pass
+    try:
+        c = catalog.get(b, a)
+    except KeyError:
+        raise argparse.ArgumentTypeError(
+            f"no built-in conversion between {a!r} and {b!r}"
+        ) from None
+    return name, Conversion(c.quantity, a, b, 1.0 / c.scale, -c.offset / c.scale)
+
+
+def _num(s: str) -> float:
+    try:
+        return float(s)
+    except ValueError:
+        return float("nan")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="unitsleuth",
+        description="Detect rows recorded in a different measurement unit.",
+    )
+    p.add_argument("csv", help="input CSV file with a header row")
+    p.add_argument(
+        "--col",
+        action="append",
+        required=True,
+        type=_resolve,
+        metavar="NAME:FROM,TO",
+        help="column and the conversion to test, e.g. weight:kg,lb (repeatable)",
+    )
+    p.add_argument("--out", help="write a repaired CSV to this path")
+    p.add_argument(
+        "--target",
+        choices=["majority", "base", "converted"],
+        default="majority",
+        help="unit to harmonise to when repairing (default: majority)",
+    )
+    p.add_argument("--min-delta-bic", type=float, default=10.0)
+    return p
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Run the CLI. Exit code 1 means mixed units were detected."""
+    args = build_parser().parse_args(argv)
+    convs = dict(args.col)
+    with open(args.csv, newline="", encoding="utf-8") as fh:
+        reader = csv.DictReader(fh)
+        fields = list(reader.fieldnames or [])
+        rows = list(reader)
+    missing = [c for c in convs if c not in fields]
+    if missing:
+        print(f"error: columns not in file: {missing}", file=sys.stderr)
+        return 2
+    data = {c: np.array([_num(r[c]) for r in rows]) for c in convs}
+    try:
+        fit = detect_rows(data, convs, min_delta_bic=args.min_delta_bic)
+    except ValueError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+    print(fit.summary())
+    if args.out:
+        fixed = repair(data, fit, target=args.target)
+        for i, r in enumerate(rows):
+            for c in convs:
+                if np.isfinite(fixed[c][i]):
+                    r[c] = repr(float(fixed[c][i]))
+        with open(args.out, "w", newline="", encoding="utf-8") as fh:
+            w = csv.DictWriter(fh, fieldnames=fields)
+            w.writeheader()
+            w.writerows(rows)
+        print(f"wrote repaired data to {args.out}")
+    return 1 if fit.detected else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

unitsleuth/detect.py

@@ -0,0 +1,236 @@
+"""Univariate detection of mixed units in a single numeric column.
+
+Model (after Di Zio, Guarnera & Luzi, 2005): true values follow a normal
+distribution on a working scale (log for multiplicative conversions of positive
+data, raw otherwise). A fraction ``weight`` of rows was recorded in a different
+unit, i.e. passed through a known conversion. The mixture is fitted by EM and
+compared with the single-unit model through BIC.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+import numpy as np
+from numpy.typing import ArrayLike, NDArray
+
+from unitsleuth import catalog
+from unitsleuth.catalog import Conversion
+
+FloatArray = NDArray[np.float64]
+BoolArray = NDArray[np.bool_]
+Direction = Literal["forward", "inverse"]
+Scale = Literal["log", "linear"]
+
+_LOG_2PI = float(np.log(2.0 * np.pi))
+
+
+@dataclass(frozen=True)
+class UnivariateFit:
+    """Result of testing one conversion against one column."""
+
+    conversion: Conversion
+    direction: Direction
+    scale: Scale
+    weight: float
+    mu: float
+    sigma: float
+    loglik: float
+    loglik_null: float
+    delta_bic: float
+    detected: bool
+    posterior: FloatArray
+
+    @property
+    def converted(self) -> BoolArray:
+        """Rows whose posterior probability of being converted exceeds 0.5."""
+        with np.errstate(invalid="ignore"):
+            return np.asarray(self.posterior > 0.5, dtype=np.bool_)
+
+    @property
+    def label(self) -> str:
+        """Human-readable description of the hypothesis, e.g. ``'cm->in'``."""
+        if self.direction == "forward":
+            return self.conversion.name
+        return f"{self.conversion.to_unit}->{self.conversion.from_unit}"
+
+
+def _normal_logpdf(
+    t: FloatArray, mu: FloatArray | float, sigma: FloatArray | float
+) -> FloatArray:
+    z = (t - mu) / sigma
+    return np.asarray(-0.5 * z * z - np.log(sigma) - 0.5 * _LOG_2PI, dtype=np.float64)
+
+
+def _fit_mixture(
+    t: FloatArray, shift: float, mult: float, jac: float, max_iter: int, tol: float
+) -> tuple[float, float, float, float]:
+    """EM for tied-parameter two-component model.
+
+    Converted rows are pulled back with ``y = (t - shift) / mult`` and then share
+    ``mu`` and ``sigma`` with the base rows. ``jac`` is the log-density Jacobian
+    of the converted component. Returns ``(weight, mu, sigma, loglik)``.
+    """
+    n = t.size
+    y = (t - shift) / mult
+    sd_all = float(np.std(t))
+    floor = 1e-3 * sd_all if sd_all > 0 else 1e-9
+    med = float(np.median(t))
+    mad = 1.4826 * float(np.median(np.abs(t - med)))
+    sig0 = max(mad / 2.0, floor)
+    best: tuple[float, float, float, float] | None = None
+    for mu0 in np.quantile(t, [0.25, 0.5, 0.75]):
+        for w0 in (0.05, 0.2, 0.4):
+            mu, sigma, w = float(mu0), sig0, w0
+            prev = -np.inf
+            ll = -np.inf
+            for _ in range(max_iter):
+                a = np.log1p(-w) + _normal_logpdf(t, mu, sigma)
+                b = np.log(w) + _normal_logpdf(y, mu, sigma) + jac
+                m = np.maximum(a, b)
+                lse = m + np.log(np.exp(a - m) + np.exp(b - m))
+                ll = float(lse.sum())
+                r = np.exp(b - lse)
+                w = float(np.clip(r.mean(), 1e-6, 1 - 1e-6))
+                mu = float(np.sum((1 - r) * t + r * y) / n)
+                var = float(np.sum((1 - r) * (t - mu) ** 2 + r * (y - mu) ** 2) / n)
+                sigma = max(np.sqrt(var), floor)
+                if abs(ll - prev) < tol * (1.0 + abs(ll)):
+                    break
+                prev = ll
+            if best is None or ll > best[3]:
+                best = (w, mu, sigma, ll)
+    assert best is not None
+    return best
+
+
+def _params(conv: Conversion, direction: Direction) -> tuple[float, float]:
+    if direction == "forward":
+        return conv.scale, conv.offset
+    return 1.0 / conv.scale, -conv.offset / conv.scale
+
+
+def fit_conversion(
+    x: ArrayLike,
+    conversion: Conversion,
+    *,
+    direction: Direction | None = None,
+    scale: Scale | None = None,
+    min_delta_bic: float = 10.0,
+    min_weight: float = 0.005,
+    min_n: int = 10,
+    max_iter: int = 500,
+    tol: float = 1e-10,
+) -> UnivariateFit:
+    """Test whether a share of ``x`` was recorded in a different unit.
+
+    ``direction=None`` fits both ``from->to`` and ``to->from`` and keeps the better
+    one. ``scale=None`` uses the log scale for multiplicative conversions of
+    strictly positive data and the raw scale otherwise.
+    """
+    arr = np.asarray(x, dtype=np.float64).ravel()
+    ok = np.isfinite(arr)
+    xs = arr[ok]
+    if xs.size < min_n:
+        raise ValueError(f"need at least {min_n} finite values, got {xs.size}")
+    if direction is None:
+        fits = [
+            fit_conversion(
+                arr,
+                conversion,
+                direction=d,
+                scale=scale,
+                min_delta_bic=min_delta_bic,
+                min_weight=min_weight,
+                min_n=min_n,
+                max_iter=max_iter,
+                tol=tol,
+            )
+            for d in ("forward", "inverse")
+        ]
+        return max(fits, key=lambda f: f.loglik)
+
+    a, c = _params(conversion, direction)
+    can_log = c == 0.0 and a > 0.0 and bool(np.all(xs > 0))
+    if scale is None:
+        scale = "log" if can_log else "linear"
+    if scale == "log" and not can_log:
+        raise ValueError(
+            "log scale needs a positive multiplicative conversion and x > 0"
+        )
+
+    if scale == "log":
+        t = np.log(xs)
+        shift, mult, jac = float(np.log(a)), 1.0, 0.0
+    else:
+        t = xs
+        shift, mult, jac = c, a, -float(np.log(abs(a)))
+    if float(np.std(t)) == 0.0:
+        post = np.full(arr.shape, np.nan)
+        post[ok] = 0.0
+        return UnivariateFit(
+            conversion,
+            direction,
+            scale,
+            0.0,
+            float(t[0]),
+            0.0,
+            0.0,
+            0.0,
+            0.0,
+            False,
+            post,
+        )
+
+    w, mu, sigma, ll = _fit_mixture(t, shift, mult, jac, max_iter, tol)
+
+    mu0 = float(np.mean(t))
+    sig0 = float(np.std(t))
+    ll0 = float(_normal_logpdf(t, mu0, sig0).sum())
+    n = t.size
+    delta_bic = (-2.0 * ll0 + 2 * np.log(n)) - (-2.0 * ll + 3 * np.log(n))
+
+    la = np.log1p(-w) + _normal_logpdf(t, mu, sigma)
+    lb = np.log(w) + _normal_logpdf((t - shift) / mult, mu, sigma) + jac
+    r = 1.0 / (1.0 + np.exp(np.clip(la - lb, -700, 700)))
+    post = np.full(arr.shape, np.nan)
+    post[ok] = r
+    return UnivariateFit(
+        conversion=conversion,
+        direction=direction,
+        scale=scale,
+        weight=w,
+        mu=mu,
+        sigma=sigma,
+        loglik=ll,
+        loglik_null=ll0,
+        delta_bic=float(delta_bic),
+        detected=bool(delta_bic >= min_delta_bic and w >= min_weight),
+        posterior=post,
+    )
+
+
+def detect(
+    x: ArrayLike,
+    conversions: list[Conversion] | tuple[Conversion, ...] | None = None,
+    *,
+    quantity: str | None = None,
+    **kwargs: float | int,
+) -> list[UnivariateFit]:
+    """Test several conversions and return fits sorted by evidence (best first).
+
+    Pass ``conversions`` explicitly, or a catalog ``quantity`` such as ``"mass"``;
+    by default every catalog conversion is tried. Extra keyword arguments go to
+    :func:`fit_conversion`.
+    """
+    cands = (
+        tuple(conversions) if conversions is not None else catalog.conversions(quantity)
+    )
+    out: list[UnivariateFit] = []
+    for conv in cands:
+        try:
+            out.append(fit_conversion(x, conv, **kwargs))  # type: ignore[arg-type]
+        except ValueError:
+            continue
+    return sorted(out, key=lambda f: f.delta_bic, reverse=True)

unitsleuth/multivariate.py

@@ -0,0 +1,247 @@
+"""Row-consistent detection across several columns, plus repair.
+
+Each row carries one latent flag: either every column is in its base unit, or
+every column was recorded in the converted unit (for example a whole record
+entered in imperial units). The flag is shared by all observed columns of the
+row, so a row with a clear height gives evidence about its ambiguous weight.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any, Literal
+
+import numpy as np
+from numpy.typing import ArrayLike, NDArray
+
+from unitsleuth._frame import columns_of, is_frame, replace_columns
+from unitsleuth.catalog import Conversion
+from unitsleuth.detect import Scale, _normal_logpdf
+
+FloatArray = NDArray[np.float64]
+BoolArray = NDArray[np.bool_]
+Target = Literal["majority", "base", "converted"]
+
+
+@dataclass(frozen=True)
+class RowFit:
+    """Joint fit of one shared row flag over several columns."""
+
+    columns: tuple[str, ...]
+    conversions: dict[str, Conversion]
+    scales: dict[str, Scale]
+    weight: float
+    mu: dict[str, float]
+    sigma: dict[str, float]
+    loglik: float
+    loglik_null: float
+    delta_bic: float
+    detected: bool
+    posterior: FloatArray
+
+    @property
+    def converted(self) -> BoolArray:
+        """Rows whose posterior probability of being converted exceeds 0.5."""
+        with np.errstate(invalid="ignore"):
+            return np.asarray(self.posterior > 0.5, dtype=np.bool_)
+
+    def summary(self) -> str:
+        """Plain-text report of the fit."""
+        n_conv = int(self.converted.sum())
+        n_obs = int(np.isfinite(self.posterior).sum())
+        lines = [
+            f"unitsleuth row-consistent fit over {len(self.columns)} column(s)",
+            f"  rows analysed: {n_obs}",
+            f"  rows flagged as converted: {n_conv} "
+            f"({self.weight:.1%} estimated share)",
+            f"  evidence (delta BIC vs no mixing): {self.delta_bic:.1f} "
+            f"-> {'mixed units detected' if self.detected else 'no mixing detected'}",
+        ]
+        for col in self.columns:
+            c = self.conversions[col]
+            lines.append(
+                f"  {col}: {c.name} on {self.scales[col]} scale, "
+                f"mu={self.mu[col]:.4g}, sigma={self.sigma[col]:.4g}"
+            )
+        return "\n".join(lines)
+
+
+def _to_arrays(data: Mapping[str, ArrayLike]) -> dict[str, FloatArray]:
+    out: dict[str, FloatArray] = {
+        k: np.asarray(v, dtype=np.float64).ravel() for k, v in data.items()
+    }
+    lengths = {a.size for a in out.values()}
+    if len(lengths) > 1:
+        raise ValueError("all columns must have the same length")
+    return out
+
+
+def detect_rows(
+    data: Any,
+    conversions: Mapping[str, Conversion],
+    *,
+    min_delta_bic: float = 10.0,
+    min_weight: float = 0.005,
+    min_n: int = 10,
+    max_iter: int = 500,
+    tol: float = 1e-10,
+) -> RowFit:
+    """Fit a shared row flag over the columns named in ``conversions``.
+
+    A flagged row is one where every observed column is ``conversion.forward`` of
+    its base-unit value. The share of flagged rows is free, so the flagged unit may
+    be the majority.
+    """
+    cols = tuple(conversions)
+    if not cols:
+        raise ValueError("conversions is empty")
+    missing = [c for c in cols if isinstance(data, Mapping) and c not in data]
+    if missing:
+        raise KeyError(f"columns not in data: {missing}")
+    arrays = _to_arrays(columns_of(data, cols))
+    raw = np.column_stack([arrays[c] for c in cols])
+    mask = np.isfinite(raw)
+    row_obs = mask.any(axis=1)
+    if int(row_obs.sum()) < min_n:
+        raise ValueError(f"need at least {min_n} rows with a finite value")
+
+    t_cols: list[FloatArray] = []
+    y_cols: list[FloatArray] = []
+    jac = np.zeros(len(cols))
+    scales: dict[str, Scale] = {}
+    for j, c in enumerate(cols):
+        conv = conversions[c]
+        x = raw[:, j]
+        obs = mask[:, j]
+        positive = bool(np.all(x[obs] > 0)) if obs.any() else False
+        if conv.is_multiplicative and conv.scale > 0 and positive:
+            scales[c] = "log"
+            t = np.where(obs, np.log(np.where(obs, x, 1.0)), 0.0)
+            y = t - np.log(conv.scale)
+        else:
+            scales[c] = "linear"
+            t = np.where(obs, x, 0.0)
+            y = (t - conv.offset) / conv.scale
+            jac[j] = -np.log(abs(conv.scale))
+        if obs.sum() < 2 or float(np.std(t[obs])) == 0.0:
+            raise ValueError(f"column {c!r} is constant or has too few values")
+        t_cols.append(t)
+        y_cols.append(y)
+    t_mat = np.column_stack(t_cols)
+    y_mat = np.column_stack(y_cols)
+    m = mask.astype(np.float64)
+
+    n_obs_col = m.sum(axis=0)
+    sd = np.array([np.std(t_mat[mask[:, j], j]) for j in range(len(cols))])
+    floor = 1e-3 * sd
+    med = np.array([np.median(t_mat[mask[:, j], j]) for j in range(len(cols))])
+    mad = np.array(
+        [
+            1.4826 * np.median(np.abs(t_mat[mask[:, j], j] - med[j]))
+            for j in range(len(cols))
+        ]
+    )
+    sig_init = np.maximum(mad / 2.0, floor)
+
+    def logpost(
+        w: float, mu: FloatArray, sig: FloatArray
+    ) -> tuple[FloatArray, FloatArray]:
+        a = np.log1p(-w) + (_normal_logpdf(t_mat, mu, sig) * m).sum(axis=1)
+        b = np.log(w) + ((_normal_logpdf(y_mat, mu, sig) + jac) * m).sum(axis=1)
+        return a, b
+
+    best: tuple[float, FloatArray, FloatArray, float] | None = None
+    for q in (0.1, 0.25, 0.5, 0.75, 0.9):
+        mu_start = np.array(
+            [np.quantile(t_mat[mask[:, j], j], q) for j in range(len(cols))]
+        )
+        for w0 in (0.05, 0.2, 0.5, 0.8):
+            mu, sig, w = mu_start.copy(), sig_init.copy(), w0
+            prev, ll = -np.inf, -np.inf
+            for _ in range(max_iter):
+                a, b = logpost(w, mu, sig)
+                mx = np.maximum(a, b)
+                lse = mx + np.log(np.exp(a - mx) + np.exp(b - mx))
+                ll = float(lse[row_obs].sum())
+                r = np.exp(b - lse)
+                w = float(np.clip(r[row_obs].mean(), 1e-6, 1 - 1e-6))
+                rc = r[:, None]
+                mu = np.sum(m * ((1 - rc) * t_mat + rc * y_mat), axis=0) / n_obs_col
+                var = (
+                    np.sum(
+                        m * ((1 - rc) * (t_mat - mu) ** 2 + rc * (y_mat - mu) ** 2),
+                        axis=0,
+                    )
+                    / n_obs_col
+                )
+                sig = np.maximum(np.sqrt(var), floor)
+                if abs(ll - prev) < tol * (1.0 + abs(ll)):
+                    break
+                prev = ll
+            if best is None or ll > best[3]:
+                best = (w, mu, sig, ll)
+    assert best is not None
+    w, mu, sig, ll = best
+
+    ll0 = 0.0
+    for j in range(len(cols)):
+        tj = t_mat[mask[:, j], j]
+        ll0 += float(_normal_logpdf(tj, float(tj.mean()), float(tj.std())).sum())
+    n = int(row_obs.sum())
+    j_cols = len(cols)
+    delta_bic = (-2.0 * ll0 + 2 * j_cols * np.log(n)) - (
+        -2.0 * ll + (2 * j_cols + 1) * np.log(n)
+    )
+
+    a, b = logpost(w, mu, sig)
+    post = 1.0 / (1.0 + np.exp(np.clip(a - b, -700, 700)))
+    post[~row_obs] = np.nan
+    return RowFit(
+        columns=cols,
+        conversions=dict(conversions),
+        scales=scales,
+        weight=w,
+        mu={c: float(mu[j]) for j, c in enumerate(cols)},
+        sigma={c: float(sig[j]) for j, c in enumerate(cols)},
+        loglik=ll,
+        loglik_null=ll0,
+        delta_bic=float(delta_bic),
+        detected=bool(delta_bic >= min_delta_bic and w >= min_weight),
+        posterior=post,
+    )
+
+
+def repair(
+    data: Any,
+    fit: RowFit,
+    *,
+    threshold: float = 0.5,
+    target: Target = "majority",
+) -> Any:
+    """Return copies of the fitted columns harmonised to a single unit.
+
+    ``target="base"`` converts flagged rows back to the base unit,
+    ``"converted"`` converts unflagged rows forward, and ``"majority"`` (default)
+    picks whichever unit most rows are already in. Rows with an undefined
+    posterior are left untouched. For a dataframe input the result is a
+    dataframe of the same type with the fitted columns replaced; for a mapping
+    it is a dict of arrays.
+    """
+    arrays = _to_arrays(columns_of(data, fit.columns))
+    post = fit.posterior
+    with np.errstate(invalid="ignore"):
+        flagged = post > threshold
+        unflagged = post <= threshold
+    if target == "majority":
+        target = "base" if int(unflagged.sum()) >= int(flagged.sum()) else "converted"
+    out: dict[str, FloatArray] = {}
+    for c in fit.columns:
+        conv = fit.conversions[c]
+        x = arrays[c].copy()
+        if target == "base":
+            x[flagged] = conv.inverse(x[flagged])
+        else:
+            x[unflagged] = conv.forward(x[unflagged])
+        out[c] = x
+    return replace_columns(data, out) if is_frame(data) else out

unitsleuth-0.1.0.dist-info/METADATA

@@ -0,0 +1,88 @@
+Metadata-Version: 2.5
+Name: unitsleuth
+Version: 0.1.0
+Summary: Detect mixed measurement units in unlabeled numeric columns from the numbers alone.
+Project-URL: Homepage, https://github.com/omarfaruqe/unitsleuth
+Project-URL: Issues, https://github.com/omarfaruqe/unitsleuth/issues
+Author-email: Md Omar Faruqe <faruqe@ru.ac.bd>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: data-cleaning,data-quality,measurement,mixture-model,units
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: narwhals>=1.20
+Requires-Dist: numpy>=1.24
+Description-Content-Type: text/markdown
+
+# unitsleuth
+
+Detect mixed measurement units in an unlabeled numeric column, from the numbers alone.
+
+A height column where most rows are centimetres and a few are inches, or a
+temperature column that mixes °C and °F, looks fine to a schema check. unitsleuth
+fits a mixture model per known unit conversion and tells you which rows were
+probably recorded in the other unit.
+
+This is a Python implementation of the mixture-model approach of Di Zio, Guarnera
+and Luzi (2005, *Survey Methodology*), extended with affine conversions
+(e.g. °C to °F) and a catalog of common unit pairs. It is not a new method.
+
+## Quick start
+
+```python
+import numpy as np
+import unitsleuth as us
+
+heights = np.concatenate([np.random.normal(170, 9, 900), np.random.normal(67, 3.5, 100)])
+best = us.detect(heights, quantity="length")[0]
+print(best.label, best.weight, best.detected)
+rows = best.converted  # boolean mask of rows likely in the other unit
+```
+
+### Whole rows in the wrong unit
+
+When a record was entered in one unit system, every column in that row is
+affected. `detect_rows` shares one flag per row across columns, and `repair`
+harmonises the data. `df` can be a pandas or Polars DataFrame (via
+[Narwhals](https://narwhals-dev.github.io/narwhals/)) or a dict of arrays:
+
+```python
+cat = us.catalog
+fit = us.detect_rows(df, {"height": cat.get("cm", "in"), "weight": cat.get("kg", "lb")})
+print(fit.summary())
+fixed = us.repair(df, fit)  # same dataframe type, columns corrected
+```
+
+### Command line
+
+```
+unitsleuth data.csv --col height:cm,in --col weight:kg,lb --out data_fixed.csv
+```
+
+Prints the report and, with `--out`, writes a harmonised CSV. The exit code is 1
+when mixed units are detected, 0 when not and 2 on usage errors, so it can gate a
+data pipeline.
+
+## Status
+
+Alpha. Univariate and row-consistent multivariate detection and repair work;
+dataframe integration (pandas, Polars) and a CLI are planned. Detection needs the units to be
+separated relative to the spread of the data, so it cannot distinguish them when
+natural variation is wide compared with the conversion factor.
+
+## Development
+
+```
+uv sync
+uv run pytest
+```
+
+## Validation
+
+- **Negative control (real data):** on the Palmer penguins dataset (342 birds, 4 numeric columns, 24 catalog conversions each), nothing was flagged: 0 of 96 tests, despite three species with different body sizes.
+- **Positive control (semi-real):** converting a known 5%, 15% or 30% of the penguins' body mass (kg to lb) and bill length (cm to in) and running `detect_rows` over 3 random seeds each, the injected rows were found with 100% row accuracy in all 9 runs, and the estimated share was close to the injected share.
+- **Limits:** these results are for well-separated unit pairs. When the natural spread of a column is wide relative to the conversion factor, the units cannot be told apart from the numbers alone, and the evidence score will say so.

unitsleuth-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+unitsleuth/__init__.py,sha256=R1_4M93tZAMMpx9Fb8tGV9ybpA3_eQmOIO4aqsTFxZk,450
+unitsleuth/__main__.py,sha256=Ug-gHuKo4HYuqaiIMLyzTPbSgaLmc3uwDd6b-uBwkSI,58
+unitsleuth/_frame.py,sha256=c3pwI0preMfDNnfNtCvSU_VhcXsiePigj11IMoji9JE,1665
+unitsleuth/catalog.py,sha256=gYmU-8biQ--mxzfyHV7dIyfphuswMmgPHf4xwLDFBRA,3961
+unitsleuth/cli.py,sha256=HFqTViBDZCdgMlZNAXl0Hho2Wnm56G0F08ATnB9x0ns,3301
+unitsleuth/detect.py,sha256=uE1IH1CQgY2mYxaWjgzyP9HW1BAIbT1XJVaLIBAqn-U,7714
+unitsleuth/multivariate.py,sha256=WnviqhaRg7oXS4npAFbdPkR60q4fJFXC7iN3ZySnzYY,8899
+unitsleuth/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+unitsleuth-0.1.0.dist-info/METADATA,sha256=Ecm7Pqkl-fLcoydpuLurA9Jh2AOeiQ9WLRIMcMThCVE,3711
+unitsleuth-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+unitsleuth-0.1.0.dist-info/entry_points.txt,sha256=bWfhSgA0W__JOH0lpBcMK1-B0knTg9By_FkGpK9XvN4,51
+unitsleuth-0.1.0.dist-info/licenses/LICENSE,sha256=4qh7wJiUhE3j5jFfz3qME9Fnlv-OOuTS5sHTLw_mpUw,1071
+unitsleuth-0.1.0.dist-info/RECORD,,

unitsleuth-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

unitsleuth-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+unitsleuth = unitsleuth.cli:main

unitsleuth-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Md Omar Faruqe
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.