bayesian-pv-census 0.1.0__py3-none-any.whl

bayesian_pv_census/__init__.py

@@ -0,0 +1,50 @@
+"""bayesian_pv_census -- turn a detector's output into a census, then audit a register against it.
+
+A detector's raw count is not a measurement: it misses objects and invents
+others, at rates that vary from place to place. Given a validation sample,
+this package turns that raw count into a posterior over the true quantity, and
+turns the posterior into a verdict on whatever register claims to record it.
+
+Nothing here knows about photovoltaics, France, or geometry. A "unit" is
+anything with a raw total and a validation sample, and ``raw`` is whatever
+quantity you chose to count -- installed capacity, number of installations,
+roof area. The package ships the paper's French data as a demo; see
+:func:`load_demo`.
+
+Reference implementation of the statistical core of:
+    Kasmi, G. et al. (2026). Joule.
+"""
+
+from .datatypes import JEFFREYS, AuditFlag, BetaPrior, CorrectionResult, UnitRecord
+from .correction import correct_batch, correct_unit, fit_empirical_bayes_prior
+from .audit import (
+    BatteryResult,
+    compare_reference_variants,
+    conversion_threshold,
+    evaluate,
+    evaluate_batch,
+    run_battery,
+)
+from .budgeting import Budget, half_width_closed_form, required_sample_size
+from .aggregation import compare_aggregation_scales, pooling_penalty
+from .io import (
+    has_reported_values,
+    load_demo,
+    read_units_csv,
+    units_from_dataframe,
+    units_to_dataframe,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "UnitRecord", "BetaPrior", "CorrectionResult", "AuditFlag", "JEFFREYS",
+    "correct_unit", "correct_batch", "fit_empirical_bayes_prior",
+    "evaluate", "evaluate_batch", "run_battery", "BatteryResult",
+    "conversion_threshold", "compare_reference_variants",
+    "required_sample_size", "half_width_closed_form", "Budget",
+    "compare_aggregation_scales", "pooling_penalty",
+    "read_units_csv", "units_from_dataframe", "units_to_dataframe",
+    "load_demo", "has_reported_values",
+    "__version__",
+]

bayesian_pv_census/aggregation.py

@@ -0,0 +1,168 @@
+"""What forming the correction factor at a coarser scale costs.
+
+Correcting each unit and summing is not the same as pooling the units' rates
+and correcting once. The gap is exact and signed, and it has a closed second-
+order form -- see :func:`pooling_penalty`.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import numpy as np
+import pandas as pd
+
+from .correction import (
+    DEFAULT_N_DRAWS,
+    DEFAULT_SEED,
+    _seed_for,
+    _resolve_priors,
+    fit_empirical_bayes_prior,
+)
+from .datatypes import UnitRecord
+
+__all__ = ["pooled_factor_draws", "compare_aggregation_scales", "pooling_penalty",
+           "AggregationComparison"]
+
+
+@dataclass
+class AggregationComparison:
+    totals: pd.DataFrame
+    draws: dict[str, np.ndarray] = field(repr=False, default_factory=dict)
+
+
+def pooled_factor_draws(
+    units: list[UnitRecord],
+    weights: np.ndarray,
+    prior,
+    n_draws: int = DEFAULT_N_DRAWS,
+    seed: int = DEFAULT_SEED,
+    key: str = "pooled",
+) -> np.ndarray:
+    """One correction factor for a whole group: mean of drawn P over mean of drawn R.
+
+    Weights are the units' raw quantities, normalised. Weighting by annotation
+    sample size instead would be a different and much worse estimator: how many
+    points were annotated in a unit has nothing to do with that unit's share of
+    the quantity being corrected.
+    """
+    prior_p, prior_r = _resolve_priors(prior, None)
+    w = np.asarray(weights, dtype=float)
+    w = w / w.sum()
+
+    a_p = np.array([prior_p.alpha + u.precision_tp for u in units])
+    b_p = np.array([prior_p.beta + u.precision_fp for u in units])
+    a_r = np.array([prior_r.alpha + u.recall_tp for u in units])
+    b_r = np.array([prior_r.beta + u.recall_fn for u in units])
+
+    rng = np.random.default_rng(_seed_for(key, seed))
+    p = rng.beta(a_p, b_p, size=(n_draws, len(units))) @ w
+    r = rng.beta(a_r, b_r, size=(n_draws, len(units))) @ w
+    return np.divide(p, r, out=np.full(n_draws, np.nan), where=r > 0)
+
+
+def pooling_penalty(units: list[UnitRecord], weights: np.ndarray | None = None) -> dict:
+    """Second-order estimate of the relative cost of pooling a group.
+
+        (F_pooled / f_unit) - 1  ~=  -[ CV(R)^2 - rho * CV(P) * CV(R) ]
+
+    where the coefficients of variation and the correlation are taken across the
+    units being merged, weighted by their raw quantities. They describe the
+    heterogeneity of the population, not the sampling uncertainty of the rates.
+
+    The first term is the cost of recall heterogeneity and is always a penalty,
+    because 1/R is convex: pooling always yields a smaller factor and therefore
+    a smaller corrected total. The second offsets it when the units that are
+    well detected are well detected on both metrics. If every unit shares the
+    same recall, the penalty is exactly zero however dispersed precision is.
+    """
+    p = np.array(
+        [u.precision_tp / (u.precision_tp + u.precision_fp) for u in units]
+    )
+    r = np.array([u.recall_tp / (u.recall_tp + u.recall_fn) for u in units])
+    w = np.asarray(
+        [u.raw for u in units] if weights is None else weights, dtype=float
+    )
+    w = w / w.sum()
+
+    mp, mr = float(w @ p), float(w @ r)
+    sp = float(np.sqrt(w @ (p - mp) ** 2))
+    sr = float(np.sqrt(w @ (r - mr) ** 2))
+    cov = float(w @ ((p - mp) * (r - mr)))
+    rho = cov / (sp * sr) if sp > 0 and sr > 0 else 0.0
+    cv_p, cv_r = sp / mp, sr / mr
+
+    return {
+        "cv_precision": cv_p,
+        "cv_recall": cv_r,
+        "rho": rho,
+        "penalty": -(cv_r**2 - rho * cv_p * cv_r),
+        "penalty_recall_only": -(cv_r**2),
+    }
+
+
+def compare_aggregation_scales(
+    units: list[UnitRecord],
+    groups: dict[str, list[str]] | None = None,
+    *,
+    prior="jeffreys",
+    n_draws: int = DEFAULT_N_DRAWS,
+    seed: int = DEFAULT_SEED,
+) -> AggregationComparison:
+    """Total under three ways of forming the correction factor.
+
+    Unit level (one factor per unit, then sum) is the reference and, by
+    linearity of expectation, reproduces the sum of per-unit means exactly.
+    Group level pools within each group. Full pool uses a single factor.
+
+    ``groups`` maps a group label to a list of unit ids. Omit it to compare only
+    unit level against the full pool.
+    """
+    from .correction import correct_unit
+
+    if prior == "empirical_bayes":
+        pp, pr, _ = fit_empirical_bayes_prior(units)
+        prior = (pp, pr)
+
+    by_id = {u.unit_id: u for u in units}
+    draws: dict[str, np.ndarray] = {}
+
+    unit_total = np.zeros(n_draws)
+    for u in units:
+        unit_total += correct_unit(
+            u, prior=prior, n_draws=n_draws, seed=seed
+        ).draws
+    draws["unit"] = unit_total
+
+    if groups:
+        group_total = np.zeros(n_draws)
+        for gname, ids in groups.items():
+            members = [by_id[i] for i in ids if i in by_id]
+            if not members:
+                continue
+            w = np.array([m.raw for m in members])
+            factor = pooled_factor_draws(
+                members, w, prior, n_draws=n_draws, seed=seed, key=gname
+            )
+            group_total += w.sum() * factor
+        draws["group"] = group_total
+
+    w_all = np.array([u.raw for u in units])
+    draws["pooled"] = w_all.sum() * pooled_factor_draws(
+        units, w_all, prior, n_draws=n_draws, seed=seed, key="__all__"
+    )
+
+    ref = float(np.nanmean(draws["unit"]))
+    rows = []
+    for level, d in draws.items():
+        lo, hi = np.nanquantile(d, [0.025, 0.975])
+        rows.append(
+            {
+                "level": level,
+                "mean": float(np.nanmean(d)),
+                "ci95_low": float(lo),
+                "ci95_high": float(hi),
+                "gap_vs_unit": float(np.nanmean(d)) / ref - 1,
+            }
+        )
+    return AggregationComparison(totals=pd.DataFrame(rows), draws=draws)

bayesian_pv_census/audit.py

@@ -0,0 +1,230 @@
+"""Compare a reference value against the corrected posterior, and across specifications."""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+from .correction import DEFAULT_MIN_WEIGHTED_N, DEFAULT_N_DRAWS, DEFAULT_SEED, correct_batch
+from .datatypes import AuditFlag, CorrectionResult, UnitRecord
+
+__all__ = ["evaluate", "evaluate_batch", "run_battery", "BatteryResult",
+           "conversion_threshold", "compare_reference_variants"]
+
+
+def evaluate(
+    result: CorrectionResult,
+    reported_value: float | None = None,
+    level: float = 0.99,
+) -> AuditFlag:
+    """Where does the reported value fall relative to the corrected posterior?
+
+    ``below`` means the reference is under the lower bound: capacity is missing
+    from it. ``above`` means the reference exceeds the upper bound. ``within``
+    means the two are compatible at this credible level.
+
+    The interval is the only thing that decides. A large gap inside a wide
+    interval is not a finding; a small gap outside a tight one is.
+    """
+    if reported_value is None:
+        raise ValueError(f"{result.unit_id}: no registry value to evaluate against")
+    lo, hi = result.ci(level)
+    if reported_value < lo:
+        status = "below"
+    elif reported_value > hi:
+        status = "above"
+    else:
+        status = "within"
+    return AuditFlag(
+        unit_id=result.unit_id,
+        status=status,
+        reported_value=float(reported_value),
+        corrected_mean=result.mean,
+        ci=(lo, hi),
+        level=level,
+    )
+
+
+def evaluate_batch(
+    results: list[CorrectionResult],
+    units: list[UnitRecord],
+    level: float = 0.99,
+) -> pd.DataFrame:
+    """Evaluate every unit that carries a reference value."""
+    rows = []
+    for res, u in zip(results, units):
+        if u.reported_value is None:
+            continue
+        f = evaluate(res, u.reported_value, level=level)
+        rows.append(
+            {
+                "unit_id": f.unit_id,
+                "status": f.status,
+                "reported_value": f.reported_value,
+                "corrected_mean": f.corrected_mean,
+                "ci_low": f.ci[0],
+                "ci_high": f.ci[1],
+                "gap": f.gap,
+                "gap_pct": f.gap_pct,
+            }
+        )
+    return pd.DataFrame(rows)
+
+
+def conversion_threshold(
+    result: CorrectionResult,
+    reported_value: float,
+    direction: str,
+    level: float = 0.99,
+    retained_scale: float = 1.0,
+) -> float:
+    """Scale factor at which this unit's flag would flip, in closed form.
+
+    Rescaling every detected quantity by a constant (a different surface-to-power
+    coefficient, a different filtering threshold) multiplies the posterior and
+    both its bounds, while the reference value stays put. The status is therefore
+    monotone in that constant and flips at a single point, with no numerical
+    sweep needed.
+
+    The two directions are mirror images. A unit flagged ``below`` loses its flag
+    when the scale *falls* to ``retained_scale * registry / CI_low``; a unit
+    flagged ``above`` loses it when the scale *rises* to
+    ``retained_scale * registry / CI_high``.
+    """
+    lo, hi = result.ci(level)
+    if direction == "below":
+        return float(retained_scale * reported_value / lo)
+    if direction == "above":
+        return float(retained_scale * reported_value / hi)
+    raise ValueError("direction must be 'below' or 'above'")
+
+
+class BatteryResult:
+    """Flags for every unit under every specification."""
+
+    def __init__(self, flags: pd.DataFrame, tables: dict[str, pd.DataFrame]):
+        self.flags = flags          # unit_id x spec -> status
+        self.tables = tables        # spec -> full correction table
+
+    def concordance_table(self, spec_a: str = "A", spec_b: str = "B") -> pd.DataFrame:
+        """Crosstab of two specifications' verdicts.
+
+        The diagonal cells define the hard core. The two off-diagonal corners
+        (below/above and above/below) are the substantive check: if they are
+        empty, no unit changes sign between specifications, and the disagreement
+        is only ever about the boundary with ``within``.
+        """
+        order = ["below", "within", "above"]
+        out = pd.crosstab(self.flags[spec_a], self.flags[spec_b])
+        return out.reindex(index=order, columns=order, fill_value=0)
+
+    def hard_core(self, status: str = "below") -> list[str]:
+        """Units flagged ``status`` by every specification in the battery."""
+        cols = [c for c in self.flags.columns if c != "unit_id"]
+        mask = np.logical_and.reduce([self.flags[c] == status for c in cols])
+        return self.flags.loc[mask, "unit_id"].tolist()
+
+    def negative_control(self) -> list[str]:
+        """Units unanimously flagged in the opposite direction.
+
+        A method that manufactured discrepancies would produce these in numbers
+        comparable to the hard core. They are the control group.
+        """
+        return self.hard_core(status="above")
+
+    def summary(self) -> pd.DataFrame:
+        cols = [c for c in self.flags.columns if c != "unit_id"]
+        return pd.DataFrame(
+            {c: self.flags[c].value_counts() for c in cols}
+        ).reindex(["below", "within", "above"]).fillna(0).astype(int)
+
+
+def run_battery(
+    units: list[UnitRecord],
+    specs: dict[str, dict] | None = None,
+    *,
+    prior="jeffreys",
+    level: float = 0.99,
+    min_weighted_n: int | None = DEFAULT_MIN_WEIGHTED_N,
+    n_draws: int = DEFAULT_N_DRAWS,
+    seed: int = DEFAULT_SEED,
+) -> BatteryResult:
+    """Run the audit under several specifications and cross-tabulate the verdicts.
+
+    ``specs`` maps a label to a keyword dict. Two keys are understood:
+    ``weighted`` (bool, specification B) and ``scale`` (float, a multiplicative
+    rescaling of every raw quantity -- the paper's specification C, expressed
+    generically rather than as a surface-to-power coefficient).
+
+    The default battery is the paper's: A on counts, B capacity-weighted.
+    """
+    if specs is None:
+        specs = {"A": {"weighted": False}, "B": {"weighted": True}}
+
+    flags = pd.DataFrame({"unit_id": [u.unit_id for u in units]})
+    tables: dict[str, pd.DataFrame] = {}
+
+    for label, cfg in specs.items():
+        scale = cfg.get("scale", 1.0)
+        scaled = units
+        if scale != 1.0:
+            scaled = [
+                UnitRecord(**{**u.__dict__, "raw": u.raw * scale})
+                for u in units
+            ]
+        df, results = correct_batch(
+            scaled,
+            prior=prior,
+            weighted=cfg.get("weighted", False),
+            min_weighted_n=min_weighted_n,
+            n_draws=n_draws,
+            seed=seed,
+            return_results=True,
+        )
+        tables[label] = df
+        status = []
+        for res, u in zip(results, units):
+            status.append(
+                evaluate(res, u.reported_value, level=level).status
+                if u.reported_value is not None
+                else None
+            )
+        flags[label] = status
+
+    return BatteryResult(flags=flags, tables=tables)
+
+
+def compare_reference_variants(
+    results: list[CorrectionResult],
+    units: list[UnitRecord],
+    variant_a: dict[str, float],
+    variant_b: dict[str, float],
+    level: float = 0.99,
+    names: tuple[str, str] = ("a", "b"),
+) -> pd.DataFrame:
+    """Audit the same corrected estimate against two versions of a reference.
+
+    Useful whenever a register is published in more than one form -- truncated
+    and complete, provisional and consolidated -- to separate what the reference
+    does from what the audit does. The corrected estimate is held fixed, so any
+    change in verdict belongs to the reference.
+    """
+    na, nb = names
+    rows = []
+    for res, u in zip(results, units):
+        va, vb = variant_a.get(u.unit_id), variant_b.get(u.unit_id)
+        if va is None or vb is None:
+            continue
+        sa = evaluate(res, va, level=level).status
+        sb = evaluate(res, vb, level=level).status
+        rows.append(
+            {
+                "unit_id": u.unit_id,
+                f"value_{na}": va,
+                f"value_{nb}": vb,
+                f"status_{na}": sa,
+                f"status_{nb}": sb,
+                "changed": sa != sb,
+            }
+        )
+    return pd.DataFrame(rows)

bayesian_pv_census/budgeting.py

@@ -0,0 +1,93 @@
+"""How many annotations does a target precision buy, and vice versa.
+
+The delta method on ``log(P/R)`` gives a closed form for the credible interval's
+half-width, which inverts into a sample-size rule. Planning the annotation
+effort therefore needs no simulation, only a guess at the detector's precision
+and recall.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+
+from scipy.stats import norm
+
+__all__ = ["half_width_closed_form", "required_sample_size", "Budget"]
+
+
+@dataclass
+class Budget:
+    n_precision: int
+    n_recall: int
+    achieved_half_width: float
+
+    @property
+    def total(self) -> int:
+        return self.n_precision + self.n_recall
+
+
+def half_width_closed_form(
+    precision: float,
+    recall: float,
+    n_precision: float,
+    n_recall: float,
+    level: float = 0.99,
+) -> float:
+    """Relative half-width of the credible interval on the correction factor.
+
+        h = z * sqrt( (1-P)/(P*n_P) + (1-R)/(R*n_R) )
+
+    The two terms are the sampling variance of ``log P`` and of ``log R``. They
+    add because precision and recall are estimated on independent samples. The
+    expression is a first-order approximation; it agrees with the bootstrap to
+    a correlation above 0.999 over the range of rates seen in practice.
+    """
+    for name, v in (("precision", precision), ("recall", recall)):
+        if not 0 < v < 1:
+            raise ValueError(f"{name} must be strictly between 0 and 1")
+    z = norm.ppf(1 - (1 - level) / 2)
+    var = (1 - precision) / (precision * n_precision) + (1 - recall) / (recall * n_recall)
+    return float(z * math.sqrt(var))
+
+
+def required_sample_size(
+    target_half_width: float,
+    expected_precision: float,
+    expected_recall: float,
+    level: float = 0.99,
+    allocation: str = "equal_variance",
+) -> Budget:
+    """Annotations needed per unit to reach a target half-width.
+
+    ``allocation='equal_variance'`` splits the variance budget evenly between
+    the two terms, which is what the asymmetric floors in the paper approximate:
+    recall is the noisier of the two, so it gets the larger share of the budget
+    for a given target. ``allocation='equal_n'`` solves for a single sample size
+    used on both sides, which is simpler to explain but wastes annotations on
+    whichever rate is already well estimated.
+
+    The result is a floor, not a recommendation: it assumes the guessed rates
+    are right and ignores stratification.
+    """
+    if not 0 < target_half_width < 1:
+        raise ValueError("target_half_width must be a fraction strictly between 0 and 1")
+    z = norm.ppf(1 - (1 - level) / 2)
+    budget = (target_half_width / z) ** 2  # total allowed variance
+
+    vp = (1 - expected_precision) / expected_precision
+    vr = (1 - expected_recall) / expected_recall
+
+    if allocation == "equal_variance":
+        n_p = math.ceil(2 * vp / budget)
+        n_r = math.ceil(2 * vr / budget)
+    elif allocation == "equal_n":
+        n = math.ceil((vp + vr) / budget)
+        n_p = n_r = n
+    else:
+        raise ValueError("allocation must be 'equal_variance' or 'equal_n'")
+
+    achieved = half_width_closed_form(
+        expected_precision, expected_recall, n_p, n_r, level=level
+    )
+    return Budget(n_precision=n_p, n_recall=n_r, achieved_half_width=achieved)