setuhq 0.1.0__py3-none-any.whl

setuhq/__init__.py

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# SPDX-FileCopyrightText: 2026 Ashforde OÜ
+"""SetuHQ: an open-source transit search in the style of Planet Hunters CHEOPS.
+
+Volunteers are replaced by a reviewer panel. The panel can include simulated
+volunteers (deterministic detectors) and Sentinels (vision models, Claude by default, given
+the same task). Consensus is weighted by skill measured on injected transits.
+"""
+
+__version__ = "0.1.0"

setuhq/aggregate.py

@@ -0,0 +1,208 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# SPDX-FileCopyrightText: 2026 Ashforde OÜ
+"""Consensus: turn many independent classifications into ranked events.
+
+This follows the Planet Hunters idea. Marks from different reviewers that land
+on the same dip are clustered in time. Each cluster is scored by the share of
+the panel's skill-weighted reviewers who marked it. A cluster counts only if
+the subject is also labelled ``potential_planet`` by enough of the weighted panel.
+
+Clustering is single-linkage on intervals: marks are taken in order of start
+time, and a mark joins the current cluster when its interval overlaps the
+cluster's interval (the union of its marks so far) or its centre is within
+``gap_days`` of the previous mark's centre. A wide mark that covers a dip
+therefore counts even when its centre sits off the dip's centre.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass
+from statistics import median
+
+from setuhq.reviewers.base import POTENTIAL_PLANET, Classification
+
+
+@dataclass(frozen=True)
+class Consensus:
+    subject_id: str
+    t_start: float
+    t_end: float
+    score: float
+    n_reviewers: int
+    reviewers: tuple[str, ...]
+    planet_fraction: float
+    # The marks that counted, one per reviewer: (reviewer_id, t_start, t_end).
+    marks: tuple = ()
+
+    @property
+    def center(self) -> float:
+        return 0.5 * (self.t_start + self.t_end)
+
+
+def cluster_marks(entries, gap_days: float) -> list[list[tuple[str, float, float]]]:
+    """Group ``(reviewer_id, t_start, t_end)`` entries into time clusters. Input order does not matter.
+
+    A mark joins the current cluster when its interval overlaps the cluster's
+    interval, or when its centre is within ``gap_days`` of the previous mark's centre.
+    """
+    ordered = sorted(entries, key=lambda e: (e[1], e[2]))
+    clusters: list[list[tuple[str, float, float]]] = []
+    hi = last_center = None
+    for entry in ordered:
+        t_start, t_end = entry[1], entry[2]
+        center = 0.5 * (t_start + t_end)
+        joins = hi is not None and (t_start <= hi or abs(center - last_center) <= gap_days)
+        if joins:
+            clusters[-1].append(entry)
+            hi = max(hi, t_end)
+        else:
+            clusters.append([entry])
+            hi = t_end
+        last_center = center
+    return clusters
+
+
+def _one_mark_per_reviewer(cluster) -> dict[str, tuple[float, float]]:
+    """Each reviewer's mark in a cluster: the one whose centre is nearest the cluster's median centre."""
+    middle = median(0.5 * (a + b) for _, a, b in cluster)
+    chosen: dict[str, tuple[float, float]] = {}
+    for reviewer_id, t_start, t_end in cluster:
+        best = chosen.get(reviewer_id)
+        if best is None or abs(0.5 * (t_start + t_end) - middle) < abs(0.5 * sum(best) - middle):
+            chosen[reviewer_id] = (t_start, t_end)
+    return chosen
+
+
+def aggregate(
+    classifications: list[Classification],
+    weights: dict[str, float] | None = None,
+    *,
+    gap_days: float = 0.05,
+    min_score: float = 0.5,
+    min_planet_fraction: float = 0.5,
+) -> list[Consensus]:
+    """Return consensus events that pass both thresholds, grouped by subject and ordered by score."""
+    weights = weights or {}
+    by_subject: dict[str, list[Classification]] = defaultdict(list)
+    for classification in classifications:
+        by_subject[classification.subject_id].append(classification)
+
+    results: list[Consensus] = []
+    for subject_id in sorted(by_subject):
+        panel = _latest_per_reviewer(by_subject[subject_id])
+        total_weight = sum(_weight(weights, c.reviewer_id) for c in panel.values())
+        if total_weight <= 0:
+            continue
+        planet_weight = sum(_weight(weights, c.reviewer_id) for c in panel.values() if c.label == POTENTIAL_PLANET)
+        planet_fraction = planet_weight / total_weight
+        if planet_fraction < min_planet_fraction:
+            continue
+
+        entries = [(c.reviewer_id, m.t_start, m.t_end) for c in panel.values() for m in c.marks]
+        for cluster in cluster_marks(entries, gap_days):
+            first_mark = _one_mark_per_reviewer(cluster)
+            score = sum(_weight(weights, r) for r in first_mark) / total_weight
+            if score < min_score:
+                continue
+            results.append(
+                Consensus(
+                    subject_id=subject_id,
+                    t_start=median(t for t, _ in first_mark.values()),
+                    t_end=median(t for _, t in first_mark.values()),
+                    score=score,
+                    n_reviewers=len(first_mark),
+                    reviewers=tuple(sorted(first_mark)),
+                    planet_fraction=planet_fraction,
+                    marks=tuple((r, a, b) for r, (a, b) in sorted(first_mark.items())),
+                )
+            )
+    return sorted(_merge_duplicates(results, gap_days), key=lambda c: c.score, reverse=True)
+
+
+def counting_rule(gap_days: float, min_score: float) -> str:
+    """The rule that decides which marks count toward a consensus event, in words."""
+    return (
+        f"marks are taken in order of start time and a mark joins the cluster when its interval overlaps the "
+        f"cluster's interval or its centre lies within {gap_days:g} d of the previous mark's centre (interval "
+        f"single-linkage clustering, aggregate.py); each reviewer counts once, with the mark nearest the cluster's "
+        f"median centre; a cluster whose weighted share is below {min_score:g} is dropped, and of two clusters on one "
+        f"dip only the stronger is kept"
+    )
+
+
+def audit_marks(event: Consensus, classifications: list[Classification], gap_days: float) -> list[dict]:
+    """For one event, every reviewer's marks over its window and whether each counted.
+
+    This only reports the rule in :func:`aggregate`; it does not change it. A
+    mark counts when it is the reviewer's mark in the event's cluster
+    (``event.marks``). ``overlap_frac`` is the share of the consensus window
+    [t_start, t_end] the mark covers. ``centre_gap_days`` is the distance from
+    the mark's centre to the nearest centre of a counted mark (another one, for
+    a counted mark). A mark that overlaps no counted mark and whose centre is
+    more than ``gap_days`` from every counted centre fell in a separate cluster.
+    """
+    panel = _latest_per_reviewer([c for c in classifications if c.subject_id == event.subject_id])
+    counted = {r: (a, b) for r, a, b in event.marks}
+    centres = {r: 0.5 * (a + b) for r, (a, b) in counted.items()}
+    width = event.t_end - event.t_start
+    out = []
+    for reviewer_id in sorted(panel):
+        rows = []
+        for m in panel[reviewer_id].marks:
+            overlap = max(0.0, min(m.t_end, event.t_end) - max(m.t_start, event.t_start))
+            is_counted = counted.get(reviewer_id) == (m.t_start, m.t_end)
+            if overlap <= 0 and not is_counted:
+                continue
+            centre = 0.5 * (m.t_start + m.t_end)
+            others = [c for r, c in centres.items() if not (is_counted and r == reviewer_id)]
+            rows.append({
+                "t_start": m.t_start,
+                "t_end": m.t_end,
+                "overlap_frac": overlap / width if width > 0 else None,
+                "centre_gap_days": min(abs(centre - c) for c in others) if others else None,
+                "counted": is_counted,
+            })
+        out.append({
+            "reviewer_id": reviewer_id,
+            "counted": reviewer_id in counted,
+            # The counted mark's overlap when one counted, else the largest overlap of any mark.
+            "overlap_frac": (next((r["overlap_frac"] for r in rows if r["counted"]), None) if reviewer_id in counted
+                             else max((r["overlap_frac"] or 0.0 for r in rows), default=0.0)) if width > 0 else None,
+            "marks": rows,
+        })
+    return out
+
+
+def _merge_duplicates(events: list[Consensus], gap_days: float) -> list[Consensus]:
+    """Keep the strongest of any two events in one subject whose centres are close.
+
+    Two events are the same dip when their centres lie within half the sum of
+    their durations plus the clustering gap. Clustering alone can split one dip
+    when reviewers' boundaries disagree, and this step puts it back together.
+    """
+    kept: list[Consensus] = []
+    # Ties go to the event with more reviewers, then to the wider one: the more complete dip wins.
+    ranked = sorted(events, key=lambda c: (c.score, c.n_reviewers, c.t_end - c.t_start), reverse=True)
+    for event in ranked:
+        duplicate = any(
+            other.subject_id == event.subject_id
+            and abs(other.center - event.center)
+            <= 0.5 * ((other.t_end - other.t_start) + (event.t_end - event.t_start)) + gap_days
+            for other in kept
+        )
+        if not duplicate:
+            kept.append(event)
+    return kept
+
+
+def _latest_per_reviewer(classifications: list[Classification]) -> dict[str, Classification]:
+    """One classification per reviewer per subject. A later duplicate replaces an earlier one."""
+    panel: dict[str, Classification] = {}
+    for classification in classifications:
+        panel[classification.reviewer_id] = classification
+    return panel
+
+
+def _weight(weights: dict[str, float], reviewer_id: str) -> float:
+    return float(weights.get(reviewer_id, 1.0))

setuhq/agreement.py

@@ -0,0 +1,83 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# SPDX-FileCopyrightText: 2026 Ashforde OÜ
+"""How independent are the Sentinels? Pairwise agreement (φ) on the same stars.
+
+Two reviewers that share a model make correlated errors and count as one voter, not two:
+on the stated 40-star subsample of the real-TOI benchmark the two Claude personas agree with
+φ = 0.89. Sentinels on different providers may disagree more, which is what makes a panel
+worth having. This module measures it, from per-star outcomes only:
+
+    hits = {star: {reviewer_id: True/False}}   # did the reviewer mark the TOI's dip as a planet
+
+φ is the Pearson correlation of two binary outcomes (the Matthews coefficient):
+
+    φ = (n11·n00 − n10·n01) / sqrt((n11+n10)(n01+n00)(n11+n01)(n10+n00))
+
+It is None when either reviewer gave the same answer on every star (the denominator is zero).
+Only stars both reviewers answered are compared. ``scripts/benchmark_real_tois.py sentinels``
+collects the outcomes; its ``report`` step writes the table.
+"""
+
+from __future__ import annotations
+
+from itertools import combinations
+
+
+def phi(pairs: list[tuple[bool, bool]]) -> dict:
+    """Agreement of two reviewers over paired binary outcomes: counts, fraction agreeing and φ."""
+    n11 = sum(1 for a, b in pairs if a and b)
+    n00 = sum(1 for a, b in pairs if not a and not b)
+    n10 = sum(1 for a, b in pairs if a and not b)
+    n01 = sum(1 for a, b in pairs if not a and b)
+    den = ((n11 + n10) * (n01 + n00) * (n11 + n01) * (n10 + n00)) ** 0.5
+    return {"n": len(pairs), "agree": n11 + n00, "n11": n11, "n10": n10, "n01": n01, "n00": n00,
+            "phi": (n11 * n00 - n10 * n01) / den if den else None}
+
+
+def pairwise(hits: dict[str, dict[str, bool]], reviewers: list[str] | None = None,
+             identities: dict[str, dict] | None = None) -> list[dict]:
+    """φ for every pair of reviewers, over the stars both answered.
+
+    ``identities`` ({reviewer: {"provider": ..., "model": ...}}) labels each pair as the same model
+    (two personas of one model: expect high φ) or different providers/models (the diversity being measured).
+    """
+    ids = reviewers or sorted({rid for per in hits.values() for rid in per})
+    identities = identities or {}
+    out = []
+    for a, b in combinations(ids, 2):
+        pairs = [(per[a], per[b]) for per in hits.values() if a in per and b in per]
+        ia, ib = identities.get(a) or {}, identities.get(b) or {}
+        same = bool(ia) and bool(ib) and (ia.get("provider"), ia.get("model")) == (ib.get("provider"), ib.get("model"))
+        out.append({"a": a, "b": b, **phi(pairs),
+                    "same_model": same if ia and ib else None,
+                    "a_provider": ia.get("provider"), "a_model": ia.get("model"),
+                    "b_provider": ib.get("provider"), "b_model": ib.get("model")})
+    return out
+
+
+def constant_reviewers(hits: dict[str, dict[str, bool]], reviewers: list[str]) -> list[str]:
+    """Reviewers that gave the same answer on every star they answered (or answered none). φ with them is
+    undefined, so they carry no information about independence and are left out of the voter count."""
+    out = []
+    for rid in reviewers:
+        answers = {per[rid] for per in hits.values() if rid in per}
+        if len(answers) < 2:
+            out.append(rid)
+    return out
+
+
+def effective_voters(table: list[dict], reviewers: list[str], exclude: list[str] = ()) -> float | None:
+    """A rough count of independent voters: k / (1 + (k − 1)·mean φ), with φ < 0 treated as 0.
+
+    k reviewers whose outcomes all correlate at φ act like this many independent ones (the design-effect
+    formula). ``exclude`` (:func:`constant_reviewers`) is left out of k and of the pairs, so k counts only
+    reviewers whose every pair has a φ. None when fewer than two remain, or a remaining pair has no φ
+    (no stars in common).
+    """
+    keep = [r for r in reviewers if r not in set(exclude)]
+    pairs = [p for p in table if p["a"] in keep and p["b"] in keep]
+    k = len(keep)
+    if k < 2 or not pairs or any(p["phi"] is None for p in pairs):
+        return None
+    mean = sum(max(0.0, p["phi"]) for p in pairs) / len(pairs)
+    return k / (1 + (k - 1) * mean)