thisness 0.1.0__py3-none-any.whl

thisness/benchmark/evaluate.py

@@ -0,0 +1,290 @@
+"""Evaluation against ground truth (PRD §45, design 03 §11).
+
+Cluster-level identity safety comes first: false-merge clusters, contaminated
+observations, the largest erroneous cluster. Pair metrics are diagnostics only:
+one bad edge contaminates a whole cluster while barely moving pair precision.
+"""
+
+from __future__ import annotations
+
+import time
+from collections import Counter, defaultdict
+from collections.abc import Callable, Iterable, Mapping
+from dataclasses import asdict, dataclass, field
+from datetime import UTC, datetime
+from itertools import combinations
+from typing import Any
+
+from thisness.benchmark.organizations import Benchmark as OrganizationBenchmark
+from thisness.benchmark.people import Benchmark as PeopleBenchmark
+from thisness.benchmark.taxa import Benchmark as TaxonBenchmark
+from thisness.consensus import components
+from thisness.engine import RunResult, Thisness
+from thisness.ids import IdFactory
+from thisness.profiles.base import ResolutionProfile
+from thisness.profiles.organization import organization
+from thisness.profiles.person import person
+from thisness.profiles.taxon import taxon
+
+Benchmark = OrganizationBenchmark | PeopleBenchmark | TaxonBenchmark
+PROFILES = {"organization": organization, "person": person, "taxon": taxon}
+
+
+@dataclass(frozen=True)
+class ClusterMetrics:
+    observations: int
+    clusters: int
+    b3_precision: float
+    b3_recall: float
+    b3_f1: float
+    false_merge_clusters: int
+    contaminated: int  # observations not of their cluster's majority organization
+    largest_bad_cluster: int
+    by_kind_recall: dict[str, float] = field(default_factory=dict)
+    mixed_by_kind: dict[str, int] = field(default_factory=dict)  # clusters mixing 2+ orgs of a kind
+
+
+def cluster_metrics(
+    assignment: Mapping[str, str], truth: Mapping[str, str], kind: Mapping[str, str]
+) -> ClusterMetrics:
+    """``assignment``: observation -> cluster id. ``truth``: observation -> org id."""
+    clusters: dict[str, set[str]] = defaultdict(set)
+    for obs, cluster in assignment.items():
+        clusters[cluster].add(obs)
+    orgs: dict[str, set[str]] = defaultdict(set)
+    for obs, org in truth.items():
+        if obs in assignment:
+            orgs[org].add(obs)
+    p_sum = r_sum = 0.0
+    kind_r: dict[str, list[float]] = defaultdict(list)
+    for obs, cluster in assignment.items():
+        members, true = clusters[cluster], orgs[truth[obs]]
+        inter = len(members & true)
+        p_sum += inter / len(members)
+        recall = inter / len(true)
+        r_sum += recall
+        kind_r[kind[truth[obs]]].append(recall)
+    n = len(assignment) or 1
+    precision, recall = p_sum / n, r_sum / n
+    bad = [m for m in clusters.values() if len({truth[x] for x in m}) > 1]
+    contaminated = sum(len(m) - Counter(truth[x] for x in m).most_common(1)[0][1] for m in bad)
+    mixed: Counter[str] = Counter()
+    for m in clusters.values():
+        per_kind: dict[str, set[str]] = defaultdict(set)
+        for x in m:
+            per_kind[kind[truth[x]]].add(truth[x])
+        for k, ids in per_kind.items():
+            if len(ids) > 1:
+                mixed[k] += 1
+    return ClusterMetrics(
+        observations=len(assignment),
+        clusters=len(clusters),
+        b3_precision=precision,
+        b3_recall=recall,
+        b3_f1=2 * precision * recall / (precision + recall) if precision + recall else 0.0,
+        false_merge_clusters=len(bad),
+        contaminated=contaminated,
+        largest_bad_cluster=max((len(m) for m in bad), default=0),
+        by_kind_recall={k: sum(v) / len(v) for k, v in sorted(kind_r.items())},
+        mixed_by_kind=dict(sorted(mixed.items())),
+    )
+
+
+def candidate_metrics(
+    pairs: Iterable[tuple[str, str]],
+    truth: Mapping[str, str],
+    kind: Mapping[str, str] | None = None,
+) -> dict:
+    """Pair recall and entity connectivity of a candidate set (design 03 §5), with
+    connectivity per organization kind when ``kind`` is given."""
+    pairs = set(pairs)
+    orgs: dict[str, list[str]] = defaultdict(list)
+    for obs, org in truth.items():
+        orgs[org].append(obs)
+    true_pairs = {
+        (a, b) if a < b else (b, a)
+        for members in orgs.values()
+        for a, b in combinations(members, 2)
+    }
+    found = true_pairs & pairs
+    by_obs: dict[str, set[tuple[str, str]]] = defaultdict(set)
+    for p in pairs:
+        if truth.get(p[0]) is not None and truth.get(p[0]) == truth.get(p[1]):
+            by_obs[truth[p[0]]].add(p)
+    is_connected = {
+        org: len(components(members, by_obs.get(org, ()))) == 1 for org, members in orgs.items()
+    }
+    connected = sum(is_connected.values())
+    per_kind: dict[str, list[bool]] = defaultdict(list)
+    for org, ok in is_connected.items():
+        if kind is not None:
+            per_kind[kind[org]].append(ok)
+    return {
+        "connectivity_by_kind": {k: sum(v) / len(v) for k, v in sorted(per_kind.items())},
+        "candidate_pairs": len(pairs),
+        "true_pairs": len(true_pairs),
+        "pair_recall": len(found) / len(true_pairs) if true_pairs else 1.0,
+        "entity_connectivity": connected / len(orgs) if orgs else 1.0,
+    }
+
+
+@dataclass(frozen=True)
+class Report:
+    benchmark: dict[str, Any]
+    clusters: ClusterMetrics
+    candidates: dict[str, Any]
+    reviews_opened: int
+    review_observations: int
+    seconds: dict[str, float]
+    run: dict[str, Any]
+
+    def as_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    def summary(self) -> str:
+        c = self.clusters
+        return (
+            f"{self.benchmark['records']} records: B3 P {c.b3_precision:.4f} R {c.b3_recall:.4f} "
+            f"F1 {c.b3_f1:.4f} | false-merge clusters {c.false_merge_clusters} "
+            f"(contaminated {c.contaminated}, largest {c.largest_bad_cluster}) | "
+            f"reviews {self.reviews_opened}, {self.review_observations} obs in review | "
+            f"candidates: pair recall {self.candidates['pair_recall']:.3f}, connected "
+            f"{self.candidates['entity_connectivity']:.3f} | "
+            f"T_high {self.run.get('t_high', float('nan')):.1f} | "
+            f"{sum(self.seconds.values()):.1f}s"
+        )
+
+
+def run_benchmark(
+    benchmark: Benchmark,
+    *,
+    profile: ResolutionProfile | None = None,
+    random_pairs: int = 300_000,
+    engine_kwargs: Mapping[str, Any] | None = None,
+    seed: int = 7,
+    adjudicator: Callable[[Thisness, dict[str, str], Any], Any] | None = None,
+    plan: Callable[[Thisness, dict[str, str], Any], Any] | None = None,
+    store: str = ":memory:",
+) -> tuple[Report, Thisness, dict[str, str]]:
+    """Ingest the benchmark into a fresh in-memory engine, resolve it as one bulk run,
+    and measure. Returns the report, the engine, and observation id -> true entity id.
+    ``adjudicator(engine, truth, benchmark)`` builds an adjudicator for the run, so a
+    simulated one can read the ground truth (design 05 §7). ``plan(engine, truth,
+    benchmark)`` builds a run plan the same way, such as a name-only plan (§6). ``store``
+    keeps the run in a DuckDB file, so the Workbench can open it afterwards."""
+    now = datetime(2026, 10, 7, tzinfo=UTC)
+    ids = IdFactory.seeded(seed)
+    entity_type = benchmark.entity_type
+    engine = Thisness(
+        store,
+        profiles=[profile or PROFILES[entity_type]()],
+        ids=ids,
+        clock=lambda: now,
+        random_pairs=random_pairs,
+        **dict(engine_kwargs or {}),
+    )
+    seconds: dict[str, float] = {}
+    started = time.perf_counter()
+    observations = [
+        engine.observe(
+            entity_type,
+            r.attributes,
+            {"type": getattr(r, "source", "benchmark"), "id": r.record_id},
+            ingested_at=now,
+        )
+        for r in benchmark.records
+    ]
+    engine.ingest(observations)
+    seconds["ingest"] = time.perf_counter() - started
+    truth = {
+        o.observation_id: benchmark.truth[r.record_id]
+        for o, r in zip(observations, benchmark.records, strict=True)
+    }
+
+    if adjudicator is not None:
+        engine.adjudicator = adjudicator(engine, truth, benchmark)
+    started = time.perf_counter()
+    run_plan = plan(engine, truth, benchmark) if plan is not None else None
+    result: RunResult = engine.resolve_many(entity_type, plan=run_plan)
+    seconds["fit_and_resolve"] = time.perf_counter() - started
+
+    assignment = engine.store.entities_of(list(truth))
+    clusters = cluster_metrics(assignment, truth, benchmark.kind)
+    candidates = (_candidates_of(engine, result, truth, benchmark.kind) if run_plan is None
+                  else {"pair_recall": float("nan"), "entity_connectivity": float("nan")})  # fmt: skip
+    review_obs = sum(1 for o in result.outcomes.values() if o.outcome == "REQUIRES_REVIEW")
+    report = Report(
+        benchmark=benchmark.config,
+        clusters=clusters,
+        candidates=candidates,
+        reviews_opened=len(result.reviews),
+        review_observations=review_obs,
+        seconds=seconds,
+        run={k: v for k, v in result.stats.items() if k != "candidate_stage"},
+    )
+    if "adjudication" in report.run:
+        report.run["adjudication"] = {
+            **report.run["adjudication"],
+            "vs_truth": adjudication_accuracy(engine, truth, benchmark, result.run_id),
+        }
+    return report, engine, truth
+
+
+def adjudication_accuracy(
+    engine: Thisness, truth: Mapping[str, str], benchmark: Benchmark, run_id: str | None
+) -> dict[str, Any]:
+    """The model's answers against the benchmark's truth (design 06 §10.6): what a real
+    model gets right, and how often it says "different" about one entity, the error that
+    holds a whole component in decide mode. A pair of different entities in one group
+    (a company group, a genus) is "related"; "different" is a right answer there too."""
+    from thisness.benchmark.oracle import benchmark_truth
+
+    truth_of = benchmark_truth(engine, truth)
+    confusion: Counter[tuple[str, str]] = Counter()
+    for judgment in engine.store.decisions_of_type("pair"):
+        if judgment.actor.kind != "agent" or (run_id is not None and judgment.run_id != run_id):
+            continue
+        answer = next((str(s.value) for s in judgment.scores if s.kind == "llm_decision"), None)
+        a, b = truth_of(judgment.left_id), truth_of(judgment.right_id)
+        if answer is None or a is None or b is None:
+            continue
+        group_a, group_b = benchmark.group.get(a), benchmark.group.get(b)
+        actual = ("same" if a == b else
+                  "related" if group_a is not None and group_a == group_b else "different")  # fmt: skip
+        confusion[(actual, answer)] += 1
+    right = {("same", "same"), ("different", "different"), ("related", "related"),
+             ("related", "different")}  # fmt: skip
+    decided = {k: n for k, n in confusion.items() if k[1] != "unsure"}
+    same_pairs = sum(n for (actual, _), n in decided.items() if actual == "same")
+    other_pairs = sum(n for (actual, _), n in decided.items() if actual != "same")
+    total = sum(decided.values())
+
+    def rate(n: int, d: int) -> float | None:
+        return round(n / d, 4) if d else None
+
+    # Of the pairs that are one entity, answered "different" or "related": in decide
+    # mode each is a negative edge inside a true entity.
+    apart = sum(n for (a, x), n in decided.items() if a == "same" and x != "same")
+    # Of those, answered "different": the only answer that sets a negative edge.
+    split = sum(n for (a, x), n in decided.items() if a == "same" and x == "different")
+    # Of the pairs that are not one entity, answered "same": a false merge if applied.
+    merged = sum(n for (a, x), n in decided.items() if a != "same" and x == "same")
+    return {
+        "judged": sum(confusion.values()),
+        "unsure": sum(n for (_, answer), n in confusion.items() if answer == "unsure"),
+        "accuracy": rate(sum(n for k, n in decided.items() if k in right), total),
+        "wrongly_apart": rate(apart, same_pairs),
+        "wrongly_different": rate(split, same_pairs),
+        "wrongly_same": rate(merged, other_pairs),
+        "confusion": {f"{a}->{x}": n for (a, x), n in sorted(confusion.items())},
+    }
+
+
+def _candidates_of(
+    engine: Thisness, result: RunResult, truth: Mapping[str, str], kind: Mapping[str, str]
+) -> dict:
+    from thisness.candidates import distinct_pairs, generate_all
+
+    profile = engine.profiles[result.stats.get("entity_type") or next(iter(engine.profiles))]
+    pairs = distinct_pairs(generate_all(engine.store, profile.candidate_generators, result.run_id))
+    return candidate_metrics(pairs, truth, kind)

thisness/benchmark/oracle.py

@@ -0,0 +1,142 @@
+"""A simulated model that knows the benchmark's ground truth (design 05 §7). It answers
+the adjudicator's pair questions and the review assistant's review questions.
+
+It answers each pair from the truth and then errs in controlled ways, so the
+pipeline and its policy can be measured at scale without spending: how many false
+merges a model that is wrong 5% of the time causes in ``decide`` mode, or one that
+folds related companies into each other 20% of the time. It reads only what a real
+model would be sent, plus the record refs, and is deterministic for a seed.
+"""
+
+from __future__ import annotations
+
+import json
+import random
+from collections import Counter
+from collections.abc import Callable, Mapping
+from itertools import combinations
+
+from thisness.llm.providers import FunctionProvider
+from thisness.llm.types import ModelRequest
+
+
+def oracle_provider(
+    truth_of: Callable[[str], str | None],
+    related: Callable[[str, str], bool],
+    *,
+    error: float = 0.0,
+    unsure: float = 0.0,
+    fold_related: float = 0.0,
+    seed: int = 0,
+    local: bool = True,
+) -> FunctionProvider:
+    """``error``: answer the opposite of the truth (same for different and back).
+    ``unsure``: decline to answer. ``fold_related``: call related entities the same."""
+
+    def review(payload: dict) -> dict:
+        """A review recommendation: the records grouped by their true entity."""
+        relations = payload.get("relations") or ["related_to"]
+        records = payload["records"]
+        rng = random.Random(f"{seed}|{records[0]['record_id']}|review")
+        by_truth: dict[str | None, list[str]] = {}
+        for r in records:
+            truth = truth_of(r["record_id"])
+            by_truth.setdefault(truth if truth is not None else r["ref"], []).append(r["ref"])
+        groups = sorted(sorted(g) for g in by_truth.values())
+        current = sorted(sorted(g) for g in payload["current_groups"])
+        truths = [t for t in by_truth if t is not None]
+        x = rng.random()
+        if x < unsure:
+            return {"resolution": "unsure", "rationale": "simulated"}
+        if x < unsure + error:
+            groups = [sorted(r["ref"] for r in records)] if len(groups) > 1 else \
+                [[r["ref"]] for r in records]  # fmt: skip
+        elif len(truths) == 2 and related(*truths) and rng.random() < fold_related:
+            groups = [sorted(r["ref"] for r in records)]
+        cited = [c["id"] for c in payload["comparisons"] if (c["weight"] or 0) > 0]
+        if len(groups) == 1:
+            out = {"resolution": "same"}
+        elif len(truths) == 2 and groups == current and related(*truths):
+            out = {"resolution": "related", "relation": relations[0]}
+        elif groups == current:
+            out = {"resolution": "different"}
+        else:
+            out = {"resolution": "groups", "groups": groups}
+        return {**out, "cited": cited, "reason_codes": ["OTHER"], "rationale": "simulated"}
+
+    def grouping(payload: dict) -> dict:
+        """A name-only grouping: items grouped by true entity, each labeled with it, so
+        separate batches converge on one label for one entity (design 01 §5.3)."""
+        items = payload["items"]
+        rng = random.Random(f"{seed}|{items[0]['record_id']}|{len(items)}|group")
+        truth = {it["ref"]: truth_of(it["record_id"]) or it["ref"] for it in items}
+        if fold_related:
+            # A model that folds related entities together: one label per related set.
+            keys = sorted(set(truth.values()))
+            merged = {k: k for k in keys}
+            for a, b in combinations(keys, 2):
+                if related(a, b) and random.Random(f"{seed}|{a}|{b}|fold").random() < fold_related:
+                    merged[b] = merged[a]
+            truth = {ref: merged[t] for ref, t in truth.items()}
+        by_truth: dict[str, list[str]] = {}
+        for ref, t in truth.items():
+            x = rng.random()
+            if x < unsure:
+                continue
+            if x < unsure + error:
+                t = rng.choice(sorted(set(truth.values())))  # a wrong group, or the right one
+            by_truth.setdefault(t, []).append(ref)
+        return {"groups": [{"members": sorted(refs), "label": str(t)}
+                           for t, refs in sorted(by_truth.items())]}  # fmt: skip
+
+    def answer(request: ModelRequest) -> dict:
+        payload = json.loads(request.messages[0].content)
+        if "items" in payload:
+            return grouping(payload)
+        if "records" in payload:
+            return review(payload)
+        relations = payload.get("relations") or ["related_to"]
+        out = []
+        for p in payload["pairs"]:
+            a, b = p["a"]["ref"], p["b"]["ref"]
+            ta, tb = truth_of(a), truth_of(b)
+            rng = random.Random(f"{seed}|{a}|{b}")
+            if ta is not None and ta == tb:
+                correct = "same"
+            elif ta is not None and tb is not None and related(ta, tb):
+                correct = "related"
+            else:
+                correct = "different"
+            x = rng.random()
+            if x < unsure:
+                decision = "unsure"
+            elif x < unsure + error:
+                decision = "different" if correct == "same" else "same"
+            elif correct == "related" and rng.random() < fold_related:
+                decision = "same"
+            else:
+                decision = correct
+            positive = [c["id"] for c in p["comparisons"] if c["bits"] > 0]
+            negative = [c["id"] for c in p["comparisons"] if c["bits"] < 0]
+            out.append({
+                "pair": p["pair"], "decision": decision,
+                "relation": relations[0] if decision == "related" else None,
+                "supporting": positive if decision == "same" else negative,
+                "contradicting": negative if decision == "same" else positive,
+                "reason_codes": ["OTHER"], "rationale": "simulated",
+            })  # fmt: skip
+        return {"answers": out}
+
+    return FunctionProvider(answer, name="oracle", local=local)
+
+
+def benchmark_truth(engine, truth: Mapping[str, str]) -> Callable[[str], str | None]:
+    """A record's true entity; an entity's is its members' majority truth."""
+
+    def truth_of(node: str) -> str | None:
+        if node in truth:
+            return truth[node]
+        members = [truth[m] for m in engine.members(node) if m in truth]
+        return Counter(members).most_common(1)[0][0] if members else None
+
+    return truth_of