agentloop-profiler 0.1.0__py3-none-any.whl

agentloop/semantic_rules.py

@@ -0,0 +1,200 @@
+"""Canonical finding candidates from validated, explicitly scoped investigations."""
+
+from __future__ import annotations
+
+from fractions import Fraction
+from types import SimpleNamespace
+
+from agentloop.costs import is_cost_evaluable
+from agentloop.semantic_waste_types import SemanticInvestigation
+from agentloop.timing import event_interval_ms
+from agentloop.tokens import token_status
+
+_TITLES = {
+    "semantic_redundancy": "Investigate semantically redundant work",
+    "low_contribution": "Investigate low-contribution work",
+    "semantic_no_progress": "Investigate a sequence with no task progress",
+    "retry_usefulness": "Investigate retries without useful new evidence",
+    "context_relevance": "Investigate context unrelated to the decision",
+}
+
+
+def _total(values):
+    try:
+        return float(sum((Fraction(str(value)) for value in values), Fraction()))
+    except (ValueError, OverflowError):
+        return None
+
+
+def _inputs(context):
+    events = {item["event_id"]: item for item in context.report["events"]}
+    children = {item.get("parent_id") for item in events.values() if item.get("parent_id")}
+    model_events = [item for item in events.values() if item["event_type"] == "model_call"]
+    model_costs = context.report.get("cost_breakdown", {}).get("model_calls", [])
+    return {
+        "events": events,
+        "children": children,
+        "model_costs": {event["event_id"]: cost for event, cost in zip(model_events, model_costs)}
+        if len(model_events) == len(model_costs)
+        else {},
+    }
+
+
+def _estimate(context, case, row, inputs):
+    events = inputs["events"]
+    selected = [events[identity] for identity in case.target_spans]
+    leaf = all(item["event_id"] not in inputs["children"] for item in selected)
+    all_models = all(item["event_type"] == "model_call" for item in selected)
+    selected_token_status = (
+        token_status([SimpleNamespace(**item) for item in selected]) if all_models else None
+    )
+    eligible = bool(case.removal_attribution_ref and leaf and case.family != "context_relevance")
+    predictions = {
+        "latency_ms": None,
+        "cost_usd": None,
+        "input_tokens": None,
+        "output_tokens": None,
+    }
+    assumptions = [
+        "Selected evidence and approved summaries cover the relevant task context and downstream outcomes.",
+        "The finding is a hypothesis; independent paired replay must preserve the caller's quality criteria.",
+    ]
+    if row["basis"] == "deterministic_reuse":
+        assumptions.append(
+            "The referenced host reuse contract guarantees complete inputs, unchanged operation semantics, and safe result reuse without required side effects or independent verification."
+        )
+    costs = []
+    serial = False
+    if eligible:
+        assumptions.extend(
+            [
+                "The caller's removal attribution applies to every selected leaf span, including side effects and downstream dependencies.",
+                "Removing those spans requires no replacement work, cache lookup, repair, or additional validation overhead.",
+                "Resource estimates are conditional upper bounds on recorded work, not realized savings or a correctness guarantee.",
+            ]
+        )
+        if "serial" not in inputs:
+            intervals = [event_interval_ms(SimpleNamespace(**item)) for item in events.values()]
+            details = context.graph.dependency_summary()
+            inputs["serial"] = False
+            if (
+                all(value is not None for value in intervals)
+                and all(
+                    item["duration_ms"] > 0 or interval[0] == interval[1]
+                    for item, interval in zip(events.values(), intervals)
+                )
+                and all(not item.get("parent_id") for item in events.values())
+                and (details is None or details["valid"])
+            ):
+                ordered = sorted(intervals)
+                inputs["serial"] = all(
+                    left[1] <= right[0] for left, right in zip(ordered, ordered[1:])
+                )
+        serial = inputs["serial"]
+        if serial:
+            predictions["latency_ms"] = _total(item["duration_ms"] for item in selected)
+            assumptions.append(
+                "The recorded flat, nonoverlapping execution remains serial after the change, so removed span time can shorten elapsed runtime."
+            )
+        if all_models:
+            exact_tokens = selected_token_status in {
+                "exact",
+                "empty",
+            }
+            if exact_tokens:
+                predictions["input_tokens"] = sum(item["input_tokens"] for item in selected)
+                predictions["output_tokens"] = sum(item["output_tokens"] for item in selected)
+            by_id = inputs["model_costs"]
+            if all(item["event_id"] in by_id for item in selected):
+                costs = [
+                    {"span_id": item["event_id"], **by_id[item["event_id"]]} for item in selected
+                ]
+                if is_cost_evaluable(context.report.get("cost_status")) and all(
+                    item["state"] == "provider_reported"
+                    or (item["state"] == "calculated" and exact_tokens)
+                    for item in costs
+                ):
+                    predictions["cost_usd"] = _total(item["amount_usd"] for item in costs)
+    return {
+        "estimator_id": "semantic_leaf_removal",
+        "estimator_version": "1.0",
+        "method": "conditional_upper_bound" if eligible else "unavailable_attribution",
+        "confidence": row["confidence"],
+        "calibrated": False,
+        "formula": "conditional removal: sum selected nonoverlapping serial leaf durations; sum separately attributed model billing and exact model token counts",
+        "parameters": {},
+        "assumptions": assumptions,
+        "inputs": {
+            "family": case.family,
+            "attribution_eligible": eligible,
+            "all_model_calls": all_models,
+            "token_status": selected_token_status,
+            "trace_cost_status": context.report.get("cost_status"),
+            "target_spans": list(case.target_spans),
+            "leaf_spans": leaf,
+            "flat_serial_timing": serial,
+            "removal_attribution_ref": case.removal_attribution_ref,
+            "recorded_target_work_ms": _total(item["duration_ms"] for item in selected),
+            "model_costs": costs,
+        },
+        "predictions": predictions,
+        "unmodeled_metrics": [key for key, value in predictions.items() if value is None],
+    }
+
+
+def semantic_candidates(context, family):
+    from agentloop.rules import FindingCandidate, RecommendationType
+
+    evidence = context.report.get("semantic_waste")
+    if not evidence or evidence["status"] in {"invalid", "unsupported"}:
+        return []
+    result = []
+    inputs = None
+    for row in evidence["cases"]:
+        if row["status"] != "supported" or row["definition"]["family"] != family:
+            continue
+        case = SemanticInvestigation.from_dict(row["definition"])
+        if inputs is None:
+            inputs = _inputs(context)
+        estimate = _estimate(context, case, row, inputs)
+        observations = {
+            "investigation_id": case.investigation_id,
+            "criteria_ref": case.criteria_ref,
+            "reference_spans": list(case.reference_spans),
+            "target_spans": list(case.target_spans),
+            "evidence_hash": evidence["evidence_hash"],
+            "basis": row["basis"],
+            "reuse_contract_ref": case.reuse_contract_ref,
+            "judgments": [
+                {
+                    "record_hash": item["record_hash"],
+                    "judge": item["judge"],
+                    "uncertainty": item["evaluation"]["uncertainty"],
+                    "status": item["effective_status"],
+                    "value": item["effective_value"],
+                    "evidence_refs": [part["span_id"] for part in item["request"]["evidence"]],
+                }
+                for item in row["judgments"]
+            ],
+            "validation_criteria": "Run a paired intervention against the original trace, with an independent scorer implementing the referenced task criteria, preserved required side effects, and complete cost/token provenance.",
+        }
+        result.append(
+            FindingCandidate(
+                type=RecommendationType(family),
+                title=_TITLES[family],
+                why="An explicit investigation supports this hypothesis under the recorded task criteria and evidence coverage assumptions.",
+                rewrite_hint="Prepare an offline candidate and validate its independent task quality before accepting any step removal or context change.",
+                confidence=row["confidence"],
+                affected_nodes=list(case.target_spans),
+                estimated_latency_savings_ms=estimate["predictions"]["latency_ms"],
+                estimated_cost_savings_usd=estimate["predictions"]["cost_usd"],
+                evidence_level="deterministic_inference"
+                if row["basis"] == "deterministic_reuse"
+                else "semantic_judgment",
+                assumptions=estimate["assumptions"],
+                observations=observations,
+                estimate_formula=estimate["formula"],
+                estimate=estimate,
+            )
+        )
+    return result

agentloop/semantic_waste.py

@@ -0,0 +1,345 @@
+"""Optional offline investigations; report construction only reads saved evidence."""
+
+from __future__ import annotations
+
+import json
+from copy import deepcopy
+from types import SimpleNamespace
+
+from agentloop.judgment_types import (
+    JUDGMENT_KEY,
+    JudgmentRequest,
+    JudgmentSpec,
+    canonical,
+    fingerprint,
+    finite,
+)
+from agentloop.judgments import (
+    JudgmentSession,
+    _bound,
+    attach_judgment,
+    judgment_request,
+    read_judgments,
+)
+from agentloop.operations import operation_kind
+from agentloop.semantic_waste_types import (
+    QUESTIONS,
+    SEMANTIC_WASTE_KEY,
+    SEMANTIC_WASTE_VERSION,
+    SemanticInvestigation,
+)
+from agentloop.timing import event_interval_ms
+
+
+def _copy(value):
+    return json.loads(canonical(value))
+
+
+def _spec(case):
+    question = (
+        QUESTIONS[case.family]
+        + f" The first {len(case.reference_spans)} evidence spans are reference context; "
+        + f"the remaining {len(case.target_spans)} spans are targets. "
+        + "Task criteria: "
+        + case.task_criteria
+        + " Criteria reference digest: "
+        + fingerprint(case.criteria_ref)
+    )
+    return JudgmentSpec(question, "boolean")
+
+
+def _deterministic_reuse(case, events):
+    """Exact recorded equality under an explicit host reuse contract, not name counts."""
+    if (
+        case.family != "semantic_redundancy"
+        or not case.reuse_safe
+        or len(case.target_spans) != 1
+        or len(case.reference_spans) != 1
+    ):
+        return False
+    earlier = events.get(case.reference_spans[0])
+    later = events.get(case.target_spans[0])
+    if earlier is None or later is None:
+        return False
+    first, second = event_interval_ms(earlier), event_interval_ms(later)
+    ignored = {
+        "otel_span_id",
+        "otel_trace_id",
+        "provider_reported_cost_usd",
+        "cost_usd",
+        "cached_input_tokens",
+    }
+    same_context = fingerprint(
+        {key: value for key, value in earlier.metadata.items() if key not in ignored}
+    ) == fingerprint({key: value for key, value in later.metadata.items() if key not in ignored})
+    return bool(
+        first is not None
+        and second is not None
+        and first[1] <= second[0]
+        and earlier.status == later.status == "ok"
+        and earlier.name == later.name
+        and earlier.model == later.model
+        and same_context
+        and earlier.input_tokens == later.input_tokens
+        and earlier.output_tokens == later.output_tokens
+        and operation_kind(earlier) == operation_kind(later)
+        and isinstance(earlier.input_text, str)
+        and bool(earlier.input_text)
+        and isinstance(earlier.output_text, str)
+        and bool(earlier.output_text)
+        and earlier.input_text == later.input_text
+        and earlier.output_text == later.output_text
+    )
+
+
+def _scope_gap(case, events):
+    targets = [events[identity] for identity in case.target_spans]
+    references = [events[identity] for identity in case.reference_spans]
+    target_times = [event_interval_ms(item) for item in targets]
+    reference_times = [event_interval_ms(item) for item in references]
+    if case.family == "retry_usefulness":
+        if not all(
+            item.event_type == "retry" or item.metadata.get("retry_of") in case.reference_spans
+            for item in targets
+        ):
+            return "missing_retry_relation"
+    if case.family in {"semantic_redundancy", "low_contribution", "semantic_no_progress"}:
+        if any(item is None for item in target_times + reference_times):
+            return "missing_order_evidence"
+        if case.family == "semantic_redundancy" and max(item[1] for item in reference_times) > min(
+            item[0] for item in target_times
+        ):
+            return "reference_not_available"
+        if case.family == "low_contribution" and not any(
+            item[0] >= max(target[1] for target in target_times) for item in reference_times
+        ):
+            return "missing_downstream_evidence"
+        if case.family == "semantic_no_progress" and any(
+            left[1] > right[0] for left, right in zip(target_times, target_times[1:])
+        ):
+            return "unordered_sequence"
+    if case.family == "context_relevance" and not all(
+        item.event_type == "model_call" or operation_kind(item) in {"model", "classifier", "rule"}
+        for item in targets
+    ):
+        return "ineligible_context_target"
+    return None
+
+
+def evaluate_semantic_waste(trace, investigations, *, judges=(), enabled=False, timeout_s=None):
+    """Attach new purpose-bound investigations; no runtime action is proposed or run.
+
+    Source traces must remain unchanged during this offline operation. Prior
+    investigation IDs cannot be replaced. Use a new ID to record a reevaluation.
+    Approved criteria/summaries are the only caller text forwarded to a judge.
+    """
+    if type(enabled) is not bool:
+        raise ValueError("enabled must be a boolean")
+    if timeout_s is not None and (not finite(timeout_s) or timeout_s <= 0):
+        raise ValueError("timeout_s must be finite and positive")
+    if (
+        not isinstance(investigations, (tuple, list))
+        or not investigations
+        or len(investigations) > 100
+        or any(type(item) is not SemanticInvestigation for item in investigations)
+    ):
+        raise ValueError("supply between one and 100 SemanticInvestigation objects")
+    if not isinstance(judges, (tuple, list)) or len(judges) > 8:
+        raise ValueError("judges must be a list or tuple of at most eight backends")
+    identities = [item.investigation_id for item in investigations]
+    if len(set(identities)) != len(identities):
+        raise ValueError("investigation IDs must be unique")
+    old = trace.metadata.get(SEMANTIC_WASTE_KEY)
+    if old is not None:
+        current = read_semantic_waste(trace)
+        if current["status"] in {"invalid", "unsupported"}:
+            raise ValueError("existing semantic investigations are invalid or unsupported")
+        if set(identities) & {item["definition"]["investigation_id"] for item in old["cases"]}:
+            raise ValueError("use a new investigation ID rather than replacing saved evidence")
+    elif SEMANTIC_WASTE_KEY in trace.metadata:
+        raise ValueError("existing semantic investigations are invalid")
+    working = SimpleNamespace(
+        run_id=trace.run_id, events=trace.events, metadata=deepcopy(trace.metadata)
+    )
+    sessions = [JudgmentSession(backend) for backend in judges]
+    events = {event.event_id: event for event in trace.events}
+    cases = []
+    for case in investigations:
+        row = {
+            "definition": case.to_dict(),
+            "request": None,
+            "judgment_hashes": [],
+            "basis": "unavailable",
+            "reason": None,
+        }
+        try:
+            request = judgment_request(
+                trace, _spec(case), list(case.evidence_spans), summaries=dict(case.summaries)
+            )
+        except ValueError:
+            row["reason"] = "missing_or_invalid_evidence"
+            cases.append(row)
+            continue
+        row["request"] = request.to_dict()
+        if case.retention_required:
+            row["reason"] = "retention_required"
+        elif _scope_gap(case, events) is not None:
+            row["reason"] = _scope_gap(case, events)
+        elif _deterministic_reuse(case, events):
+            row["basis"] = "deterministic_reuse"
+        elif set(dict(case.summaries)) != set(case.evidence_spans):
+            row["reason"] = "missing_approved_summaries"
+        elif not sessions:
+            row["reason"] = "no_judge"
+        else:
+            row["basis"] = "semantic_judgment"
+            for session in sessions:
+                receipt = session.evaluate(request, enabled=enabled, timeout_s=timeout_s)
+                attach_judgment(working, receipt)
+                row["judgment_hashes"].append(receipt["record_hash"])
+        cases.append(row)
+    envelope = {
+        "schema_version": SEMANTIC_WASTE_VERSION,
+        "run_id": trace.run_id,
+        "cases": [*(old["cases"] if old is not None else []), *cases],
+    }
+    envelope["evidence_hash"] = fingerprint(envelope)
+    working.metadata[SEMANTIC_WASTE_KEY] = envelope
+    checked = read_semantic_waste(working)
+    if checked["status"] == "invalid" or any(
+        item["status"] == "stale" for item in checked["cases"][-len(cases) :]
+    ):
+        raise ValueError("source evidence changed during the investigation")
+    if JUDGMENT_KEY in working.metadata:
+        trace.metadata[JUDGMENT_KEY] = working.metadata[JUDGMENT_KEY]
+    trace.metadata[SEMANTIC_WASTE_KEY] = envelope
+    return checked
+
+
+def read_semantic_waste(trace, *, judgments=None):
+    """Validate the purpose, source and judge bindings without executing a backend."""
+    metadata = getattr(trace, "metadata", {})
+    if SEMANTIC_WASTE_KEY not in metadata:
+        return None
+    empty = {"status": "invalid", "cases": []}
+    try:
+        envelope = _copy(metadata[SEMANTIC_WASTE_KEY])
+        if not isinstance(envelope, dict):
+            return empty
+        if envelope.get("schema_version") != SEMANTIC_WASTE_VERSION:
+            return {**empty, "status": "unsupported"}
+        digest = envelope.pop("evidence_hash")
+        if digest != fingerprint(envelope) or not isinstance(envelope.get("cases"), list):
+            return empty
+        if judgments is None:
+            judgments = read_judgments(trace)
+        receipts = {item["record_hash"]: item for item in (judgments or {}).get("records", [])}
+        events = {event.event_id: event for event in trace.events}
+        source_hashes, seen, cases = {}, set(), []
+        for raw in envelope["cases"]:
+            case = SemanticInvestigation.from_dict(raw["definition"])
+            if case.investigation_id in seen:
+                return empty
+            seen.add(case.investigation_id)
+            hashes = raw["judgment_hashes"]
+            if (
+                not isinstance(hashes, list)
+                or any(not isinstance(value, str) for value in hashes)
+                or len(set(hashes)) != len(hashes)
+            ):
+                return empty
+            row = {**raw, "status": "unknown", "confidence": "unknown", "judgments": []}
+            if envelope["run_id"] != trace.run_id:
+                row["status"] = "stale"
+            elif raw["request"] is None:
+                if raw["basis"] != "unavailable" or hashes:
+                    return empty
+            else:
+                request = JudgmentRequest.from_dict(raw["request"])
+                if (
+                    request.spec != _spec(case)
+                    or tuple(item.span_id for item in request.evidence) != case.evidence_spans
+                    or {
+                        item.span_id: item.summary
+                        for item in request.evidence
+                        if item.summary is not None
+                    }
+                    != dict(case.summaries)
+                ):
+                    return empty
+                if not _bound(
+                    trace, {"request": raw["request"]}, events=events, hashes=source_hashes
+                ):
+                    row["status"] = "stale"
+                elif case.retention_required:
+                    row["status"] = "retained"
+                elif _scope_gap(case, events) is not None:
+                    row["reason"] = _scope_gap(case, events)
+                elif raw["basis"] == "deterministic_reuse":
+                    if hashes or not _deterministic_reuse(case, events):
+                        return empty
+                    row.update(status="supported", confidence="medium")
+                elif raw["basis"] == "semantic_judgment":
+                    if not hashes:
+                        return empty
+                    selected = [receipts.get(value) for value in hashes]
+                    if any(item is None or item["request"] != raw["request"] for item in selected):
+                        row["reason"] = "missing_or_mismatched_judgment"
+                    else:
+                        row["judgments"] = selected
+                        if any(item["effective_status"] != "known" for item in selected):
+                            row["reason"] = "unknown_or_failed_judgment"
+                        elif len({item["effective_value"] for item in selected}) != 1:
+                            row["status"] = "disagreement"
+                        elif not selected[0]["effective_value"]:
+                            row["status"] = "not_supported"
+                        elif any(
+                            item["evaluation"]["uncertainty"]["confidence"] is not None
+                            and item["evaluation"]["uncertainty"]["confidence"]
+                            < case.minimum_confidence
+                            for item in selected
+                        ):
+                            row["status"] = "low_confidence"
+                        else:
+                            row.update(status="supported", confidence="low")
+                elif raw["basis"] != "unavailable" or hashes:
+                    return empty
+            cases.append(row)
+        return {
+            "status": "complete"
+            if cases
+            and all(item["status"] in {"supported", "not_supported", "retained"} for item in cases)
+            else "incomplete",
+            "cases": cases,
+            "evidence_hash": digest,
+        }
+    except (ValueError, TypeError, KeyError, AttributeError):
+        return empty
+
+
+def semantic_waste_markdown(evidence):
+    from agentloop.markdown import markdown_table_cell
+
+    lines = [
+        "",
+        "## Offline semantic investigations",
+        "",
+        "Observed facts, deterministic inferences and judge opinions are distinct. Unsupported cases do not produce findings; conditional savings require independent paired replay.",
+        "",
+        f"Investigation completeness: {markdown_table_cell(evidence['status'])}",
+        "",
+        "| Investigation | Family | Status | Basis | Task criteria reference | Judge references |",
+        "| --- | --- | --- | --- | --- | --- |",
+    ]
+    for item in evidence["cases"]:
+        case = item["definition"]
+        cells = [
+            case["investigation_id"],
+            case["family"],
+            item["status"],
+            item["basis"],
+            case["criteria_ref"],
+            ", ".join(item["judgment_hashes"]) or "none",
+        ]
+        lines.append("| " + " | ".join(markdown_table_cell(str(value)) for value in cells) + " |")
+    return lines

agentloop/semantic_waste_types.py

@@ -0,0 +1,88 @@
+"""Explicit investigation declarations for optional semantic waste findings."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass
+
+from agentloop.judgment_types import canonical, finite, text
+
+SEMANTIC_WASTE_KEY = "agentloop.semantic_waste"
+SEMANTIC_WASTE_VERSION = "1.0"
+QUESTIONS = {
+    "semantic_redundancy": "Do the target steps materially repeat work already available in the reference steps, without a required independent check?",
+    "low_contribution": "Do the target steps add no material information needed by the supplied downstream result under the task criteria?",
+    "semantic_no_progress": "Does the target sequence fail to make meaningful progress under the task criteria, rather than perform required polling, verification or state transitions?",
+    "retry_usefulness": "Do the target retries repeat the same failure or input without useful new evidence, rather than satisfy an intentional retry policy?",
+    "context_relevance": "Is the supplied context in the target steps unrelated to the decision under the task criteria?",
+}
+
+
+@dataclass(frozen=True)
+class SemanticInvestigation:
+    investigation_id: str
+    family: str
+    target_spans: tuple[str, ...]
+    reference_spans: tuple[str, ...]
+    task_criteria: str
+    criteria_ref: str
+    summaries: tuple[tuple[str, str], ...] = ()
+    retention_required: bool = False
+    reuse_safe: bool = False
+    reuse_contract_ref: str | None = None
+    removal_attribution_ref: str | None = None
+    minimum_confidence: float = 0.8
+
+    def __post_init__(self):
+        for key in ("investigation_id", "task_criteria", "criteria_ref"):
+            text(getattr(self, key), key)
+        if len(self.task_criteria) > 2048:
+            raise ValueError("task_criteria must be at most 2048 characters")
+        if not finite(self.minimum_confidence) or not 0 <= self.minimum_confidence <= 1:
+            raise ValueError("minimum_confidence must be in [0, 1]")
+        if self.family not in QUESTIONS:
+            raise ValueError("unsupported semantic finding family")
+        for key in ("target_spans", "reference_spans"):
+            value = getattr(self, key)
+            if not isinstance(value, (tuple, list)) or not value:
+                raise ValueError(f"{key} requires at least one span ID")
+            for item in value:
+                text(item, key)
+            if len(set(value)) != len(value):
+                raise ValueError(f"{key} must be unique")
+            object.__setattr__(self, key, tuple(value))
+        if set(self.target_spans) & set(self.reference_spans):
+            raise ValueError("target and reference spans must be distinct")
+        if len(self.evidence_spans) > 64:
+            raise ValueError("an investigation can reference at most 64 spans")
+        if self.family == "semantic_no_progress" and len(self.target_spans) < 2:
+            raise ValueError("no-progress investigation requires a sequence of target spans")
+        for key in ("retention_required", "reuse_safe"):
+            if type(getattr(self, key)) is not bool:
+                raise ValueError(f"{key} must be a boolean declaration")
+        for key in ("reuse_contract_ref", "removal_attribution_ref"):
+            text(getattr(self, key), key, optional=True)
+        if self.reuse_safe and self.reuse_contract_ref is None:
+            raise ValueError("safe reuse requires a supporting contract reference")
+        values = self.summaries.items() if isinstance(self.summaries, dict) else self.summaries
+        try:
+            pairs = tuple((key, value) for key, value in values)
+        except (TypeError, ValueError):
+            raise ValueError("summaries must map span IDs to approved derived text") from None
+        for key, value in pairs:
+            text(key, "summary span ID")
+            text(value, "summary")
+        if len(dict(pairs)) != len(pairs) or set(dict(pairs)) - set(self.evidence_spans):
+            raise ValueError("summaries must name unique selected spans")
+        object.__setattr__(self, "summaries", tuple(sorted(pairs)))
+
+    @property
+    def evidence_spans(self):
+        return self.reference_spans + self.target_spans
+
+    def to_dict(self):
+        return json.loads(canonical(asdict(self)))
+
+    @classmethod
+    def from_dict(cls, value):
+        return cls(**value)