structuremappingmemory 1.0.0__py3-none-any.whl

sma/encoders/structured.py

@@ -0,0 +1,57 @@
+"""Structured JSON/CSV/XML-ish Tier-0 encoder."""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+from collections.abc import Mapping
+
+from sma.ir.schema import Statement, entity, make_case, stmt
+
+from .base import EncodeResult
+
+
+class StructuredEncoder:
+    adapter_id = "structured"
+    version = "0.1.0"
+
+    def encode(self, artifact: str, **kwargs) -> EncodeResult:
+        fmt = kwargs.get("format") or infer_format(artifact)
+        statements: list[Statement] = []
+        if fmt == "json":
+            data = json.loads(artifact)
+            encode_json(data, statements, "root")
+        else:
+            reader = csv.DictReader(io.StringIO(artifact))
+            for i, row in enumerate(reader):
+                row_ent = entity(f"row_{i}", "row")
+                for key, value in row.items():
+                    if value is not None and value != "":
+                        statements.append(stmt(key, row_ent, entity(value, "value")))
+        case = make_case(statements or [stmt("emptyStructured", entity("root"))], {"adapter": self.adapter_id, "tier": 0})
+        return EncodeResult(case, ())
+
+
+def infer_format(artifact: str) -> str:
+    stripped = artifact.lstrip()
+    if stripped.startswith("{") or stripped.startswith("["):
+        return "json"
+    return "csv"
+
+
+def encode_json(value, statements: list[Statement], path: str) -> None:
+    subject = entity(path, "json_node")
+    if isinstance(value, Mapping):
+        for key, child in sorted(value.items()):
+            child_path = f"{path}.{key}"
+            statements.append(stmt(str(key), subject, entity(child_path, "json_node")))
+            encode_json(child, statements, child_path)
+    elif isinstance(value, list):
+        for i, child in enumerate(value):
+            child_path = f"{path}.{i}"
+            statements.append(stmt("item", subject, entity(child_path, "json_node")))
+            encode_json(child, statements, child_path)
+    else:
+        statements.append(stmt("value", subject, entity(str(value), "value")))
+

sma/encoders/traces.py

@@ -0,0 +1,45 @@
+"""Stack trace and exception grammar encoder."""
+
+from __future__ import annotations
+
+import re
+
+from sma.ir.schema import Statement, entity, make_case, stmt
+
+from .base import EncodeResult
+
+FRAME_RE = re.compile(r'File "?(?P<file>[^",\n]+)"?, line (?P<line>\d+), in (?P<func>[A-Za-z_][\w.]*)')
+JAVA_FRAME_RE = re.compile(r"\s*at (?P<class>[\w.$]+)\.(?P<func>\w+)\((?P<file>[^:]+):(?P<line>\d+)\)")
+CAUSE_RE = re.compile(r"(?:(?:Caused by|The above exception).*?:\s*)?(?P<exc>[A-Za-z_][\w.]*Error|[A-Za-z_][\w.]*Exception)")
+
+
+class TraceEncoder:
+    adapter_id = "traces"
+    version = "0.1.0"
+
+    def encode(self, artifact: str, **kwargs) -> EncodeResult:
+        statements: list[Statement] = []
+        frames: list[Statement] = []
+        for i, line in enumerate(artifact.splitlines()):
+            frame = FRAME_RE.search(line) or JAVA_FRAME_RE.search(line)
+            if frame:
+                file = frame.group("file")
+                func = frame.group("func")
+                line_no = frame.group("line")
+                frame_stmt = stmt(
+                    "frame",
+                    entity(f"f{i}", "frame"),
+                    entity(file, "file"),
+                    entity(func, "function"),
+                    entity(line_no, "line"),
+                )
+                frames.append(frame_stmt)
+                statements.append(frame_stmt)
+            cause = CAUSE_RE.search(line)
+            if cause:
+                statements.append(stmt("exception", entity(cause.group("exc"), "exception")))
+        for left, right in zip(frames, frames[1:], strict=False):
+            statements.append(stmt("calledFrom", left, right))
+        case = make_case(statements or [stmt("emptyTrace", entity("trace_0"))], {"adapter": self.adapter_id, "tier": 0})
+        return EncodeResult(case, ())
+

sma/eval/__init__.py

@@ -0,0 +1,2 @@
+"""Evaluation package."""
+

sma/eval/agentic/__init__.py

@@ -0,0 +1,35 @@
+"""Agentic ontology benchmark harness (memory-swap suite)."""
+
+from __future__ import annotations
+
+from sma.eval.agentic.memories import (
+    BM25Memory,
+    DenseMemory,
+    HippoMemory,
+    HybridRerankMemory,
+    HybridRRFMemory,
+    IndexItem,
+    Memory,
+    Query,
+    Retrieved,
+    SmaMemory,
+)
+from sma.eval.agentic.metrics import novelty_f1, risk_coverage_aurc, tail_topk
+from sma.eval.agentic.harness import run_oneshot
+
+__all__ = [
+    "run_oneshot",
+    "tail_topk",
+    "risk_coverage_aurc",
+    "novelty_f1",
+    "IndexItem",
+    "Query",
+    "Retrieved",
+    "Memory",
+    "SmaMemory",
+    "BM25Memory",
+    "DenseMemory",
+    "HippoMemory",
+    "HybridRRFMemory",
+    "HybridRerankMemory",
+]

sma/eval/agentic/arms/cyber.py

@@ -0,0 +1,48 @@
+"""Cyber arm: threat-group attribution over MITRE ATT&CK.
+
+Entities are intrusion-sets (threat groups); each is annotated with the set of
+ATT&CK techniques it is documented to USE (gold, from the STIX ``uses``
+relationships). The hard query simulates a partial/imprecise incident (a few
+observed TTPs, climbed up the sub-technique lattice, plus noise); the task is to
+attribute it to the right group. ATT&CK's sub-technique tree is the is-a lattice.
+"""
+from __future__ import annotations
+
+import json
+import pathlib
+
+from sma.ontology import load_attack_stix, mount
+
+ROOT = pathlib.Path(__file__).resolve().parents[4]
+STIX = ROOT / "data/raw/attack/enterprise-attack.json"
+# Band matches the medicine arm: >=7 techniques to be non-trivial, <=30 to keep
+# structure-mapping cases tractable (prolific groups with 100+ techniques blow up
+# SME kernel enumeration). Excludes the most-documented groups; n reported.
+MIN_TERMS, MAX_TERMS = 7, 30
+
+
+def load():
+    graph = load_attack_stix(str(STIX), name="attack")
+    mounted = mount(graph)
+    bundle = json.loads(STIX.read_text())
+    objs = bundle["objects"]
+
+    # stix-id -> ATT&CK external_id (e.g. "T1059.001"), for attack-patterns.
+    ext: dict[str, str] = {}
+    for o in objs:
+        if o.get("type") == "attack-pattern":
+            for ref in o.get("external_references", []):
+                if ref.get("source_name") == "mitre-attack" and ref.get("external_id"):
+                    ext[o["id"]] = ref["external_id"]
+    groups = {o["id"]: o.get("name", o["id"])
+              for o in objs if o.get("type") == "intrusion-set" and not o.get("revoked")}
+
+    recs: dict[str, set[str]] = {}
+    for o in objs:
+        if o.get("type") == "relationship" and o.get("relationship_type") == "uses":
+            s, t = o.get("source_ref"), o.get("target_ref")
+            if s in groups and t in ext and ext[t] in graph.terms:
+                recs.setdefault(groups[s], set()).add(ext[t])
+
+    records = {g: ts for g, ts in recs.items() if MIN_TERMS <= len(ts) <= MAX_TERMS}
+    return mounted, records

sma/eval/agentic/arms/discovery.py

@@ -0,0 +1,35 @@
+"""Discovery arm: gene-function retrieval over the Gene Ontology (GO).
+
+Entities are human proteins; each is annotated with its set of GO biological-process
+terms (gold, from the GOA GAF). The hard query is a partial/imprecise functional
+profile; the task is to retrieve the right protein by its function signature. GO's
+is-a tree is the ascension lattice and its part_of/regulates relations become
+higher-order statements.
+"""
+from __future__ import annotations
+
+import pathlib
+
+from sma.ontology import load_obo, mount
+
+ROOT = pathlib.Path(__file__).resolve().parents[4]
+GO_OBO = ROOT / "data/raw/obo/go-basic.obo"
+GAF = ROOT / "data/raw/go/goa_human.gaf"
+ASPECT = "P"            # biological process
+MIN_TERMS, MAX_TERMS = 7, 30
+
+
+def load():
+    mounted = mount(load_obo(str(GO_OBO), name="go"))
+    known = mounted.graph.terms
+    recs: dict[str, set[str]] = {}
+    for line in GAF.open():
+        if line.startswith("!"):
+            continue
+        p = line.rstrip("\n").split("\t")
+        if len(p) < 9 or p[8] != ASPECT:
+            continue
+        if p[4] in known:
+            recs.setdefault(p[1], set()).add(p[4])      # protein -> GO term
+    records = {k: v for k, v in recs.items() if MIN_TERMS <= len(v) <= MAX_TERMS}
+    return mounted, records

sma/eval/agentic/arms/finance.py

@@ -0,0 +1,38 @@
+"""Finance arm: SEC filing classification over the US-GAAP reporting taxonomy.
+
+Entities are SEC filings (accession numbers); each is annotated with the set of
+US-GAAP concepts it reports (gold, from the SEC Financial Statement Data Sets'
+num.txt). The hard query is a partial/imprecise set of reported concepts climbed
+up the statement hierarchy plus noise; the task is to retrieve the filing by its
+financial-reporting signature. The US-GAAP presentation hierarchy (statement
+headers subsume line items) is the is-a lattice (FIBO, a schema, has no instance
+corpus, so US-GAAP provides the financial entity->concept gold).
+"""
+from __future__ import annotations
+
+import pathlib
+
+from sma.ontology import mount
+from sma.ontology.usgaap import load_usgaap
+
+ROOT = pathlib.Path(__file__).resolve().parents[4]
+USGAAP_DIR = ROOT / "data/raw/finance/usgaap"
+NUM = ROOT / "data/raw/finance/sec_2024q1/num.txt"
+MIN_TERMS, MAX_TERMS = 10, 40
+
+
+def load():
+    mounted = mount(load_usgaap(str(USGAAP_DIR), name="usgaap"))
+    known = mounted.graph.terms
+    recs: dict[str, set[str]] = {}
+    with NUM.open(encoding="utf-8", errors="ignore") as fh:
+        header = fh.readline().rstrip("\n").split("\t")
+        ia, it, iv = header.index("adsh"), header.index("tag"), header.index("version")
+        for line in fh:
+            p = line.rstrip("\n").split("\t")
+            if len(p) <= iv or not p[iv].startswith("us-gaap"):
+                continue
+            if p[it] in known:
+                recs.setdefault(p[ia], set()).add(p[it])
+    records = {k: v for k, v in recs.items() if MIN_TERMS <= len(v) <= MAX_TERMS}
+    return mounted, records

sma/eval/agentic/arms/legal.py

@@ -0,0 +1,74 @@
+"""Legal/IP arm: patent classification over the Cooperative Patent Classification.
+
+Entities are granted patents; each is annotated with the set of CPC subgroup codes
+its examiner assigned (gold, from one week of USPTO grant full-text XML). The hard
+query is a partial/imprecise technical profile (a few codes, climbed up the CPC
+hierarchy, plus noise); the task is to retrieve the patent by its classification
+signature. CPC's deep section->class->subclass->group->subgroup tree is the is-a
+lattice (254k nodes), mounted via sma.ontology.load_cpc.
+"""
+from __future__ import annotations
+
+import pathlib
+import re
+import xml.etree.ElementTree as ET
+
+from sma.ontology import load_cpc, mount
+
+ROOT = pathlib.Path(__file__).resolve().parents[4]
+CPC_DIR = ROOT / "data/raw/legal/cpc"
+GRANT_XML = ROOT / "data/raw/patents/ipg161011.xml"
+MIN_TERMS, MAX_TERMS = 7, 30
+
+
+def _grant_blocks(path):
+    """Yield each <us-patent-grant>...</us-patent-grant> block (the file is many
+    concatenated XML documents, so we can't parse it as one tree)."""
+    buf, inside = [], False
+    with open(path, "r", encoding="utf-8", errors="ignore") as fh:
+        for line in fh:
+            if line.startswith("<us-patent-grant"):
+                inside, buf = True, [line]
+            elif inside:
+                buf.append(line)
+                if line.startswith("</us-patent-grant>"):
+                    yield "".join(buf)
+                    inside, buf = False, []
+
+
+def _cpc_codes(grant_el) -> set[str]:
+    """CPC subgroup codes from the bibliographic block only (not cited references)."""
+    block = grant_el.find(".//us-bibliographic-data-grant/classifications-cpc")
+    if block is None:
+        return set()
+    codes = set()
+    for c in block.iter("classification-cpc"):
+        try:
+            sec = c.findtext("section", "").strip()
+            cls = c.findtext("class", "").strip()
+            sub = c.findtext("subclass", "").strip()
+            mg = c.findtext("main-group", "").strip()
+            sg = c.findtext("subgroup", "").strip()
+        except AttributeError:
+            continue
+        if sec and cls and sub and mg and sg:
+            codes.add(f"{sec}{cls}{sub}{mg}/{sg}")
+    return codes
+
+
+def load():
+    mounted = mount(load_cpc(str(CPC_DIR), name="cpc"))
+    known = mounted.graph.terms
+    recs: dict[str, set[str]] = {}
+    for block in _grant_blocks(GRANT_XML):
+        try:
+            g = ET.fromstring(block)
+        except ET.ParseError:
+            continue
+        num = g.findtext(".//publication-reference//doc-number")
+        if not num:
+            continue
+        codes = {c for c in _cpc_codes(g) if c in known}
+        if MIN_TERMS <= len(codes) <= MAX_TERMS:
+            recs[f"US{num}"] = codes
+    return mounted, recs

sma/eval/agentic/arms/medicine.py

@@ -0,0 +1,45 @@
+"""Medicine arm: HPO ontology + rare-disease phenotype records.
+
+``load()`` mounts the Human Phenotype Ontology and parses ``phenotype.hpoa`` into
+``disease_id -> {hpo_term_id}`` records, restricted to phenotypic-abnormality
+(aspect ``P``) annotations of diseases carrying 7..30 phenotypes -- the same
+record-construction used by ``scripts/bench_ontology_suite.load_hpo_records`` and
+the 7..30 eligibility band of ``sma.eval.ontology_bench.run_arm``.
+"""
+
+from __future__ import annotations
+
+import pathlib
+
+from sma.ontology import MountedOntology, load_obo, mount
+
+ROOT = pathlib.Path(__file__).resolve().parents[4]
+HP_OBO = ROOT / "data/raw/hpo/hp.obo"
+HPOA = ROOT / "data/raw/hpo/phenotype.hpoa"
+
+MIN_TERMS = 7
+MAX_TERMS = 30
+
+
+def load_hpo_records(path: pathlib.Path = HPOA) -> dict[str, set[str]]:
+    """Parse ``phenotype.hpoa`` into ``disease_id -> {hpo_term_id}`` (aspect P).
+
+    Skips header/comment lines, keeps only phenotypic-abnormality annotations
+    (column 10 == ``"P"``), and retains diseases with 7..30 phenotypes.
+    """
+    rec: dict[str, set[str]] = {}
+    for line in path.open():
+        if line.startswith(("#", "database_id")):
+            continue
+        p = line.rstrip("\n").split("\t")
+        if len(p) < 11 or p[10] != "P":
+            continue
+        rec.setdefault(p[0], set()).add(p[3])
+    return {d: terms for d, terms in rec.items() if MIN_TERMS <= len(terms) <= MAX_TERMS}
+
+
+def load() -> tuple[MountedOntology, dict[str, set[str]]]:
+    """Return the mounted HPO ontology and its disease->phenotype records."""
+    mounted = mount(load_obo(str(HP_OBO), name="hpo"))
+    records = load_hpo_records()
+    return mounted, records

sma/eval/agentic/harness.py

@@ -0,0 +1,275 @@
+"""One-shot agentic harness: swap the :class:`Memory`, hold everything else fixed.
+
+``run_oneshot`` builds one benchmark from an arm ``(mounted ontology, entity ->
+term-set records)`` and scores every memory identically:
+
+* a deterministic ``holdout_frac`` of entities is reserved as NOVEL (their
+  queries are unanswerable and feed the abstain/novelty metrics); only the rest
+  is indexed in every memory;
+* hard queries are generated for both answerable and novel entities (sample a
+  few terms, climb 0-2 is-a levels for imprecision, add noise terms);
+* each memory retrieves the true key's rank, a confidence, and a novelty signal;
+* metrics: tail top-k (answerable; all + rare slice), risk-coverage AURC
+  (answerable), novelty F1 over ALL queries (novelty > 0.5 vs truly novel);
+* primary stat: SMA vs the best enterprise-RAG memory (best by tail top-5) on
+  per-query top-5 correctness via paired bootstrap.
+
+Determinism: every set is sorted to a list before use and every RNG is
+explicitly seeded, so identical ``seeds`` yield identical result dicts.
+"""
+
+from __future__ import annotations
+
+import math
+import random
+import statistics
+from typing import Iterable
+
+from sma.eval.agentic.memories import IndexItem, Memory, Query
+from sma.eval.agentic.metrics import (
+    ABSENT_RANK,
+    novelty_f1,
+    risk_coverage_aurc,
+    tail_topk,
+)
+from sma.eval.stats import cliffs_delta, paired_bootstrap
+from sma.ontology import MountedOntology
+
+# Enterprise-RAG/KG gauntlet — SMA's primary comparison is the best of these.
+ENTERPRISE_NAMES = ("bm25", "dense", "hybrid_rrf", "hybrid_rerank", "hippo")
+
+NOVELTY_THRESHOLD = 0.5  # fixed (not per-method tuned); noted caveat in the spec.
+
+
+# --- ontology IC machinery (closure-propagated term frequency) -------------
+# Mirrors sma/eval/ontology_bench.py so the "rare" slice is defined identically.
+def _ancestors(term: str, parents: dict[str, tuple[str, ...]], cache: dict[str, set]) -> set:
+    if term in cache:
+        return cache[term]
+    # Iterative closure with a visited set: cycle-safe (some ontologies, e.g. CPC
+    # built from repeated scheme symbols, contain is-a cycles) and immune to the
+    # recursion limit on deep hierarchies. Same result as the recursive form on
+    # acyclic graphs.
+    acc: set[str] = set()
+    stack = list(parents.get(term, ()))
+    while stack:
+        p = stack.pop()
+        if p in acc or p == term:
+            continue
+        acc.add(p)
+        stack.extend(parents.get(p, ()))
+    cache[term] = acc
+    return acc
+
+
+def _build_ic(
+    entity_terms: list[set[str]],
+    parents: dict[str, tuple[str, ...]],
+    anc_cache: dict[str, set],
+) -> dict[str, float]:
+    """Information content per term via closure-propagated frequency."""
+    n = len(entity_terms)
+    freq: dict[str, int] = {}
+    for terms in entity_terms:
+        clo = set(terms)
+        for t in terms:
+            clo |= _ancestors(t, parents, anc_cache)
+        for t in clo:
+            freq[t] = freq.get(t, 0) + 1
+    return {t: -math.log(c / n) for t, c in freq.items()} if n else {}
+
+
+def run_oneshot(
+    name: str,
+    mounted: MountedOntology,
+    records: dict[str, set[str]],
+    memories: list[Memory],
+    *,
+    seeds: Iterable[int] = (7, 17, 23),
+    n_index: int = 2000,
+    n_query: int = 120,
+    holdout_frac: float = 0.1,
+) -> dict:
+    """Run the one-shot agentic benchmark and return a result dict.
+
+    ``records`` maps ``entity_id -> set(term_id)``. Returns
+    ``{"arm", "memories", "n_all", "n_rare", "n_novel", "per_memory", "primary"}``
+    where ``per_memory[name]`` carries tail top-k (all + rare slices), AURC, and
+    novelty F1, and ``primary`` is the SMA-vs-best-enterprise paired bootstrap.
+    """
+    graph = mounted.graph
+    parents = {tid: tuple(t.parents) for tid, t in graph.terms.items()}
+
+    def term_text(t: str) -> str:
+        nm = graph.terms[t].name if t in graph.terms else ""
+        return nm or t
+
+    mem_names = [m.name for m in memories]
+
+    # Eligible entities: those with at least one known term (SORTED for determinism).
+    eligible = sorted(
+        eid
+        for eid, terms in records.items()
+        if any(t in graph.terms for t in terms)
+    )
+
+    # Pooled per-query rows across seeds. Each row holds every memory's rank plus
+    # confidence/novelty/flags for that query.
+    answerable_rows: list[dict] = []  # rank rows for tail_topk on answerable queries
+    per_mem: dict[str, dict[str, list]] = {
+        m: {"ans_conf": [], "ans_correct": [], "all_pred_novel": [], "all_is_novel": []}
+        for m in mem_names
+    }
+    # Per-query top-5 correctness on ALL queries (answerable + novel) for the
+    # paired bootstrap. A novel query is "correct" only if the memory abstains
+    # via novelty; for top-5 retrieval correctness a novel query is always a miss.
+    top5_ans: dict[str, list[float]] = {m: [] for m in mem_names}
+
+    n_novel_total = 0
+
+    for seed in seeds:
+        rng = random.Random(seed)
+        ids = list(eligible)
+        rng.shuffle(ids)
+        pool = ids[:n_index]
+
+        # Deterministic NOVEL holdout: SORTED -> shuffle -> slice.
+        pool_sorted = sorted(pool)
+        rng.shuffle(pool_sorted)
+        n_holdout = int(round(len(pool_sorted) * holdout_frac))
+        novel_ids = sorted(pool_sorted[:n_holdout])
+        index_ids = sorted(pool_sorted[n_holdout:])
+
+        # Indexed term-sets (known terms only, SORTED to lists).
+        dz = {e: sorted(t for t in records[e] if t in graph.terms) for e in index_ids}
+        dz_novel = {e: sorted(t for t in records[e] if t in graph.terms) for e in novel_ids}
+
+        # IC + rare threshold over the INDEXED records only.
+        anc_cache: dict[str, set] = {}
+        ic = _build_ic([set(v) for v in dz.values()], parents, anc_cache)
+        median_ic = statistics.median(ic.values()) if ic else 0.0
+        noise_pool = sorted(ic) or sorted({t for v in dz.values() for t in v})
+
+        # Build IndexItems and index every memory (identical input).
+        items = [
+            IndexItem(
+                key=e,
+                term_ids=frozenset(dz[e]),
+                text=" ".join(term_text(t) for t in dz[e]),
+                meta={"id": e},
+            )
+            for e in index_ids
+        ]
+        for mem in memories:
+            mem.index(items)
+
+        # Query specs: hard partial/imprecise observations for answerable AND novel.
+        def make_qspec(terms: list[str]) -> list[str]:
+            keep = rng.sample(terms, min(5, len(terms)))
+            q: list[str] = []
+            for t in keep:
+                cur = t
+                for _ in range(rng.choice([0, 0, 1, 1, 2])):
+                    ps = parents.get(cur)
+                    if ps:
+                        cur = rng.choice(sorted(ps))
+                q.append(cur)
+            if noise_pool:
+                q += rng.sample(noise_pool, min(3, len(noise_pool)))
+            return q
+
+        # Allocate the n_query budget between answerable and novel entities,
+        # preserving the holdout proportion. Both kinds get hard queries.
+        ans_candidates = [e for e in index_ids if dz[e]]
+        nov_candidates = [e for e in novel_ids if dz_novel[e]]
+        n_nov = min(len(nov_candidates), int(round(n_query * holdout_frac)))
+        n_ans = min(len(ans_candidates), n_query - n_nov)
+        ans_q = ans_candidates[:n_ans]
+        nov_q = nov_candidates[:n_nov]
+        n_novel_total += len(nov_q)
+
+        qspecs: list[tuple[str, list[str], bool]] = []
+        for e in ans_q:
+            qspecs.append((e, make_qspec(dz[e]), False))
+        for e in nov_q:
+            qspecs.append((e, make_qspec(dz_novel[e]), True))
+
+        for e, qterms, is_novel in qspecs:
+            query = Query(
+                term_ids=frozenset(qterms),
+                text=" ".join(term_text(t) for t in qterms),
+            )
+            rare = (
+                max((ic.get(t, 0.0) for t in (dz[e] if not is_novel else dz_novel[e])), default=0.0)
+                > median_ic
+            )
+            rank_row = {"rare": rare}
+            for mem in memories:
+                res = mem.retrieve(query, k=10)
+                rank = next((r.rank for r in res if r.key == e), ABSENT_RANK)
+                if is_novel:
+                    rank = ABSENT_RANK  # unanswerable: true key is not indexed
+                conf = res[0].confidence if res else 0.0
+                nov = mem.novelty(query)
+
+                rank_row[mem.name] = rank
+                top5_ans[mem.name].append(1.0 if (not is_novel and rank <= 5) else 0.0)
+                per_mem[mem.name]["all_pred_novel"].append(nov > NOVELTY_THRESHOLD)
+                per_mem[mem.name]["all_is_novel"].append(is_novel)
+                if not is_novel:
+                    correct = rank <= 5
+                    per_mem[mem.name]["ans_conf"].append(conf)
+                    per_mem[mem.name]["ans_correct"].append(correct)
+
+            if not is_novel:
+                answerable_rows.append(rank_row)
+
+    # --- aggregate ---------------------------------------------------------
+    tail5 = tail_topk(answerable_rows, k=5)
+    tail1 = tail_topk(answerable_rows, k=1)
+    tail10 = tail_topk(answerable_rows, k=10)
+
+    per_memory: dict[str, dict] = {}
+    for m in mem_names:
+        aurc, _curve = risk_coverage_aurc(
+            per_mem[m]["ans_conf"], per_mem[m]["ans_correct"]
+        )
+        f1 = novelty_f1(per_mem[m]["all_pred_novel"], per_mem[m]["all_is_novel"])
+        per_memory[m] = {
+            "tail": {
+                "top1": tail1.get(m, {"all": 0.0, "rare": 0.0}),
+                "top5": tail5.get(m, {"all": 0.0, "rare": 0.0}),
+                "top10": tail10.get(m, {"all": 0.0, "rare": 0.0}),
+            },
+            "aurc": aurc,
+            "novelty_f1": f1,
+        }
+
+    # --- primary: SMA vs best enterprise-RAG on per-query top-5 correctness --
+    primary: dict | None = None
+    present_enterprise = [m for m in ENTERPRISE_NAMES if m in tail5]
+    if "sma" in tail5 and present_enterprise:
+        best = max(present_enterprise, key=lambda m: tail5[m]["all"])
+        a = top5_ans["sma"]
+        b = top5_ans[best]
+        bs = paired_bootstrap(a, b)
+        primary = {
+            "a": "sma",
+            "b": best,
+            "best_enterprise": best,
+            "delta_top5": bs["delta"],
+            "ci_low": bs["ci_low"],
+            "ci_high": bs["ci_high"],
+            "p_value": bs["p_value"],
+            "cliffs": cliffs_delta(a, b),
+        }
+
+    return {
+        "arm": name,
+        "memories": mem_names,
+        "n_all": len(answerable_rows),
+        "n_rare": sum(1 for r in answerable_rows if r["rare"]),
+        "n_novel": n_novel_total,
+        "per_memory": per_memory,
+        "primary": primary,
+    }