structuremappingmemory 1.0.0__py3-none-any.whl

sma/eval/ssb_eval.py

@@ -0,0 +1,216 @@
+"""SSB retrieval evaluations."""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+
+from sma.index.macfac import MacFacIndex
+from sma.index.content_vectors import functor_vector, cosine
+from sma.match.types import MatchConfig
+from sma.eval.ssb_generator import build_canonicalizer
+
+# SSB matching crosses disjoint vocabularies bridged only by the generated
+# lattice: delta=2 reaches the shared concept; rho=0.95 is a pre-calibration
+# default (section 8.6 fits it properly), high enough that a full deep system
+# at rho^2 penalty still dominates a flat same-vocabulary distractor.
+def ssb_config() -> MatchConfig:
+    return MatchConfig(delta=2, rho=0.95)
+from sma.ir.sexpr import canonical_case_text
+from sma.eval.baselines.bm25 import rank_bm25_like
+from sma.eval.baselines.dense import rank_tfidf_dense, rank_tfidf_dense_batch
+from sma.eval.ssb_generator import SSBTriple, generate_triples
+
+
+@dataclass(frozen=True)
+class RetrievalEval:
+    name: str
+    rows: list[dict]
+    metrics: dict
+    latency: dict
+    # Per-query analog ranks (0 = not retrieved), in triple order; additive
+    # field used by scripts/confirmatory_battery.py for paired statistics.
+    ranks: tuple[int, ...] = ()
+
+
+def evaluate_forced_choice(n: int = 12, seed: int = 11) -> RetrievalEval:
+    triples = generate_triples(n, seed=seed)
+    rows: list[dict] = []
+    ranks: list[int] = []
+    start = time.perf_counter()
+    for i, triple in enumerate(triples):
+        index = MacFacIndex(config=ssb_config(), canon=build_canonicalizer([triple]))
+        index.build([triple.analog, triple.distractor])
+        results = index.retrieve(triple.query, k=2, shortlist=2)
+        rank = rank_of(results, triple.analog.case_id)
+        ranks.append(rank)
+        rows.extend(result_rows("ssb_forced_choice", triple.query.case_id, results))
+    elapsed = (time.perf_counter() - start) * 1000
+    return RetrievalEval(
+        name="forced_choice_fixture",
+        rows=rows,
+        metrics=rank_metrics("forced_choice_fixture", ranks, n),
+        latency={"operation": "ssb_forced_choice_macfac", "n_cases": n * 2, "p50_ms": elapsed, "p95_ms": elapsed},
+        ranks=tuple(ranks),
+    )
+
+
+def evaluate_library(
+    n: int = 100,
+    seed: int = 19,
+    k: int = 10,
+    shortlist: int | None = None,
+    fac_budget: int | None = None,
+) -> dict:
+    triples = generate_triples(n, seed=seed)
+    library_cases = []
+    documents = []
+    for triple in triples:
+        library_cases.extend([triple.analog, triple.distractor])
+        documents.append((triple.analog.case_id, canonical_case_text(triple.analog.statements)))
+        documents.append((triple.distractor.case_id, canonical_case_text(triple.distractor.statements)))
+
+    index = MacFacIndex(config=ssb_config(), canon=build_canonicalizer(triples))
+    index.build(library_cases)
+    sma_rows: list[dict] = []
+    sma_ranks: list[int] = []
+    bm25_ranks: list[int] = []
+    dense_ranks: list[int] = []
+    start = time.perf_counter()
+    shortlist = shortlist or len(library_cases)
+
+    query_texts = [canonical_case_text(triple.query.statements) for triple in triples]
+    dense_rankings = rank_tfidf_dense_batch(query_texts, documents, k=k)
+
+    for triple, query_text, dense in zip(triples, query_texts, dense_rankings):
+        sma_results = index.retrieve(triple.query, k=k, shortlist=shortlist, fac_budget=fac_budget)
+        sma_rows.extend(result_rows(f"ssb_library_{n}", triple.query.case_id, sma_results))
+        sma_ranks.append(rank_of(sma_results, triple.analog.case_id))
+
+        bm25 = rank_bm25_like(query_text, documents, k=k)
+        bm25_ranks.append(rank_of_pairs(bm25, triple.analog.case_id))
+        dense_ranks.append(rank_of_pairs(dense, triple.analog.case_id))
+
+    elapsed = (time.perf_counter() - start) * 1000
+    return {
+        "sma_rows": sma_rows,
+        # Additive: per-query analog ranks (0 = miss) in triple order, plus
+        # the matching query ids, for paired SMA-vs-baseline statistics.
+        "query_ids": [triple.query.case_id for triple in triples],
+        "ranks": {
+            "SMA": sma_ranks,
+            "BM25": bm25_ranks,
+            "TFIDF-Dense": dense_ranks,
+        },
+        "metrics": [
+            rank_metrics(f"ssb_library_{n}_sma", sma_ranks, n),
+            rank_metrics(f"ssb_library_{n}_bm25", bm25_ranks, n),
+            rank_metrics(f"ssb_library_{n}_tfidf_dense", dense_ranks, n),
+        ],
+        "latency": {
+            "operation": f"ssb_library_{n}_all_baselines_fac_budget_{fac_budget or 'unbounded'}",
+            "n_cases": len(library_cases),
+            "p50_ms": elapsed,
+            "p95_ms": elapsed,
+        },
+    }
+
+
+def evaluate_library_mac_prefilter(n: int = 1000, seed: int = 23, k: int = 10) -> dict:
+    """Fast large-library MAC-stage diagnostic.
+
+    This does not replace certified FAC. It answers whether candidate
+    generation places a structurally compatible analog into the top-k shortlist
+    before expensive SME matching.
+    """
+
+    triples = generate_triples(n, seed=seed)
+    library = []
+    documents = []
+    for triple in triples:
+        library.extend([triple.analog, triple.distractor])
+        documents.append((triple.analog.case_id, canonical_case_text(triple.analog.statements)))
+        documents.append((triple.distractor.case_id, canonical_case_text(triple.distractor.statements)))
+    canon = build_canonicalizer(triples)
+    vectors = {case.case_id: functor_vector(case, canon=canon, delta=2) for case in library}
+
+    sma_ranks: list[int] = []
+    bm25_ranks: list[int] = []
+    dense_ranks: list[int] = []
+    rows: list[dict] = []
+    start = time.perf_counter()
+    query_texts = [canonical_case_text(triple.query.statements) for triple in triples]
+    dense_rankings = rank_tfidf_dense_batch(query_texts, documents, k=k)
+
+    for triple, query_text, dense in zip(triples, query_texts, dense_rankings):
+        qv = functor_vector(triple.query, canon=canon, delta=2)
+        ranked = sorted(
+            ((case.case_id, cosine(qv, vectors[case.case_id])) for case in library),
+            key=lambda row: (-row[1], row[0]),
+        )[:k]
+        sma_ranks.append(rank_of_pairs(ranked, triple.analog.case_id))
+        for rank, (case_id, score) in enumerate(ranked, start=1):
+            rows.append(
+                {
+                    "run_id": f"ssb_library_{n}_mac_prefilter",
+                    "query_id": triple.query.case_id,
+                    "rank": rank,
+                    "case_id": case_id,
+                    "score": f"{score:.6f}",
+                    "ses_n": "",
+                    "u_bound": "",
+                    "certified": False,
+                }
+            )
+
+        bm25 = rank_bm25_like(query_text, documents, k=k)
+        bm25_ranks.append(rank_of_pairs(bm25, triple.analog.case_id))
+        dense_ranks.append(rank_of_pairs(dense, triple.analog.case_id))
+    elapsed = (time.perf_counter() - start) * 1000
+    return {
+        "sma_rows": rows,
+        "metrics": [
+            rank_metrics(f"ssb_library_{n}_mac_prefilter", sma_ranks, n),
+            rank_metrics(f"ssb_library_{n}_bm25", bm25_ranks, n),
+            rank_metrics(f"ssb_library_{n}_tfidf_dense", dense_ranks, n),
+        ],
+        "latency": {
+            "operation": f"ssb_library_{n}_mac_prefilter_all_baselines",
+            "n_cases": len(library),
+            "p50_ms": elapsed,
+            "p95_ms": elapsed,
+        },
+    }
+
+
+def result_rows(run_id: str, query_id: str, results) -> list[dict]:
+    return [
+        {
+            "run_id": run_id,
+            "query_id": query_id,
+            "rank": rank,
+            "case_id": result.case_id,
+            "score": f"{result.score:.6f}",
+            "ses_n": f"{result.ses_n:.6f}",
+            "u_bound": f"{result.u_bound:.6f}",
+            "certified": result.certified,
+        }
+        for rank, result in enumerate(results, start=1)
+    ]
+
+
+def rank_of(results, case_id: str) -> int:
+    return next((rank for rank, result in enumerate(results, start=1) if result.case_id == case_id), 0)
+
+
+def rank_of_pairs(results: list[tuple[str, float]], case_id: str) -> int:
+    return next((rank for rank, (result_id, _score) in enumerate(results, start=1) if result_id == case_id), 0)
+
+
+def rank_metrics(split: str, ranks: list[int], total: int) -> dict:
+    return {
+        "split": split,
+        "r1": f"{sum(1 for rank in ranks if rank == 1) / total:.4f}",
+        "mrr": f"{sum((1 / rank) if rank else 0.0 for rank in ranks) / total:.4f}",
+        "mapping_f1": "1.0000" if split == "forced_choice_fixture" else "",
+    }

sma/eval/ssb_generator.py

@@ -0,0 +1,116 @@
+"""Synthetic Structural Benchmark generator (de-circularized).
+
+Each triple is (query, analog, distractor) with full gold correspondences:
+
+- query: a seeded relational schema with its own functor vocabulary;
+- analog: the SAME structure under a DISJOINT functor vocabulary and renamed
+  entities - zero lexical overlap with the query. The two vocabularies are
+  bridged ONLY by a declared predicate lattice (each query functor and its
+  analog counterpart share an abstract parent concept), so matching requires
+  minimal ascension (delta >= 2) at the rho^dist penalty. No string trick
+  (the old far_-prefix bijection was known to the canonicalizer - circular);
+- distractor: the query's own vocabulary (matched content vector) with the
+  relational structure rewired - same words, broken structure.
+
+build_canonicalizer(triples) returns the Canonicalizer carrying the lattice;
+evaluations MUST use it together with a delta>=2 MatchConfig, otherwise
+analogs are unreachable by construction.
+"""
+
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass
+
+from sma.ir.canon import Canonicalizer
+from sma.ir.schema import Case, Statement, entity, make_case, stmt
+
+
+@dataclass(frozen=True)
+class SSBTriple:
+    query: Case
+    analog: Case
+    distractor: Case
+    gold: dict[str, str]
+    lattice_pairs: tuple[tuple[str, str], ...]  # (child_functor, parent_concept)
+
+
+def generate_triples(n: int = 100, seed: int = 13) -> list[SSBTriple]:
+    rng = random.Random(seed)
+    return [generate_triple(rng, i) for i in range(n)]
+
+
+def _fresh_name(rng: random.Random, prefix: str) -> str:
+    return f"{prefix}{rng.randrange(1 << 24):06x}"
+
+
+def generate_triple(rng: random.Random, idx: int) -> SSBTriple:
+    depth = rng.randint(2, 4)
+    width = rng.randint(2, 4)
+    # Functor slots for this schema; each slot gets a query name and an analog
+    # name drawn from disjoint random pools, joined only through a concept.
+    slots = [f"rel{i}" for i in range(width - 1)] + [f"ho{layer}" for layer in range(1, depth)]
+    q_name: dict[str, str] = {}
+    a_name: dict[str, str] = {}
+    lattice_pairs: list[tuple[str, str]] = []
+    for slot in slots:
+        q = _fresh_name(rng, f"q{idx}")
+        a = _fresh_name(rng, f"a{idx}")
+        concept = f"c{idx}_{slot}"
+        q_name[slot], a_name[slot] = q, a
+        lattice_pairs.append((q, concept))
+        lattice_pairs.append((a, concept))
+
+    query = schema_case(f"q{idx}", depth, width, q_name)
+    analog = schema_case(f"a{idx}", depth, width, a_name)
+    distractor = rewire_case(query, f"d{idx}", q_name)
+    gold = {f"E:{e.name}": f"E:{e.name.replace(f'q{idx}', f'a{idx}')}" for e in query.entities()}
+    return SSBTriple(query=query, analog=analog, distractor=distractor, gold=gold,
+                     lattice_pairs=tuple(lattice_pairs))
+
+
+def schema_case(prefix: str, depth: int, width: int, names: dict[str, str]) -> Case:
+    entities = [entity(f"{prefix}_e{i}") for i in range(width)]
+    base_rels: list[Statement] = []
+    for i in range(width - 1):
+        base_rels.append(stmt(names[f"rel{i}"], entities[i], entities[i + 1]))
+    current = base_rels
+    statements = list(base_rels)
+    for layer in range(1, depth):
+        next_layer: list[Statement] = []
+        for i in range(max(1, len(current) - 1)):
+            relation = stmt(names[f"ho{layer}"], current[i], current[min(i + 1, len(current) - 1)])
+            next_layer.append(relation)
+            statements.append(relation)
+        current = next_layer
+    return make_case(statements, {"adapter": "ssb", "tier": 0})
+
+
+def rewire_case(case: Case, prefix: str, names: dict[str, str]) -> Case:
+    """Same vocabulary as the query, broken structure.
+
+    The base relations form a STAR (every relation points into one hub
+    entity) instead of the query's chain. A chain has no entity with
+    in-degree >= 2, a star does, so for width >= 3 the two are provably
+    non-isomorphic under ordered relations - the old (i, i+2 mod n)
+    rewiring could reproduce the chain up to relabeling at small widths
+    (the matcher then correctly scored the 'distractor' 1.0)."""
+    ents = [entity(f"{prefix}_e{i}") for i, _ in enumerate(case.entities())]
+    statements: list[Statement] = []
+    functors = [s.functor for s in case.statements
+                if s.arity == 2 and not any(isinstance(a, Statement) for a in s.args)]
+    hub = ents[-1]
+    for i, functor in enumerate(functors):
+        statements.append(stmt(functor, ents[i % max(len(ents) - 1, 1)], hub))
+    if len(statements) >= 2 and "ho1" in names:
+        statements.append(stmt(names["ho1"], statements[-1], statements[0]))
+    return make_case(statements or [stmt("empty", entity(prefix))], {"adapter": "ssb", "tier": 0})
+
+
+def build_canonicalizer(triples: list[SSBTriple]) -> Canonicalizer:
+    """Canonicalizer whose lattice is the ONLY bridge between vocabularies."""
+    canon = Canonicalizer()
+    for triple in triples:
+        for child, parent in triple.lattice_pairs:
+            canon.lattice.add(child, parent)
+    return canon

sma/eval/stats.py

@@ -0,0 +1,108 @@
+"""Pre-registered statistics for the confirmatory battery (prereg section 5).
+
+Per-query paired bootstrap (10,000 resamples) for SMA-vs-baseline deltas
+with 95% percentile CIs, Holm-Bonferroni step-down correction within each
+dataset's family of baseline comparisons, and Cliff's delta as the effect
+size. Everything is deterministic: the bootstrap uses an explicitly seeded
+numpy Generator and no global RNG state.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+DEFAULT_RESAMPLES = 10_000
+DEFAULT_SEED = 12345
+
+# Resample index matrices are drawn in chunks so a 10k-resample bootstrap
+# over thousands of pooled queries stays within a few tens of MB.
+_CHUNK = 1_000
+
+
+def paired_bootstrap(
+    a: list[float],
+    b: list[float],
+    n_resamples: int = DEFAULT_RESAMPLES,
+    seed: int = DEFAULT_SEED,
+) -> dict:
+    """Paired bootstrap of mean(a - b) over per-query scores.
+
+    ``a`` and ``b`` are per-query scores for two methods on the SAME queries
+    (paired by position). Returns::
+
+        {"delta":   observed mean(a - b),
+         "ci_low":  2.5th percentile of the bootstrap distribution,
+         "ci_high": 97.5th percentile,
+         "p_value": two-sided bootstrap p for delta != 0}
+
+    The p-value is the doubled smaller tail of the bootstrap distribution
+    around zero, with a +1/(R+1) correction so it is never exactly 0.
+    """
+    a_arr = np.asarray(a, dtype=float)
+    b_arr = np.asarray(b, dtype=float)
+    if a_arr.ndim != 1 or b_arr.ndim != 1:
+        raise ValueError("paired_bootstrap expects 1-D score lists")
+    if a_arr.shape != b_arr.shape:
+        raise ValueError(
+            f"paired scores must have equal length (got {a_arr.size} vs {b_arr.size})"
+        )
+    if a_arr.size == 0:
+        raise ValueError("paired_bootstrap requires at least one paired observation")
+    if n_resamples < 1:
+        raise ValueError("n_resamples must be >= 1")
+
+    diffs = a_arr - b_arr
+    rng = np.random.default_rng(seed)
+    deltas = np.empty(n_resamples, dtype=float)
+    done = 0
+    while done < n_resamples:
+        size = min(_CHUNK, n_resamples - done)
+        idx = rng.integers(0, diffs.size, size=(size, diffs.size))
+        deltas[done : done + size] = diffs[idx].mean(axis=1)
+        done += size
+
+    ci_low, ci_high = np.percentile(deltas, [2.5, 97.5])
+    p_low = (np.count_nonzero(deltas <= 0.0) + 1) / (n_resamples + 1)
+    p_high = (np.count_nonzero(deltas >= 0.0) + 1) / (n_resamples + 1)
+    return {
+        "delta": float(diffs.mean()),
+        "ci_low": float(ci_low),
+        "ci_high": float(ci_high),
+        "p_value": float(min(1.0, 2.0 * min(p_low, p_high))),
+    }
+
+
+def holm_bonferroni(p_values: dict[str, float]) -> dict[str, float]:
+    """Holm step-down adjusted p-values, keyed like the input.
+
+    Sort the m raw p-values ascending; the i-th (1-based) is multiplied by
+    (m - i + 1), running maxima enforce monotonicity, and everything is
+    capped at 1.0. Ties are processed in sorted (p, key) order, which does
+    not affect the adjusted values.
+    """
+    m = len(p_values)
+    adjusted: dict[str, float] = {}
+    running = 0.0
+    for i, (key, p) in enumerate(sorted(p_values.items(), key=lambda kv: (kv[1], kv[0]))):
+        if not 0.0 <= p <= 1.0:
+            raise ValueError(f"p-value for {key!r} outside [0, 1]: {p}")
+        running = max(running, (m - i) * p)
+        adjusted[key] = min(1.0, running)
+    return adjusted
+
+
+def cliffs_delta(a: list[float], b: list[float]) -> float:
+    """Standard Cliff's delta in [-1, 1]: P(a > b) - P(a < b) over all pairs.
+
+    +1 means every a exceeds every b; -1 the reverse; 0 means stochastic
+    equality. Computed via sorted ranks (O((n+m) log m)), so pooled
+    multi-seed score lists are fine.
+    """
+    a_arr = np.asarray(a, dtype=float)
+    b_arr = np.asarray(b, dtype=float)
+    if a_arr.size == 0 or b_arr.size == 0:
+        raise ValueError("cliffs_delta requires non-empty score lists")
+    b_sorted = np.sort(b_arr)
+    n_b_below = np.searchsorted(b_sorted, a_arr, side="left").sum()
+    n_b_above = (b_arr.size - np.searchsorted(b_sorted, a_arr, side="right")).sum()
+    return float((int(n_b_below) - int(n_b_above)) / (a_arr.size * b_arr.size))