memtrust-cli 0.3.0__py3-none-any.whl

memtrust/evals/result_consistency.py

@@ -0,0 +1,239 @@
+"""MemTrust's repeated-query determinism eval.
+
+Every other eval in this package classifies a SINGLE query() response, or
+compares two responses separated by some other event (a crash, a resync,
+a migration). None of them can see a backend that is internally
+non-deterministic: each individual response can look perfectly well-formed
+-- real records, real content, no error -- while the *set* of records
+returned for the exact same query, against data that never changed,
+varies from call to call.
+
+Motivating case: volcengine/OpenViking#204 (contributor ponsde, a repeat
+contributor to this project). `search()`/`find()` returned non-
+deterministic result sets for identical repeated queries: 5 runs of the
+same query returned an average pairwise Jaccard similarity of 0.11 over
+the returned memory/resource URIs, and a later, more rigorous 3-query x
+5-run test found some query/method combinations sharing ZERO common URIs
+across all 5 runs. ponsde self-diagnosed two candidate root causes through
+extensive hands-on follow-up: an embedding-dimension mismatch (a
+production vector collection created at `Dimension:3072` while the active
+embedding config specified `1024`), and/or non-deterministic graph
+traversal inside the HNSW ANN search implementation itself. This eval
+reuses ponsde's own methodology directly: issue the identical query N
+times against unchanged fixture data, and compute the average pairwise
+(consecutive-run) Jaccard similarity over the returned record ids.
+
+See ConsistencySignal in adapters/base.py for the taxonomy this eval
+scores against.
+
+Honest limitation (same convention as every other eval's module docstring
+in this package): this eval can only measure what it can observe through
+`query()` -- it has no access to OpenViking's internal ANN index state or
+embedding-config metadata, so it cannot itself distinguish ponsde's two
+candidate root causes from each other, or from an unrelated source of
+non-determinism (e.g. a backend that intentionally randomizes tie-breaks).
+It proves the harness can *detect* the same result-instability shape
+ponsde documented, using his own methodology, against any adapter pointed
+at it -- not that any specific root cause is present.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from memtrust.adapters.base import BackendAPIError, ConsistencySignal, MemoryBackendAdapter
+
+DEFAULT_FIXTURE_PATH = (
+    Path(__file__).resolve().parents[3] / "tests" / "fixtures" / "result_consistency_cases.json"
+)
+
+#: Default number of times each case's query is repeated. ponsde's own
+#: repro used 5 runs per query; kept as this eval's default for direct
+#: comparability with his reported numbers.
+DEFAULT_N_REPEATS = 5
+
+#: Default minimum average pairwise Jaccard similarity for a case to be
+#: classified CONSISTENT. Not 1.0 (perfect overlap every run): a
+#: genuinely ranked-and-truncated top_k result can reasonably reorder
+#: near a score tie without that being a real bug -- this tolerance is
+#: deliberately generous compared to ponsde's own measured 0.11 average
+#: for the broken case, so a backend has to be substantially unstable to
+#: fail this threshold.
+DEFAULT_CONSISTENCY_THRESHOLD = 0.9
+
+
+@dataclass
+class ConsistencySeedRecord:
+    content: str
+    metadata: dict[str, str] = field(default_factory=dict)
+
+
+@dataclass
+class ConsistencyCase:
+    case_id: str
+    session_id: str
+    query: str
+    records: list[ConsistencySeedRecord]
+    n_repeats: int = DEFAULT_N_REPEATS
+    threshold: float = DEFAULT_CONSISTENCY_THRESHOLD
+
+
+@dataclass
+class ConsistencyCaseResult:
+    case: ConsistencyCase
+    signal: ConsistencySignal
+    average_jaccard: float | None
+    """Average pairwise (consecutive-run) Jaccard similarity over the
+    result-id sets from each successful query() call. None when fewer
+    than 2 calls succeeded."""
+    n_successful_runs: int
+    result_id_sets: list[list[str]]
+    """Each successful run's returned record-id set, in call order, kept
+    for audit purposes -- e.g. to see exactly which ids appeared/vanished
+    between runs."""
+    error: str | None = None
+
+
+@dataclass
+class ConsistencyEvalResult:
+    backend_name: str
+    dataset_path: str
+    case_results: list[ConsistencyCaseResult] = field(default_factory=list)
+
+    @property
+    def scored_cases(self) -> list[ConsistencyCaseResult]:
+        return [c for c in self.case_results if c.error is None]
+
+    def _fraction(self, signal: ConsistencySignal) -> float | None:
+        scored = self.scored_cases
+        if not scored:
+            return None
+        matching = sum(1 for c in scored if c.signal == signal)
+        return matching / len(scored)
+
+    @property
+    def consistent_rate(self) -> float | None:
+        return self._fraction(ConsistencySignal.CONSISTENT)
+
+    @property
+    def inconsistent_rate(self) -> float | None:
+        """Fraction of cases whose repeated, identical query against
+        unchanged data produced meaningfully different result sets --
+        the headline metric this eval exists to surface, and the exact
+        volcengine/OpenViking#204 shape."""
+        return self._fraction(ConsistencySignal.INCONSISTENT)
+
+
+def load_dataset(path: Path | str = DEFAULT_FIXTURE_PATH) -> list[ConsistencyCase]:
+    data = json.loads(Path(path).read_text())
+    cases: list[dict[str, Any]] = data["cases"]
+    return [
+        ConsistencyCase(
+            case_id=c["case_id"],
+            session_id=c["session_id"],
+            query=c["query"],
+            n_repeats=c.get("n_repeats", DEFAULT_N_REPEATS),
+            threshold=c.get("threshold", DEFAULT_CONSISTENCY_THRESHOLD),
+            records=[
+                ConsistencySeedRecord(content=r["content"], metadata=r.get("metadata", {}))
+                for r in c["records"]
+            ],
+        )
+        for c in cases
+    ]
+
+
+def _jaccard(a: set[str], b: set[str]) -> float:
+    """Jaccard similarity between two id sets. Two empty sets are treated
+    as perfectly similar (1.0) -- "returned nothing, twice" is a
+    consistency-neutral result for THIS metric (a separate eval,
+    evals/contradiction.py's EMPTY_OR_LOST handling, already covers
+    "returned nothing when it should not have")."""
+    if not a and not b:
+        return 1.0
+    union = a | b
+    if not union:
+        return 1.0
+    return len(a & b) / len(union)
+
+
+def classify_consistency_case(
+    result_id_sets: list[set[str]],
+    threshold: float = DEFAULT_CONSISTENCY_THRESHOLD,
+) -> tuple[ConsistencySignal, float | None]:
+    """Classify a case's outcome from its list of per-run result-id sets,
+    using ponsde's own consecutive-pair Jaccard-similarity methodology
+    (volcengine/OpenViking#204's own hand-built repro script compares run
+    i against run i+1, not every pair against every other pair).
+
+    Returns (signal, average_jaccard). average_jaccard is None when fewer
+    than 2 runs are available to form a pair -- NOT_APPLICABLE in that
+    case, never guessed at either way.
+    """
+    if len(result_id_sets) < 2:
+        return ConsistencySignal.NOT_APPLICABLE, None
+    pairwise = [
+        _jaccard(result_id_sets[i], result_id_sets[i + 1]) for i in range(len(result_id_sets) - 1)
+    ]
+    average = sum(pairwise) / len(pairwise)
+    if average >= threshold:
+        return ConsistencySignal.CONSISTENT, average
+    return ConsistencySignal.INCONSISTENT, average
+
+
+def _result_ids(records: list[Any]) -> set[str]:
+    """Prefer memory_id when the adapter reports a real one; fall back to
+    the record's own content as an id-surrogate for adapters/fixtures
+    where memory_id can be empty on some responses -- same
+    "id-first, content-fallback" convention verify_store() already uses
+    in adapters/base.py."""
+    ids: set[str] = set()
+    for record in records:
+        ids.add(record.memory_id if record.memory_id else f"content:{record.content}")
+    return ids
+
+
+def run_result_consistency_eval(
+    adapter: MemoryBackendAdapter,
+    dataset_path: Path | str = DEFAULT_FIXTURE_PATH,
+) -> ConsistencyEvalResult:
+    cases = load_dataset(dataset_path)
+    result = ConsistencyEvalResult(backend_name=adapter.name, dataset_path=str(dataset_path))
+
+    for case in cases:
+        try:
+            for seed in case.records:
+                adapter.store(case.session_id, seed.content, metadata=seed.metadata)
+
+            result_id_sets: list[set[str]] = []
+            for _ in range(case.n_repeats):
+                query_result = adapter.query(case.session_id, case.query, top_k=len(case.records))
+                result_id_sets.append(_result_ids(query_result.records))
+        except BackendAPIError as exc:
+            result.case_results.append(
+                ConsistencyCaseResult(
+                    case=case,
+                    signal=ConsistencySignal.NOT_APPLICABLE,
+                    average_jaccard=None,
+                    n_successful_runs=0,
+                    result_id_sets=[],
+                    error=str(exc),
+                )
+            )
+            continue
+
+        signal, average = classify_consistency_case(result_id_sets, case.threshold)
+        result.case_results.append(
+            ConsistencyCaseResult(
+                case=case,
+                signal=signal,
+                average_jaccard=average,
+                n_successful_runs=len(result_id_sets),
+                result_id_sets=[sorted(s) for s in result_id_sets],
+            )
+        )
+
+    return result

memtrust/evals/scale_fixtures.py

@@ -0,0 +1,189 @@
+"""Deterministic large-scale synthetic corpus generator for
+evals/scale_stress.py.
+
+Every other fixture in this repo (`tests/fixtures/*.json`) is a hand-written
+file with 3-7 examples -- fine for exercising contradiction/ranking/
+resource-sync logic, but 3-5 orders of magnitude below the ~100K-record or
+~300-episode scale where several real, documented vendor bugs actually
+manifest:
+
+  * volcengine/OpenViking#2850 (lg320531124, still open): BM25 search
+    silently returning empty results once a corpus grows large. A
+    hand-written 5-record fixture structurally cannot reach the corpus size
+    where this shows up.
+  * getzep/graphiti#1275 (rafaelreis-r): O(n) entity-resolution context
+    growth causing episodes to be silently dropped once ingestion passes
+    roughly 300 episodes. Same problem -- no bundled fixture has 300 of
+    anything.
+
+A hand-written JSON file cannot solve this: nobody is going to hand-type
+10,000 realistic-looking memory facts into a fixture file, and even if they
+did, it would not be reproducible/adjustable the way a parameterized
+generator is. This module is a function, not a file -- `generate_scale_corpus
+(n, seed=...)` produces N synthetic records with realistic-shaped content,
+deterministically, for any N a caller asks for (5 or 100,000), so
+evals/scale_stress.py can be pointed at whatever scale a real run needs
+without checking a giant file into git.
+
+Design notes:
+  * Determinism is via `random.Random(seed)` -- the same (n, seed) pair
+    always produces byte-identical output, which is what makes a scale run
+    reproducible across two separate `memtrust run` invocations.
+  * Every record's content embeds a unique, greppable marker token
+    (`SCALEMARK{index:06d}`) that appears nowhere else in the corpus. This
+    is deliberate: it turns "is this specific record still findable" into a
+    literal substring/keyword search, the same shape of query BM25-style
+    lexical search actually serves, and the shape #2850 concerns (a literal
+    keyword search coming back empty). A record's `marker` field is what
+    evals/scale_stress.py queries for when it wants to test recall of one
+    specific record.
+  * The generated content is deliberately fact-shaped prose (a name, a
+    topic, an action, a detail), not lorem-ipsum noise or a bare marker
+    string -- so a backend that does real embedding/tokenization work has
+    something realistic to index, not a degenerate one-token input that
+    would never occur in a real ingestion pipeline.
+"""
+
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass, field
+
+#: Format for each record's unique, greppable marker token. Six digits
+#: comfortably covers every N this module is meant to be used at (up to
+#: 999,999); a caller requesting more than that gets a ValueError rather
+#: than silently colliding marker tokens.
+_MARKER_FORMAT = "SCALEMARK{index:06d}"
+_MAX_RECORDS = 999_999
+
+_FIRST_NAMES = (
+    "Priya",
+    "Jordan",
+    "Sam",
+    "Alex",
+    "Morgan",
+    "Taylor",
+    "Casey",
+    "Jamie",
+    "Riley",
+    "Devon",
+    "Nadia",
+    "Kenji",
+    "Elena",
+    "Marcus",
+    "Ines",
+    "Tobias",
+    "Ling",
+    "Omar",
+    "Freya",
+    "Dante",
+)
+_TOPICS = (
+    "the quarterly budget review",
+    "the onboarding checklist",
+    "the API rate limit increase",
+    "the deployment pipeline",
+    "the customer escalation queue",
+    "the on-call rotation",
+    "the vendor security questionnaire",
+    "the data retention policy",
+    "the staging environment refresh",
+    "the incident postmortem",
+    "the roadmap planning doc",
+    "the support ticket backlog",
+    "the pricing experiment",
+    "the migration runbook",
+    "the compliance audit",
+)
+_ACTIONS = (
+    "moved to",
+    "was updated to reference",
+    "got flagged for follow-up regarding",
+    "was rescheduled around",
+    "now depends on",
+    "was assigned to",
+    "was blocked by",
+    "was reprioritized after",
+    "was archived following",
+    "needs another look because of",
+)
+_DETAILS = (
+    "Thursday afternoon",
+    "the new staging environment",
+    "ticket ORT-4471",
+    "the Q3 roadmap",
+    "last week's incident review",
+    "the new compliance requirement",
+    "a customer escalation from EMEA",
+    "the vendor's revised SLA",
+    "the platform team's capacity",
+    "next sprint's planning session",
+)
+
+
+@dataclass
+class ScaleFixtureRecord:
+    """One synthetic record in a generated scale corpus.
+
+    `index` is this record's 0-based position in generation order --
+    evals/scale_stress.py uses it as the "insertion order" ground truth,
+    the same role `RankingQualitySeedRecord`'s list position plays in
+    evals/ranking_quality.py. `marker` is the unique greppable token
+    embedded in `content`; querying for it (rather than for arbitrary
+    natural-language text) is what turns "is this record still
+    recoverable" into a deterministic, unambiguous substring check.
+    """
+
+    index: int
+    content: str
+    marker: str
+    metadata: dict[str, str] = field(default_factory=dict)
+
+
+def generate_scale_corpus(
+    n: int, seed: int = 42, session_id: str = "scale-stress-session"
+) -> list[ScaleFixtureRecord]:
+    """Generate `n` deterministic, realistic-shaped synthetic records.
+
+    Same (n, seed) always produces the same output -- this is what lets a
+    scale run be reproduced exactly (e.g. to confirm a degradation finding
+    wasn't a one-off fluke) without checking a giant fixture file into git.
+    Architected to scale from a handful of records (a fast CI smoke test)
+    up to 10K+ (a real stress run against a live backend); the only cost of
+    a larger N is the loop below, there is no fixed-size data file to
+    outgrow.
+
+    Args:
+        n: how many records to generate. Must be >= 1 and <= 999,999 (see
+            `_MAX_RECORDS` -- the marker format's fixed width).
+        seed: random seed. Same seed -> byte-identical corpus.
+        session_id: stamped into each record's metadata for callers that
+            want it; scale_stress.py's own session scoping is independent
+            of this (it passes its own session_id to store()/query()).
+
+    Raises:
+        ValueError: if `n` is out of the supported range.
+    """
+    if n < 1:
+        raise ValueError(f"n must be >= 1, got {n}")
+    if n > _MAX_RECORDS:
+        raise ValueError(f"n must be <= {_MAX_RECORDS}, got {n}")
+
+    rng = random.Random(seed)
+    records: list[ScaleFixtureRecord] = []
+    for index in range(n):
+        name = rng.choice(_FIRST_NAMES)
+        topic = rng.choice(_TOPICS)
+        action = rng.choice(_ACTIONS)
+        detail = rng.choice(_DETAILS)
+        marker = _MARKER_FORMAT.format(index=index)
+        content = f"On day {index}, {name} noted that {topic} {action} {detail}. (ref: {marker})"
+        records.append(
+            ScaleFixtureRecord(
+                index=index,
+                content=content,
+                marker=marker,
+                metadata={"scale_index": str(index), "session_id": session_id},
+            )
+        )
+    return records