memtrust-cli 0.3.0__py3-none-any.whl

memtrust/evals/stats_accuracy.py

@@ -0,0 +1,240 @@
+"""MemTrust's stats/dashboard-accuracy eval.
+
+Every other eval in this package treats a backend's own store()/query()
+responses as the only source of truth about what is actually stored (with
+appropriate independent cross-checks -- see crash_recovery.py's
+raw_store_contains(), resource_sync_safety.py's list_resource_paths()).
+None of them look at a backend's *self-reported summary statistics* at
+all. This eval closes that specific gap: it checks whether a backend's own
+"how many memories do you have" endpoint agrees with an independently
+verified count of what is actually retrievable.
+
+Motivating case: volcengine/OpenViking#1255 (contributor SeeYangZhi).
+`GET /api/v1/stats/memories` returned an all-zero count
+(`{"total_memories": 0, "by_category": {...all zero...}, ...}`) on a fresh
+instance immediately after session-commit-triggered memory extraction had
+already completed -- confirmed still present via two independent
+observations quoted in the bug report itself: Docker logs showing
+"Extracted 10 candidate memories" / "Created memory file: ...", and a
+`POST /v1/search/find` call that successfully returned those same memory
+files with their extracted abstracts. The reporter's own root-cause read:
+the stats endpoint reads from a dedicated stats/metadata index populated
+only by a synchronous "content/write" code path, never by the async
+extraction pipeline that actually wrote the memories -- so the counter and
+the real data structurally diverge, with no exception anywhere to signal
+the mismatch.
+
+This eval reproduces that shape generically: store N records, independently
+verify how many of them are actually retrievable via the adapter's normal
+query() path (the same surface #1255's own report used `/v1/search/find`
+for), then call get_stats() and compare its self-reported count against
+that independently verified number.
+
+**Why a new StatsSignal enum rather than a new ConflictSignal member.**
+`ConflictSignal.EMPTY_OR_LOST` (adapters/base.py) fires when a single
+query() call succeeds but returns zero records -- it is about *retrieval*
+silently coming back empty. This eval's failure mode is different in kind:
+query() itself may work perfectly (as it did in #1255's own report --
+`/v1/search/find` DID return the memories), while a *separate,
+independently-maintained* stats/dashboard endpoint reports a number that
+disagrees with that working retrieval path. Folding this into
+ConflictSignal would blur "retrieval failed" and "retrieval works, but a
+different, unrelated counter is wrong" into the same signal -- the same
+"a cross-call/cross-endpoint comparison is a structurally different kind
+of classification than a single QueryResult" reasoning that already
+justifies CrashRecoverySignal, ResourceSyncSignal, and RankingSignal each
+being their own enum (see their docstrings in adapters/base.py and
+evals/crash_recovery.py).
+
+**Honest scope.** Like every other eval added without a live, configured
+vendor instance available (see evals/scale_stress.py, evals/
+embedding_drift.py, docs/methodology.md), this eval's classification logic
+is proven correct against purpose-built fake adapters in
+tests/test_evals.py -- one reproducing #1255's exact "always reports zero
+regardless of real stored count" shape, one reporting an accurate count as
+the negative control. It has not been run against a live OpenViking
+instance; running it there requires OPENVIKING_API_KEY (or a self-hosted
+OPENVIKING_BASE_URL) to be configured, which is not the case in the
+environment this eval was built in.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+from uuid import uuid4
+
+from memtrust.adapters.base import BackendAPIError, MemoryBackendAdapter
+
+#: Small by default (matches every other bundled fixture's single-digit
+#: scale, see scale_stress.py's DEFAULT_N_RECORDS docstring for why that
+#: is a deliberate, fast-in-CI choice) -- this eval only needs enough
+#: records to tell "the endpoint reports a real number" apart from "the
+#: endpoint always reports zero/undercounts," not to exercise scale.
+DEFAULT_N_RECORDS = 5
+DEFAULT_SESSION_ID = "stats-accuracy-session"
+
+
+class StatsSignal(StrEnum):
+    """Whether a backend's self-reported stats/dashboard count agrees with
+    an independently verified count of what is actually retrievable.
+
+    Defined locally in this module rather than in adapters/base.py,
+    following the same precedent evals/scale_stress.py's ScaleSignal and
+    evals/resource_sync_safety.py's ResourceSyncSignal already set: this is
+    a harness-computed classification derived from ground truth (records
+    this eval itself stored and independently re-verified), not a signal
+    any adapter self-reports.
+    """
+
+    STATS_MATCH = "stats_match"
+    """get_stats()'s reported count is greater than or equal to the
+    independently verified retrievable count. The good outcome -- the
+    stats endpoint is not undercounting relative to what this eval could
+    confirm is actually there. (A report strictly higher than verified is
+    still STATS_MATCH, not a separate "overcounted" signal -- this eval
+    exists to catch #1255's undercounting shape specifically; a backend
+    reporting MORE than this eval's own narrow verification found has no
+    known real-world motivating bug case to justify inventing a
+    classification for it, and would likely just mean the backend's own
+    count includes records this eval's own query-based verification
+    under-samples for unrelated reasons.)"""
+
+    STATS_UNDERCOUNTED = "stats_undercounted"
+    """get_stats()'s reported count is strictly less than the independently
+    verified retrievable count -- the exact volcengine/OpenViking#1255
+    shape: the memories are demonstrably there (this eval could retrieve
+    them via the normal query() path) but the dedicated stats/dashboard
+    endpoint reports fewer of them, including the reported #1255 case of
+    reporting zero when the real count is nonzero."""
+
+    NOT_APPLICABLE = "not_applicable"
+    """Either the adapter has no stats endpoint at all
+    (MemoryBackendAdapter.supports_stats is False -- the eval is skipped,
+    not run), or nothing was successfully stored/verified in the first
+    place, so there is no independently verified count to compare
+    get_stats() against."""
+
+
+@dataclass
+class StatsAccuracyEvalResult:
+    backend_name: str
+    n_records_requested: int
+    records_stored: int = 0
+    verified_count: int | None = None
+    """Independently verified count of the stored records that are
+    actually retrievable via adapter.query() -- the ground truth this
+    eval compares get_stats() against. `None` if nothing was ever stored
+    successfully."""
+    reported_count: int | None = None
+    """adapter.get_stats().total_memories, verbatim -- the number under
+    test. `None` if get_stats() reported no usable count."""
+    signal: StatsSignal = StatsSignal.NOT_APPLICABLE
+    skipped: bool = False
+    skip_reason: str | None = None
+    error: str | None = None
+
+    @property
+    def undercount_gap(self) -> int | None:
+        """verified_count - reported_count, i.e. how many retrievable
+        records the stats endpoint failed to count. `None` unless both
+        counts are known. Never negative in practice under STATS_MATCH
+        (see StatsSignal.STATS_MATCH's docstring on why an over-report
+        is not itself flagged), but not clamped here -- this is a raw
+        diagnostic value, not the classification itself."""
+        if self.verified_count is None or self.reported_count is None:
+            return None
+        return self.verified_count - self.reported_count
+
+
+def classify_stats_accuracy(verified_count: int | None, reported_count: int | None) -> StatsSignal:
+    """Classify one run's outcome from its two independently obtained
+    counts. Never trusts get_stats() alone -- the same "never trust a
+    single response" principle every eval in this package follows (see
+    evals/crash_recovery.py's classify_crash_recovery_case and
+    evals/resource_sync_safety.py's classify_resource_sync_file).
+    """
+    if verified_count is None or reported_count is None:
+        return StatsSignal.NOT_APPLICABLE
+    if reported_count < verified_count:
+        return StatsSignal.STATS_UNDERCOUNTED
+    return StatsSignal.STATS_MATCH
+
+
+def _verify_stored_count(
+    adapter: MemoryBackendAdapter, session_id: str, markers: list[str], top_k: int
+) -> int:
+    """Independently confirm how many of `markers` are actually
+    retrievable via the adapter's normal query() path -- the same surface
+    #1255's own bug report used (`POST /v1/search/find`) to prove the
+    memories existed despite the stats endpoint reporting zero.
+
+    Never trusts "query() didn't raise" as proof a record came back --
+    each marker must actually appear in the joined content of the
+    response, the same rule evals/scale_stress.py's _query_needle applies.
+    """
+    found = 0
+    for marker in markers:
+        try:
+            query_result = adapter.query(session_id, marker, top_k=top_k)
+        except BackendAPIError:
+            continue
+        if any(marker in record.content for record in query_result.records):
+            found += 1
+    return found
+
+
+def run_stats_accuracy_eval(
+    adapter: MemoryBackendAdapter,
+    n_records: int = DEFAULT_N_RECORDS,
+    session_id: str = DEFAULT_SESSION_ID,
+    top_k: int = 5,
+) -> StatsAccuracyEvalResult:
+    """Store `n_records` uniquely-markered records, independently verify
+    how many are actually retrievable via query(), then compare that
+    ground truth against adapter.get_stats()'s self-reported count.
+
+    Args:
+        adapter: the backend under test.
+        n_records: how many synthetic records to store and verify.
+        session_id: session/scope every record is stored under.
+        top_k: top_k passed to each verification query() call.
+    """
+    result = StatsAccuracyEvalResult(backend_name=adapter.name, n_records_requested=n_records)
+
+    if not adapter.supports_stats:
+        result.skipped = True
+        result.skip_reason = (
+            f"{adapter.name} does not implement get_stats() (supports_stats=False) -- "
+            "skipped, not run. See adapters/base.py's MemoryBackendAdapter.get_stats() "
+            "and evals/stats_accuracy.py's module docstring."
+        )
+        return result
+
+    markers = [f"stats-accuracy-marker-{i}-{uuid4().hex[:8]}" for i in range(n_records)]
+    stored_markers: list[str] = []
+    for index, marker in enumerate(markers):
+        try:
+            adapter.store(session_id, f"Fact #{index}: unique marker {marker}.")
+            stored_markers.append(marker)
+            result.records_stored += 1
+        except BackendAPIError as exc:
+            if result.error is None:
+                result.error = f"store failed for marker {marker}: {exc}"
+
+    if not stored_markers:
+        result.signal = StatsSignal.NOT_APPLICABLE
+        return result
+
+    result.verified_count = _verify_stored_count(adapter, session_id, stored_markers, top_k)
+
+    try:
+        stats = adapter.get_stats(session_id)
+    except BackendAPIError as exc:
+        result.error = f"get_stats failed: {exc}"
+        result.signal = StatsSignal.NOT_APPLICABLE
+        return result
+
+    result.reported_count = stats.total_memories
+    result.signal = classify_stats_accuracy(result.verified_count, result.reported_count)
+    return result

memtrust/evals/temporal_kg_boundary.py

@@ -0,0 +1,281 @@
+"""memtrust's temporal-KG as_of point-in-time boundary eval for
+MemPalaceAdapter's `kg_add()`/`kg_invalidate()`/`kg_query()` primitives.
+
+Motivating case: MemPalace/mempalace#1913, fixed by merged PR#1914
+(contributor ggettert; `gh pr view 1914 --repo MemPalace/mempalace
+--comments`). `mempalace/knowledge_graph.py`'s `_temporal_filter_sql` used a
+*closed* interval on both ends (`valid_from <= as_of AND valid_to >=
+as_of`), so a fact whose `valid_to` equals the query's `as_of` instant still
+matched. When a caller hand-rolls a fact change as `kg_invalidate(ended=T)`
+immediately followed by `kg_add(valid_from=T)` at the identical boundary
+instant `T` -- the exact pattern MemPalace's own pre-fix `PALACE_PROTOCOL`
+wake-up guidance told every agent to do -- an `as_of=T` query matched BOTH
+the old, just-ended fact and the new, just-started fact simultaneously, so a
+single-valued predicate (e.g. "uses_model") reported two contradictory
+values at once with no error or warning. PR#1914's real fix switched the
+upper-bound comparison to strict (`valid_to > as_of`), i.e. a half-open
+interval `[valid_from, valid_to)`: a fact that ended exactly at the query
+instant no longer matches, so only its successor does. PR#1914 also added a
+new `supersede()` primitive (`mempalace_kg_supersede`) as the *preferred*
+way to make this kind of change atomically -- this eval deliberately does
+NOT use it, because `MemPalaceAdapter` does not wire `tool_kg_supersede` at
+all (only `tool_kg_add`/`tool_kg_invalidate`/`tool_kg_query` are confirmed
+real and wired -- see mempalace_adapter.py's module docstring). This eval
+exercises exactly the failure-prone hand-rolled pattern PR#1914's own
+PALACE_PROTOCOL fix warns callers away from, because that pattern is what
+`MemPalaceAdapter`'s current, narrower capability surface actually produces
+-- adding `kg_supersede` support to this adapter is a separate, later
+change, out of scope here.
+
+Scope note -- what this eval is and is not. This is a genuinely scoped
+fix for ONE of the two capabilities a prior backlog item bundled together
+("Add MemPalace temporal-KG as_of point-in-time queries" +
+"drawer neighbor-expansion scoping"). This eval covers only the temporal-KG
+as_of half. Drawer neighbor-expansion/parent_drawer_id-leak scoping is a
+separate, larger, still-deferred capability this eval does not touch.
+
+Honest scope on live verification (same convention every eval module in
+this package states plainly, see docs/methodology.md): the real `mempalace`
+PyPI package is not installed in this build environment, and PR#1914 itself
+had not shipped in a released `mempalace` version as of this adapter's
+"live-verified at version 3.5.0" build (the fix lands under CHANGELOG.md's
+`## [Unreleased]` section, above the `## [3.5.0]` entry -- confirmed by
+reading the merged PR's own diff). This eval's own test suite
+(tests/test_temporal_kg_boundary.py) therefore proves the *classification
+logic* is correct against two from-scratch fake `_MCPToolsProtocol`
+implementations that reproduce the real, confirmed pre-#1914 (closed-
+interval) and post-#1914 (half-open) `_temporal_filter_sql` comparison
+exactly -- not against a live MemPalace instance running either version.
+
+Design:
+
+  1. `kg_add(subject, predicate, old_object, valid_from=<seed>)` -- seed the
+     fact this case will supersede.
+  2. `kg_invalidate(subject, predicate, old_object, ended=<boundary>)` --
+     end it at a known instant.
+  3. `kg_add(subject, predicate, new_object, valid_from=<boundary>)` -- the
+     hand-rolled handover, sharing the exact same boundary instant as (2).
+  4. `kg_query(subject, as_of=<boundary>, direction="outgoing")` -- query at
+     the exact shared instant.
+  5. Classify the response two ways: `MemPalaceAdapter.kg_query()`'s own
+     self-reported `boundary_signal` (see adapters/base.py's
+     `TemporalBoundarySignal` and mempalace_adapter.py's
+     `_classify_boundary_signal`), and this eval's own, independently
+     written `classify_temporal_kg_boundary_case()` below, which re-derives
+     the verdict directly from the raw `KGFact` list `kg_query()` returned
+     (how many distinct objects exist for the queried `(subject,
+     predicate)`). The eval's own classification -- never the adapter's
+     self-report -- is what `TemporalKGBoundaryCaseResult.signal` records;
+     `adapter_reported_signal` is kept only for comparison/transparency,
+     the same "claim, not proof" convention evals/ranking_quality.py's
+     `classify_ranking_case()` already establishes for
+     `RankingSignal.SIGNAL_DRIVEN`.
+"""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+
+from memtrust.adapters.base import BackendAPIError, TemporalBoundarySignal
+from memtrust.adapters.mempalace_adapter import KGQueryResult, MemPalaceAdapter
+
+
+@dataclass
+class TemporalKGBoundaryCase:
+    case_id: str
+    subject: str
+    predicate: str
+    old_object: str
+    """Value of the fact before the boundary."""
+    new_object: str
+    """Value of the fact from the boundary onward -- the hand-rolled
+    successor, added via kg_add(valid_from=boundary), sharing the exact
+    same instant as the kg_invalidate(ended=boundary) call that closed
+    old_object."""
+    seed_valid_from: str
+    """When old_object's fact starts being true, well before `boundary`."""
+    boundary: str
+    """The single shared instant used for BOTH kg_invalidate(ended=...)
+    and the successor kg_add(valid_from=...), and then queried via
+    kg_query(as_of=...) -- the exact MemPalace/mempalace#1913 shape."""
+
+
+def default_cases(subject_prefix: str | None = None) -> list[TemporalKGBoundaryCase]:
+    """One case exercising the exact PR#1914 boundary shape. `subject` is
+    randomized by default (uuid4-suffixed) -- see mempalace_adapter.py's
+    module docstring "KNOWLEDGE-GRAPH STORAGE IGNORES MEMPALACE_PALACE_PATH"
+    section: the real KG store is one fixed, environment-global file with
+    no per-run isolation, so a fixed literal subject would make this case's
+    assertions invalid against any prior run's leftover facts (the same
+    randomization tests/test_adapters.py's own
+    `test_real_mempalace_kg_add_invalidate_query_round_trip` already uses).
+    """
+    subject = subject_prefix or f"memtrust-temporal-kg-boundary-{uuid.uuid4().hex}"
+    return [
+        TemporalKGBoundaryCase(
+            case_id="mt-tkb-001",
+            subject=subject,
+            predicate="uses_model",
+            old_object="claude-opus-4-7",
+            new_object="claude-opus-4-8",
+            seed_valid_from="2026-05-01T00:00:00Z",
+            boundary="2026-06-02T12:00:00Z",
+        ),
+    ]
+
+
+@dataclass
+class TemporalKGBoundaryCaseResult:
+    case: TemporalKGBoundaryCase
+    signal: TemporalBoundarySignal
+    """This eval's own, independently-derived verdict -- see
+    `classify_temporal_kg_boundary_case()` below. Never copied straight
+    from the adapter's self-report."""
+    adapter_reported_signal: TemporalBoundarySignal | None
+    """`KGQueryResult.boundary_signal` as MemPalaceAdapter.kg_query()
+    itself reported it, kept for comparison/transparency only. `None` when
+    the call sequence failed before a query response ever came back."""
+    objects_at_boundary: list[str]
+    """Every distinct `object` value returned for `(case.subject,
+    case.predicate)` at `case.boundary` -- the evidence `signal` was
+    computed from. Length 2+ is the literal double-count."""
+    error: str | None = None
+
+    @property
+    def self_report_agrees(self) -> bool:
+        return self.adapter_reported_signal == self.signal
+
+
+@dataclass
+class TemporalKGBoundaryEvalResult:
+    backend_name: str
+    case_results: list[TemporalKGBoundaryCaseResult] = field(default_factory=list)
+
+    @property
+    def scored_cases(self) -> list[TemporalKGBoundaryCaseResult]:
+        return [r for r in self.case_results if r.error is None]
+
+    def _fraction(self, signal: TemporalBoundarySignal) -> float | None:
+        scored = self.scored_cases
+        if not scored:
+            return None
+        return sum(1 for r in scored if r.signal == signal) / len(scored)
+
+    @property
+    def double_count_rate(self) -> float | None:
+        """Fraction of cases where an as_of query at the exact boundary
+        instant returned both the superseded fact and its successor -- the
+        headline metric this eval exists to surface, and the one that
+        would flag MemPalace/mempalace#1913's exact shape against a
+        pre-#1914 deployment."""
+        return self._fraction(TemporalBoundarySignal.DOUBLE_COUNT)
+
+    @property
+    def clean_rate(self) -> float | None:
+        return self._fraction(TemporalBoundarySignal.CLEAN)
+
+    @property
+    def not_applicable_rate(self) -> float | None:
+        return self._fraction(TemporalBoundarySignal.NOT_APPLICABLE)
+
+    @property
+    def self_report_agreement_rate(self) -> float | None:
+        """Fraction of scored cases where MemPalaceAdapter.kg_query()'s own
+        self-reported boundary_signal matched this eval's independently
+        derived verdict -- a low rate here would mean the adapter's
+        self-classification (adapters/mempalace_adapter.py's
+        `_classify_boundary_signal`) itself has a bug, distinct from
+        whether the *backend* double-counts."""
+        scored = self.scored_cases
+        if not scored:
+            return None
+        return sum(1 for r in scored if r.self_report_agrees) / len(scored)
+
+
+def classify_temporal_kg_boundary_case(
+    case: TemporalKGBoundaryCase, query_result: KGQueryResult
+) -> tuple[TemporalBoundarySignal, list[str]]:
+    """Independently classify one case's outcome straight from the raw
+    `KGFact` list `query_result.facts` carries -- deliberately NOT calling
+    or trusting `query_result.boundary_signal` (the adapter's own
+    self-report; see TemporalKGBoundarySignal in adapters/base.py for why
+    an eval must never treat a signal field as the final answer on its
+    own). Returns (signal, distinct_objects_returned).
+
+    Only facts matching this case's exact `(subject, predicate)` on the
+    `outgoing` direction are considered -- `kg_query()` was called with
+    `direction="outgoing"`, so every returned fact should already match,
+    but this filters defensively rather than assuming the vendor response
+    never carries an unrelated fact.
+    """
+    matching = [
+        f
+        for f in query_result.facts
+        if f.direction == "outgoing" and f.subject == case.subject and f.predicate == case.predicate
+    ]
+    objects = sorted({f.object for f in matching})
+    if not matching:
+        return TemporalBoundarySignal.NOT_APPLICABLE, objects
+    if len(objects) >= 2:
+        return TemporalBoundarySignal.DOUBLE_COUNT, objects
+    return TemporalBoundarySignal.CLEAN, objects
+
+
+def run_temporal_kg_boundary_eval(
+    adapter: MemPalaceAdapter,
+    cases: list[TemporalKGBoundaryCase] | None = None,
+) -> TemporalKGBoundaryEvalResult:
+    resolved_cases = cases if cases is not None else default_cases()
+    result = TemporalKGBoundaryEvalResult(backend_name=adapter.name)
+
+    for case in resolved_cases:
+        try:
+            add_old = adapter.kg_add(
+                case.subject, case.predicate, case.old_object, valid_from=case.seed_valid_from
+            )
+            if not add_old.success:
+                raise BackendAPIError(
+                    adapter.name, add_old.error or "kg_add(old_object) reported failure"
+                )
+
+            invalidate = adapter.kg_invalidate(
+                case.subject, case.predicate, case.old_object, ended=case.boundary
+            )
+            if not invalidate.success:
+                raise BackendAPIError(
+                    adapter.name, invalidate.error or "kg_invalidate() reported failure"
+                )
+
+            add_new = adapter.kg_add(
+                case.subject, case.predicate, case.new_object, valid_from=case.boundary
+            )
+            if not add_new.success:
+                raise BackendAPIError(
+                    adapter.name, add_new.error or "kg_add(new_object) reported failure"
+                )
+
+            query_result = adapter.kg_query(case.subject, as_of=case.boundary, direction="outgoing")
+        except BackendAPIError as exc:
+            result.case_results.append(
+                TemporalKGBoundaryCaseResult(
+                    case=case,
+                    signal=TemporalBoundarySignal.NOT_APPLICABLE,
+                    adapter_reported_signal=None,
+                    objects_at_boundary=[],
+                    error=str(exc),
+                )
+            )
+            continue
+
+        signal, objects = classify_temporal_kg_boundary_case(case, query_result)
+        result.case_results.append(
+            TemporalKGBoundaryCaseResult(
+                case=case,
+                signal=signal,
+                adapter_reported_signal=query_result.boundary_signal,
+                objects_at_boundary=objects,
+            )
+        )
+
+    return result