memtrust-cli 0.3.0__py3-none-any.whl

memtrust/evals/ranking_quality.py

@@ -0,0 +1,303 @@
+"""MemTrust's ranking/relevance-quality eval.
+
+`evals/contradiction.py`'s `ConflictSignal` taxonomy classifies whether
+returned *content* is correct after a contradiction. This eval targets a
+structurally different failure mode that `ConflictSignal` cannot see at
+all: the returned content can be fully correct, and the backend can still
+be silently broken at ORDERING it.
+
+Motivating case: mempalace/mempalace#1733 (GitHub user Kartalops, found
+while validating memtrust against real MemPalace usage). `mempalace/
+layers.py`'s `Layer1.generate()` sorts drawers by `importance`/
+`emotional_weight`/`weight`, but no ingest path in the real package ever
+writes those keys -- confirmed 0/45,969 drawers on a real palace. So
+`importance` silently defaults to a constant, the "ranked by importance"
+sort degenerates to plain insertion order, and a `wake-up`/recall call
+documented to return "high importance, recent" moments instead silently
+returns the oldest moments first. Every individual returned drawer is a
+real, correctly-stored memory -- there is no contradiction anywhere in
+this bug, which is exactly why `ConflictSignal` structurally cannot flag
+it: that taxonomy only classifies cases where a fact was stored, then
+recontradicted, then queried. This bug has no contradiction step at all.
+
+Design principle (same as evals/contradiction.py's classify_case and
+evals/resource_sync_safety.py's classify_resource_sync_file): classifying
+a case never blindly trusts what the adapter reports about its own
+ranking behavior (`QueryResult.ranking_signal`, see adapters/base.py's
+RankingSignal and adapters/mempalace_adapter.py's `_classify_ranking_signal`
+for how an adapter derives that self-report). This eval owns the ground
+truth for what order each case's records were stored in and what values
+they carried for the case's declared `ranking_field`, and computes the
+final signal from that directly:
+
+  * If the ranking field is missing from every returned record, or present
+    but carrying the identical value on every returned record, no real
+    per-record signal exists to have driven the order -- classified
+    MISSING_ORDERING_KEY regardless of what the adapter self-reports.
+  * If the field carries genuinely varied values, this eval checks whether
+    the actual returned order is sorted by descending value. If it is,
+    the field is a genuine ranking signal -- SIGNAL_DRIVEN. If it is not,
+    a real signal exists and the backend still is not ordering by it --
+    ORDER_INCONSISTENT, a distinct and equally worth-flagging bug from
+    MISSING_ORDERING_KEY.
+  * Fewer than 2 returned records, or zero records at all, gives this eval
+    nothing to compare -- NOT_APPLICABLE.
+
+Honest limitation (see docs/methodology.md for the full write-up): this
+eval can only ever prove "no real per-record signal was observed driving
+this response's order." It cannot always distinguish "the backend
+genuinely has no meaningful variation to rank by for this query" from
+"the backend forgot to populate the ranking field" -- both produce the
+identical observable symptom (every returned record shares one value).
+MISSING_ORDERING_KEY is the correct, honest name for what is actually
+detected: the *absence of a driving signal*, not a proven claim about the
+backend's internal cause. Kartalops's mempalace#1733 finding (0/45,969
+drawers with a real value, confirmed by direct inspection of a live
+palace) is the strong form of this; this eval's black-box query-response
+view alone would only ever justify the weaker, still-honest claim above.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from memtrust.adapters.base import (
+    BackendAPIError,
+    MemoryBackendAdapter,
+    MemoryRecord,
+    QueryResult,
+    RankingSignal,
+)
+
+DEFAULT_FIXTURE_PATH = (
+    Path(__file__).resolve().parents[3] / "tests" / "fixtures" / "ranking_quality_cases.json"
+)
+
+
+@dataclass
+class RankingQualitySeedRecord:
+    """One record to store before querying, in the order it appears in
+    the case's `records` list -- that storage order is this eval's ground
+    truth for "insertion order," which `classify_ranking_case` compares
+    the actual returned order against."""
+
+    content: str
+    metadata: dict[str, str] = field(default_factory=dict)
+
+
+@dataclass
+class RankingQualityCase:
+    case_id: str
+    session_id: str
+    query: str
+    ranking_field: str
+    """Which metadata key this case is testing as the ranking-relevant
+    field (e.g. "importance") -- see RankingSignal in adapters/base.py."""
+    records: list[RankingQualitySeedRecord]
+
+
+@dataclass
+class RankingQualityCaseResult:
+    case: RankingQualityCase
+    signal: RankingSignal
+    adapter_reported_signal: RankingSignal | None
+    field_values: list[float | None]
+    """The case's `ranking_field` value read off each returned record, in
+    returned order, parsed as float where possible (None where missing or
+    unparseable) -- the evidence `signal` above was computed from."""
+    matches_insertion_order: bool | None
+    """Whether the returned record order exactly matches the order the
+    case's records were stored in. None when this could not be determined
+    (e.g. returned memory_ids don't correspond to any stored id)."""
+    retrieved_content: str
+    error: str | None = None
+
+
+@dataclass
+class RankingQualityEvalResult:
+    backend_name: str
+    dataset_path: str
+    case_results: list[RankingQualityCaseResult] = field(default_factory=list)
+
+    @property
+    def scored_cases(self) -> list[RankingQualityCaseResult]:
+        return [c for c in self.case_results if c.error is None]
+
+    def _fraction(self, signal: RankingSignal) -> float | None:
+        scored = self.scored_cases
+        if not scored:
+            return None
+        matching = sum(1 for c in scored if c.signal == signal)
+        return matching / len(scored)
+
+    @property
+    def signal_driven_rate(self) -> float | None:
+        return self._fraction(RankingSignal.SIGNAL_DRIVEN)
+
+    @property
+    def missing_ordering_key_rate(self) -> float | None:
+        """Fraction of cases where no real per-record ranking signal was
+        observed -- the headline metric this eval exists to surface, and
+        the one that would have caught mempalace/mempalace#1733's exact
+        shape (0/45,969 drawers ever getting a real `importance` value)."""
+        return self._fraction(RankingSignal.MISSING_ORDERING_KEY)
+
+    @property
+    def order_inconsistent_rate(self) -> float | None:
+        return self._fraction(RankingSignal.ORDER_INCONSISTENT)
+
+    @property
+    def not_applicable_rate(self) -> float | None:
+        return self._fraction(RankingSignal.NOT_APPLICABLE)
+
+
+def load_dataset(path: Path | str = DEFAULT_FIXTURE_PATH) -> list[RankingQualityCase]:
+    data = json.loads(Path(path).read_text())
+    cases: list[dict[str, Any]] = data["cases"]
+    return [
+        RankingQualityCase(
+            case_id=c["case_id"],
+            session_id=c["session_id"],
+            query=c["query"],
+            ranking_field=c["ranking_field"],
+            records=[
+                RankingQualitySeedRecord(
+                    content=r["content"],
+                    metadata=r.get("metadata", {}),
+                )
+                for r in c["records"]
+            ],
+        )
+        for c in cases
+    ]
+
+
+def _parse_float(value: str | None) -> float | None:
+    if value is None:
+        return None
+    try:
+        return float(value)
+    except ValueError:
+        return None
+
+
+def _field_value(case: RankingQualityCase, record: MemoryRecord) -> float | None:
+    """Read a case's declared `ranking_field` off one returned record.
+
+    `MemoryRecord.score` (adapters/base.py) is a dedicated, already-typed
+    dataclass field every mem0-backed adapter populates from its own
+    search response -- it is NOT part of `metadata` (a vendor-agnostic,
+    string-only dict this eval otherwise reads from). Before this change,
+    a fixture case declaring `ranking_field: "score"` would always read
+    `metadata.get("score")`, which is never populated by any adapter, so
+    every such case landed on MISSING_ORDERING_KEY unconditionally --
+    never actually testing real score-based ranking at all.
+
+    Motivating case: mem0ai/mem0#3144 (GitHub user Duguce, closed as
+    duplicate of #4453) -- mem0's local vector-store search `score` field
+    doesn't semantically make sense: most vector stores return raw
+    distance as "score" and mem0 does not invert it into similarity, so a
+    LESS-relevant memory can carry a HIGHER score. Reading `.score`
+    directly (case.ranking_field == "score") is what lets this eval
+    actually classify that shape -- SIGNAL_DRIVEN only when the returned
+    order is genuinely sorted by descending score, ORDER_INCONSISTENT when
+    a real (if semantically inverted) score value exists but the returned
+    order doesn't track it -- instead of silently treating every
+    score-labeled case as "no signal present" via the metadata-only read.
+    """
+    if case.ranking_field == "score":
+        return record.score
+    return _parse_float(record.metadata.get(case.ranking_field))
+
+
+def classify_ranking_case(
+    case: RankingQualityCase,
+    query_result: QueryResult,
+    stored_ids_in_order: list[str],
+) -> tuple[RankingSignal, list[float | None], bool | None]:
+    """Classify a single case's outcome from the adapter's query response.
+
+    Returns (final_signal, field_values_in_returned_order,
+    matches_insertion_order). Never trusts
+    `query_result.ranking_signal` (the adapter's own self-report) as the
+    final answer -- see this module's docstring for why -- it is only
+    threaded through by the caller for transparency/comparison in the
+    result record.
+    """
+    records = query_result.records
+    field_values = [_field_value(case, r) for r in records]
+
+    matches_insertion_order: bool | None = None
+    if records and stored_ids_in_order:
+        returned_ids = [r.memory_id for r in records]
+        known_ids = [rid for rid in returned_ids if rid in stored_ids_in_order]
+        if known_ids:
+            expected = [rid for rid in stored_ids_in_order if rid in returned_ids]
+            matches_insertion_order = known_ids == expected
+
+    if len(records) < 2:
+        return RankingSignal.NOT_APPLICABLE, field_values, matches_insertion_order
+
+    present_values = [v for v in field_values if v is not None]
+
+    if len(present_values) < len(field_values) or len(set(present_values)) <= 1:
+        # The field is missing from at least one returned record, or is
+        # present everywhere but identical -- either way, no real
+        # per-record signal exists to have driven the order. This is the
+        # exact mempalace/mempalace#1733 shape.
+        return RankingSignal.MISSING_ORDERING_KEY, field_values, matches_insertion_order
+
+    is_sorted_descending = all(
+        a >= b for a, b in zip(present_values, present_values[1:], strict=False)
+    )
+    if is_sorted_descending:
+        return RankingSignal.SIGNAL_DRIVEN, field_values, matches_insertion_order
+    return RankingSignal.ORDER_INCONSISTENT, field_values, matches_insertion_order
+
+
+def run_ranking_quality_eval(
+    adapter: MemoryBackendAdapter,
+    dataset_path: Path | str = DEFAULT_FIXTURE_PATH,
+) -> RankingQualityEvalResult:
+    cases = load_dataset(dataset_path)
+    result = RankingQualityEvalResult(backend_name=adapter.name, dataset_path=str(dataset_path))
+
+    for case in cases:
+        stored_ids: list[str] = []
+        try:
+            for seed in case.records:
+                store_result = adapter.store(case.session_id, seed.content, metadata=seed.metadata)
+                stored_ids.append(store_result.memory_id)
+            query_result = adapter.query(case.session_id, case.query, top_k=len(case.records))
+        except BackendAPIError as exc:
+            result.case_results.append(
+                RankingQualityCaseResult(
+                    case=case,
+                    signal=RankingSignal.NOT_APPLICABLE,
+                    adapter_reported_signal=None,
+                    field_values=[],
+                    matches_insertion_order=None,
+                    retrieved_content="",
+                    error=str(exc),
+                )
+            )
+            continue
+
+        final_signal, field_values, matches_insertion_order = classify_ranking_case(
+            case, query_result, stored_ids
+        )
+        result.case_results.append(
+            RankingQualityCaseResult(
+                case=case,
+                signal=final_signal,
+                adapter_reported_signal=query_result.ranking_signal,
+                field_values=field_values,
+                matches_insertion_order=matches_insertion_order,
+                retrieved_content=" ".join(r.content for r in query_result.records),
+            )
+        )
+
+    return result

memtrust/evals/resource_sync_safety.py

@@ -0,0 +1,336 @@
+"""MemTrust's directory/resource-sync safety eval.
+
+The store/query/update model the other evals in this package exercise
+only covers single-key memory operations. It has no concept of a
+multi-file directory mirror/resync operation, so it cannot observe a
+resync mechanism silently deleting user-owned files that a backend's
+ingestion watcher did not itself generate.
+
+This eval closes that specific gap. It is modeled directly on a real,
+high-severity bug report: volcengine/OpenViking#3029, where OpenViking's
+Feishu resync mechanism silently deleted user-owned files sitting
+alongside the files its own ingestion watcher had generated. The eval
+seeds a mix of "generated" files (standing in for what a watcher itself
+would produce) and "user" files (standing in for files a person added to
+the same resource path independently of the watcher) under one resource
+prefix, triggers a resync, and re-lists the prefix to see what survived:
+
+  * PRESERVED             -- the file is still present after the resync,
+                              and (where content could be re-verified)
+                              its content is unchanged.
+  * DELETED_USER_FILE      -- the file was present before the resync and
+                              is gone afterward, with no error and no
+                              signal to the caller. This is the exact
+                              #3029 failure mode: a resync silently
+                              destroying data it never wrote.
+  * OVERWRITTEN_UNCHANGED  -- the path itself still exists after the
+                              resync (existence "unchanged"), but its
+                              content no longer matches what was seeded,
+                              i.e. the resync silently overwrote it
+                              rather than deleting it outright.
+  * NESTED_CONTENT_UNINDEXED -- the path itself still exists after the
+                              resync (list_resource_paths() reports it
+                              present) and a query() was actually issued,
+                              but the search index never returned a
+                              record for that path at all -- distinct
+                              from OVERWRITTEN_UNCHANGED, which means a
+                              record for the path *was* found but its
+                              content had changed. This is the
+                              volcengine/OpenViking#1703 failure mode:
+                              index_resource() skipped every subdirectory
+                              during reindex, so nested-directory content
+                              was never vectorized in the first place --
+                              "never indexed," not "deleted" or
+                              "overwritten." A file at a top-level path
+                              (no "/" in its resource path) failing this
+                              way would just look like a generic search
+                              miss; this signal is specifically for paths
+                              that are nested, where "never indexed" has
+                              a concrete, reproducible mechanism to point
+                              to.
+
+Optional/flagged capability: this eval only runs against adapters that
+set MemoryBackendAdapter.supports_resource_sync = True. Adapters without
+a directory/resource-mirror concept (the store/query/update-only model)
+are skipped cleanly -- reported as skipped, never silently dropped from
+the results table and never crashed by calling an unimplemented method.
+
+Design principle (see evals/contradiction.py's classify_case for the same
+pattern applied to contradiction detection): classification never blindly
+trusts that a seeded file is "fine" just because the adapter didn't error. Each file's
+final signal is derived from the actual before/after path listings (and,
+where possible, a re-query of the file's own content) -- not from any
+self-report the adapter makes about the resync succeeding.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from enum import StrEnum
+from pathlib import Path
+from typing import Any
+
+from memtrust.adapters.base import BackendAPIError, MemoryBackendAdapter
+
+DEFAULT_FIXTURE_PATH = (
+    Path(__file__).resolve().parents[3] / "tests" / "fixtures" / "resource_sync_cases.json"
+)
+
+
+class ResourceSyncSignal(StrEnum):
+    """How one seeded file fared across a trigger_resync() call."""
+
+    PRESERVED = "preserved"
+    """Present after the resync; content re-verified as unchanged (or the
+    adapter offered no way to re-verify content, in which case presence
+    alone is treated as preserved)."""
+
+    DELETED_USER_FILE = "deleted_user_file"
+    """Present before the resync, gone afterward, with no error raised.
+    The exact volcengine/OpenViking#3029 failure mode."""
+
+    OVERWRITTEN_UNCHANGED = "overwritten_unchanged"
+    """Still present after the resync, but its content no longer matches
+    what was seeded -- silently overwritten rather than deleted."""
+
+    NESTED_CONTENT_UNINDEXED = "nested_content_unindexed"
+    """Still present after the resync (the path itself was not deleted or
+    overwritten), but no query() call ever returned a record for that
+    path at all -- the content exists on disk/in the filesystem mirror
+    but was never indexed for search. This is "never indexed," not
+    "deleted": the exact volcengine/OpenViking#1703 failure mode, where
+    a reindex's directory walk skipped every subdirectory and left
+    nested content permanently unsearchable with no error raised."""
+
+    NOT_APPLICABLE = "not_applicable"
+    """Either the backend has no resource-sync primitive this eval can
+    exercise (MemoryBackendAdapter.supports_resource_sync is False), or a
+    file's before/after state could not be established at all (e.g. it
+    was never observed present before the resync in the first place)."""
+
+
+@dataclass
+class ResourceSyncSeedFile:
+    path_suffix: str
+    origin: str  # "generated" | "user"
+    content: str
+
+
+@dataclass
+class ResourceSyncCase:
+    case_id: str
+    prefix: str
+    seed_files: list[ResourceSyncSeedFile]
+
+
+@dataclass
+class ResourceSyncFileResult:
+    case_id: str
+    path_suffix: str
+    origin: str
+    stored_path: str | None
+    present_before_resync: bool
+    present_after_resync: bool
+    content_matches_after_resync: bool | None
+    indexed_after_resync: bool | None
+    signal: ResourceSyncSignal
+    error: str | None = None
+
+
+@dataclass
+class ResourceSyncEvalResult:
+    backend_name: str
+    dataset_path: str
+    file_results: list[ResourceSyncFileResult] = field(default_factory=list)
+    skipped: bool = False
+    skip_reason: str | None = None
+
+    @property
+    def scored_files(self) -> list[ResourceSyncFileResult]:
+        return [f for f in self.file_results if f.error is None]
+
+    def _fraction(self, signal: ResourceSyncSignal, origin: str | None = None) -> float | None:
+        scored = self.scored_files
+        if origin is not None:
+            scored = [f for f in scored if f.origin == origin]
+        if not scored:
+            return None
+        matching = sum(1 for f in scored if f.signal == signal)
+        return matching / len(scored)
+
+    @property
+    def user_file_deletion_rate(self) -> float | None:
+        """Fraction of user-origin seed files silently deleted by a
+        resync -- the headline metric this eval exists to surface."""
+        return self._fraction(ResourceSyncSignal.DELETED_USER_FILE, origin="user")
+
+    @property
+    def preserved_rate(self) -> float | None:
+        return self._fraction(ResourceSyncSignal.PRESERVED)
+
+    @property
+    def overwritten_unchanged_rate(self) -> float | None:
+        return self._fraction(ResourceSyncSignal.OVERWRITTEN_UNCHANGED)
+
+    @property
+    def nested_content_unindexed_rate(self) -> float | None:
+        """Fraction of seed files that survived the resync on disk but
+        were never returned by any query() call -- the volcengine/
+        OpenViking#1703 signal, distinct from deletion or overwrite."""
+        return self._fraction(ResourceSyncSignal.NESTED_CONTENT_UNINDEXED)
+
+
+def load_dataset(path: Path | str = DEFAULT_FIXTURE_PATH) -> list[ResourceSyncCase]:
+    data = json.loads(Path(path).read_text())
+    cases: list[dict[str, Any]] = data["cases"]
+    return [
+        ResourceSyncCase(
+            case_id=c["case_id"],
+            prefix=c["prefix"],
+            seed_files=[
+                ResourceSyncSeedFile(
+                    path_suffix=sf["path_suffix"],
+                    origin=sf["origin"],
+                    content=sf["content"],
+                )
+                for sf in c["seed_files"]
+            ],
+        )
+        for c in cases
+    ]
+
+
+def classify_resource_sync_file(
+    present_before: bool,
+    present_after: bool,
+    content_matches_after_resync: bool | None,
+    indexed_after_resync: bool | None = None,
+) -> ResourceSyncSignal:
+    """Classify a single seeded file's outcome from its before/after
+    presence and (if re-verifiable) content/index match.
+
+    Returns DELETED_USER_FILE whenever a file that was confirmed present
+    before the resync is gone afterward -- deliberately independent of
+    whether the caller passes a "generated" or "user" origin here, since
+    the eval only computes rates *by* origin afterward (see
+    ResourceSyncEvalResult.user_file_deletion_rate); a generated file
+    disappearing unexpectedly is just as real a signal, it is simply not
+    the metric #3029 was about.
+
+    Returns NESTED_CONTENT_UNINDEXED whenever a file is still present
+    after the resync but `indexed_after_resync` is explicitly False --
+    meaning a query() call was actually made and returned no record at
+    all for this path, as opposed to returning a record with stale
+    content (OVERWRITTEN_UNCHANGED). `indexed_after_resync=None` (the
+    default) means the caller never attempted to distinguish "no record
+    found" from "record found with wrong content" -- existing callers
+    that only pass the first three arguments keep the prior behavior
+    unchanged (OVERWRITTEN_UNCHANGED wins on a content mismatch).
+
+    A file never observed present before the resync has nothing
+    meaningful to classify (its "before" state is unknown), so it is
+    recorded as NOT_APPLICABLE rather than guessed at either way.
+    """
+    if present_before and not present_after:
+        return ResourceSyncSignal.DELETED_USER_FILE
+    if present_after and indexed_after_resync is False:
+        return ResourceSyncSignal.NESTED_CONTENT_UNINDEXED
+    if present_after and content_matches_after_resync is False:
+        return ResourceSyncSignal.OVERWRITTEN_UNCHANGED
+    if present_after:
+        return ResourceSyncSignal.PRESERVED
+    return ResourceSyncSignal.NOT_APPLICABLE
+
+
+def run_resource_sync_eval(
+    adapter: MemoryBackendAdapter,
+    dataset_path: Path | str = DEFAULT_FIXTURE_PATH,
+) -> ResourceSyncEvalResult:
+    cases = load_dataset(dataset_path)
+    result = ResourceSyncEvalResult(backend_name=adapter.name, dataset_path=str(dataset_path))
+
+    if not adapter.supports_resource_sync:
+        result.skipped = True
+        result.skip_reason = (
+            f"{adapter.name} does not support resource-sync operations "
+            "(supports_resource_sync=False) -- skipped, not run."
+        )
+        return result
+
+    for case in cases:
+        stored_paths: dict[str, str] = {}
+        try:
+            for seed in case.seed_files:
+                store_result = adapter.store(
+                    case.prefix,
+                    seed.content,
+                    metadata={"resource_path": seed.path_suffix, "origin": seed.origin},
+                )
+                stored_paths[seed.path_suffix] = store_result.memory_id
+
+            paths_before = set(adapter.list_resource_paths(case.prefix))
+            adapter.trigger_resync(case.prefix)
+            paths_after = set(adapter.list_resource_paths(case.prefix))
+        except BackendAPIError as exc:
+            for seed in case.seed_files:
+                result.file_results.append(
+                    ResourceSyncFileResult(
+                        case_id=case.case_id,
+                        path_suffix=seed.path_suffix,
+                        origin=seed.origin,
+                        stored_path=stored_paths.get(seed.path_suffix),
+                        present_before_resync=False,
+                        present_after_resync=False,
+                        content_matches_after_resync=None,
+                        indexed_after_resync=None,
+                        signal=ResourceSyncSignal.NOT_APPLICABLE,
+                        error=str(exc),
+                    )
+                )
+            continue
+
+        for seed in case.seed_files:
+            stored_path = stored_paths[seed.path_suffix]
+            present_before = stored_path in paths_before
+            present_after = stored_path in paths_after
+
+            content_matches: bool | None = None
+            indexed_after: bool | None = None
+            if present_after:
+                try:
+                    query_result = adapter.query(case.prefix, seed.content, top_k=5)
+                    # Two distinct questions, not one: did the search index
+                    # return *any* record for this exact path at all
+                    # (indexed_after -- the #1703 signal), and separately,
+                    # among whatever records did come back, does any one's
+                    # content match what was seeded (content_matches -- the
+                    # #3029-era overwrite signal). Collapsing these into one
+                    # boolean would make "never indexed" indistinguishable
+                    # from "indexed with stale content."
+                    indexed_after = any(r.memory_id == stored_path for r in query_result.records)
+                    content_matches = any(
+                        seed.content.lower() in r.content.lower() for r in query_result.records
+                    )
+                except BackendAPIError:
+                    content_matches = None
+                    indexed_after = None
+
+            signal = classify_resource_sync_file(
+                present_before, present_after, content_matches, indexed_after
+            )
+            result.file_results.append(
+                ResourceSyncFileResult(
+                    case_id=case.case_id,
+                    path_suffix=seed.path_suffix,
+                    origin=seed.origin,
+                    stored_path=stored_path,
+                    present_before_resync=present_before,
+                    present_after_resync=present_after,
+                    content_matches_after_resync=content_matches,
+                    indexed_after_resync=indexed_after,
+                    signal=signal,
+                )
+            )
+
+    return result