trisynapse-memory 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- trisynapse_memory/__init__.py +109 -0
- trisynapse_memory/_version.py +34 -0
- trisynapse_memory/adapters/__init__.py +10 -0
- trisynapse_memory/adapters/agent_events.py +82 -0
- trisynapse_memory/adapters/benchmarks/__init__.py +29 -0
- trisynapse_memory/adapters/benchmarks/base.py +79 -0
- trisynapse_memory/adapters/benchmarks/halumem.py +80 -0
- trisynapse_memory/adapters/benchmarks/locomo.py +81 -0
- trisynapse_memory/adapters/benchmarks/longmemeval.py +75 -0
- trisynapse_memory/adapters/benchmarks/memorydoc.py +83 -0
- trisynapse_memory/adapters/trisynapse_live.py +74 -0
- trisynapse_memory/api.py +1325 -0
- trisynapse_memory/benchmarks/__init__.py +18 -0
- trisynapse_memory/benchmarks/evaluation.py +138 -0
- trisynapse_memory/benchmarks/release.py +85 -0
- trisynapse_memory/benchmarks/runner.py +213 -0
- trisynapse_memory/cli.py +575 -0
- trisynapse_memory/engine/__init__.py +111 -0
- trisynapse_memory/engine/compilation.py +180 -0
- trisynapse_memory/engine/embedding.py +130 -0
- trisynapse_memory/engine/formation.py +181 -0
- trisynapse_memory/engine/loaders.py +93 -0
- trisynapse_memory/engine/memory.py +2148 -0
- trisynapse_memory/engine/models.py +531 -0
- trisynapse_memory/engine/privacy.py +87 -0
- trisynapse_memory/engine/providers.py +611 -0
- trisynapse_memory/engine/retrieval.py +588 -0
- trisynapse_memory/engine/sources.py +600 -0
- trisynapse_memory/engine/trace.py +1191 -0
- trisynapse_memory/engine/vector_cache.py +98 -0
- trisynapse_memory/prompts/__init__.py +59 -0
- trisynapse_memory/prompts/answer.md +7 -0
- trisynapse_memory/prompts/benchmark_judge.md +3 -0
- trisynapse_memory/prompts/episode_recall.md +4 -0
- trisynapse_memory/prompts/extraction.md +3 -0
- trisynapse_memory/prompts/image_extraction.md +10 -0
- trisynapse_memory/studio/dist/assets/graph-mPIvIbRV.js +327 -0
- trisynapse_memory/studio/dist/assets/index-CnsjvBGs.css +1 -0
- trisynapse_memory/studio/dist/assets/index-D54oQBbv.js +14 -0
- trisynapse_memory/studio/dist/assets/logo-16WU-rcz.png +0 -0
- trisynapse_memory/studio/dist/assets/markdown-ubUk7Dpl.js +29 -0
- trisynapse_memory/studio/dist/assets/react-CvQExsv3.js +3 -0
- trisynapse_memory/studio/dist/index.html +18 -0
- trisynapse_memory/terminal.py +585 -0
- trisynapse_memory-0.1.0.dist-info/METADATA +456 -0
- trisynapse_memory-0.1.0.dist-info/RECORD +50 -0
- trisynapse_memory-0.1.0.dist-info/WHEEL +5 -0
- trisynapse_memory-0.1.0.dist-info/entry_points.txt +2 -0
- trisynapse_memory-0.1.0.dist-info/licenses/LICENSE +201 -0
- trisynapse_memory-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""Public Trace & Recall API for Trisynapse Memory."""
|
|
2
|
+
|
|
3
|
+
from trisynapse_memory._version import __version__
|
|
4
|
+
from trisynapse_memory.engine import (
|
|
5
|
+
Actor,
|
|
6
|
+
Citation,
|
|
7
|
+
CompiledClaim,
|
|
8
|
+
CompletionProvider,
|
|
9
|
+
ConnectionTestResult,
|
|
10
|
+
EmbeddingRebuildRequired,
|
|
11
|
+
EpisodeInfo,
|
|
12
|
+
EpisodeRecallView,
|
|
13
|
+
LoadedDocument,
|
|
14
|
+
MemoryDelta,
|
|
15
|
+
MemoryEngine,
|
|
16
|
+
MemoryHistory,
|
|
17
|
+
MemoryGraphEdge,
|
|
18
|
+
MemoryGraphNode,
|
|
19
|
+
MemoryGraphPage,
|
|
20
|
+
MemoryJob,
|
|
21
|
+
MemoryNamespace,
|
|
22
|
+
MemoryPage,
|
|
23
|
+
MemoryQueryResult,
|
|
24
|
+
MemorySearchResult,
|
|
25
|
+
ModelConfiguration,
|
|
26
|
+
ModelConfigurationChange,
|
|
27
|
+
ModelDescriptor,
|
|
28
|
+
PrivacyFilter,
|
|
29
|
+
ProviderError,
|
|
30
|
+
ProviderDescriptor,
|
|
31
|
+
ProviderRole,
|
|
32
|
+
ProviderSelection,
|
|
33
|
+
ProviderSettings,
|
|
34
|
+
QueryCandidateSnapshot,
|
|
35
|
+
QueryRun,
|
|
36
|
+
QueryRunPage,
|
|
37
|
+
QueryRunRemoveRequest,
|
|
38
|
+
QueryStep,
|
|
39
|
+
RetrievalConfiguration,
|
|
40
|
+
IngestionRun,
|
|
41
|
+
RemoveRequest,
|
|
42
|
+
RemoveResult,
|
|
43
|
+
RecallSnapshot,
|
|
44
|
+
RedactionResult,
|
|
45
|
+
RetrievalTrace,
|
|
46
|
+
SearchHit,
|
|
47
|
+
SnapshotDiff,
|
|
48
|
+
SourceIngestionResult,
|
|
49
|
+
SourceInput,
|
|
50
|
+
SourceRecord,
|
|
51
|
+
SourcePreview,
|
|
52
|
+
SourcePreviewItem,
|
|
53
|
+
TraceVerification,
|
|
54
|
+
load_document,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
__all__ = [
|
|
58
|
+
"__version__",
|
|
59
|
+
"Actor",
|
|
60
|
+
"Citation",
|
|
61
|
+
"CompiledClaim",
|
|
62
|
+
"CompletionProvider",
|
|
63
|
+
"ConnectionTestResult",
|
|
64
|
+
"EmbeddingRebuildRequired",
|
|
65
|
+
"EpisodeInfo",
|
|
66
|
+
"EpisodeRecallView",
|
|
67
|
+
"LoadedDocument",
|
|
68
|
+
"MemoryDelta",
|
|
69
|
+
"MemoryEngine",
|
|
70
|
+
"MemoryHistory",
|
|
71
|
+
"MemoryGraphEdge",
|
|
72
|
+
"MemoryGraphNode",
|
|
73
|
+
"MemoryGraphPage",
|
|
74
|
+
"MemoryJob",
|
|
75
|
+
"MemoryNamespace",
|
|
76
|
+
"MemoryPage",
|
|
77
|
+
"MemoryQueryResult",
|
|
78
|
+
"MemorySearchResult",
|
|
79
|
+
"ModelConfiguration",
|
|
80
|
+
"ModelConfigurationChange",
|
|
81
|
+
"ModelDescriptor",
|
|
82
|
+
"PrivacyFilter",
|
|
83
|
+
"ProviderError",
|
|
84
|
+
"ProviderDescriptor",
|
|
85
|
+
"ProviderRole",
|
|
86
|
+
"ProviderSelection",
|
|
87
|
+
"ProviderSettings",
|
|
88
|
+
"QueryCandidateSnapshot",
|
|
89
|
+
"QueryRun",
|
|
90
|
+
"QueryRunPage",
|
|
91
|
+
"QueryRunRemoveRequest",
|
|
92
|
+
"QueryStep",
|
|
93
|
+
"RetrievalConfiguration",
|
|
94
|
+
"IngestionRun",
|
|
95
|
+
"RemoveRequest",
|
|
96
|
+
"RemoveResult",
|
|
97
|
+
"RecallSnapshot",
|
|
98
|
+
"RedactionResult",
|
|
99
|
+
"RetrievalTrace",
|
|
100
|
+
"SearchHit",
|
|
101
|
+
"SnapshotDiff",
|
|
102
|
+
"SourceIngestionResult",
|
|
103
|
+
"SourceInput",
|
|
104
|
+
"SourceRecord",
|
|
105
|
+
"SourcePreview",
|
|
106
|
+
"SourcePreviewItem",
|
|
107
|
+
"TraceVerification",
|
|
108
|
+
"load_document",
|
|
109
|
+
]
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Resolve the public package version from one canonical source."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import tomllib
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _source_tree_version() -> str | None:
|
|
11
|
+
"""Read pyproject.toml when running directly from a source checkout."""
|
|
12
|
+
|
|
13
|
+
pyproject = Path(__file__).resolve().parents[2] / "pyproject.toml"
|
|
14
|
+
if not pyproject.is_file():
|
|
15
|
+
return None
|
|
16
|
+
with pyproject.open("rb") as stream:
|
|
17
|
+
value = tomllib.load(stream).get("project", {}).get("version")
|
|
18
|
+
return str(value) if value else None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def get_version() -> str:
|
|
22
|
+
"""Return the source version in a checkout or installed distribution metadata."""
|
|
23
|
+
|
|
24
|
+
source_version = _source_tree_version()
|
|
25
|
+
if source_version is not None:
|
|
26
|
+
return source_version
|
|
27
|
+
try:
|
|
28
|
+
return version("trisynapse-memory")
|
|
29
|
+
except PackageNotFoundError:
|
|
30
|
+
return "0+unknown"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
__version__ = get_version()
|
|
34
|
+
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Package-wide external integration and dataset-adapter boundary.
|
|
2
|
+
|
|
3
|
+
Live integrations are exported here; benchmark adapters are available from
|
|
4
|
+
``trisynapse_memory.adapters.benchmarks`` to avoid eagerly loading their registry.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from trisynapse_memory.adapters.agent_events import AgentEvent, capture_agent_event
|
|
8
|
+
from trisynapse_memory.adapters.trisynapse_live import open_vault_engine
|
|
9
|
+
|
|
10
|
+
__all__ = ["AgentEvent", "capture_agent_event", "open_vault_engine"]
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""Generic coding-agent lifecycle events for automatic capture and context injection."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, Literal
|
|
6
|
+
|
|
7
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
8
|
+
|
|
9
|
+
from trisynapse_memory.engine import MemoryEngine
|
|
10
|
+
from trisynapse_memory.engine.models import MemoryNamespace
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class AgentEvent(BaseModel):
|
|
14
|
+
model_config = ConfigDict(extra="allow")
|
|
15
|
+
|
|
16
|
+
type: Literal[
|
|
17
|
+
"session_start", "user_prompt", "pre_tool", "post_tool", "post_tool_failure",
|
|
18
|
+
"pre_compact", "subagent_start", "subagent_stop", "stop", "session_end",
|
|
19
|
+
]
|
|
20
|
+
session_id: str
|
|
21
|
+
agent_id: str
|
|
22
|
+
project_id: str = "default"
|
|
23
|
+
user_id: str | None = None
|
|
24
|
+
tool_name: str | None = None
|
|
25
|
+
content: str | None = None
|
|
26
|
+
input: dict[str, Any] | None = None
|
|
27
|
+
output: str | None = None
|
|
28
|
+
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def capture_agent_event(engine: MemoryEngine, event: AgentEvent | dict[str, Any]) -> dict[str, Any]:
|
|
32
|
+
value = event if isinstance(event, AgentEvent) else AgentEvent.model_validate(event)
|
|
33
|
+
namespace = MemoryNamespace(
|
|
34
|
+
user_id=value.user_id,
|
|
35
|
+
agent_id=value.agent_id,
|
|
36
|
+
project_id=value.project_id,
|
|
37
|
+
session_id=value.session_id,
|
|
38
|
+
)
|
|
39
|
+
episode_id = f"agent:{value.agent_id}:{value.session_id}"
|
|
40
|
+
if value.type in {"session_start", "pre_compact"}:
|
|
41
|
+
query = value.content or "current project decisions, constraints, preferences, failures, and next steps"
|
|
42
|
+
profile = engine.compile_profile(namespace=namespace, query=query)
|
|
43
|
+
return {"captured": False, "context": profile, "namespace": namespace.model_dump(mode="json")}
|
|
44
|
+
|
|
45
|
+
text = _event_text(value)
|
|
46
|
+
if not text:
|
|
47
|
+
return {"captured": False, "reason": "event contains no durable content"}
|
|
48
|
+
delta = engine.ingest_observation(
|
|
49
|
+
text,
|
|
50
|
+
episode_id=episode_id,
|
|
51
|
+
source_ref={"type": "agent_event", "agent_id": value.agent_id, "session_id": value.session_id},
|
|
52
|
+
locator={"event_type": value.type, "tool_name": value.tool_name},
|
|
53
|
+
scope={"event_type": value.type, **value.metadata},
|
|
54
|
+
namespace=namespace,
|
|
55
|
+
external_key=value.metadata.get("event_id"),
|
|
56
|
+
)
|
|
57
|
+
response: dict[str, Any] = {"captured": True, "delta_id": delta.id}
|
|
58
|
+
if value.type in {"stop", "session_end"}:
|
|
59
|
+
engine.build_episode_recall([episode_id], namespace=namespace)
|
|
60
|
+
response["context"] = engine.compile_profile(namespace=namespace)
|
|
61
|
+
return response
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _event_text(event: AgentEvent) -> str:
|
|
65
|
+
if event.type == "user_prompt":
|
|
66
|
+
return f"User prompt: {event.content or ''}".strip()
|
|
67
|
+
if event.type == "pre_tool":
|
|
68
|
+
return f"Tool planned: {event.tool_name or 'unknown'} input={_bounded(event.input)}"
|
|
69
|
+
if event.type in {"post_tool", "post_tool_failure"}:
|
|
70
|
+
status = "failed" if event.type == "post_tool_failure" else "completed"
|
|
71
|
+
return f"Tool {event.tool_name or 'unknown'} {status}. Output: {(event.output or '')[:8000]}".strip()
|
|
72
|
+
if event.type in {"subagent_start", "subagent_stop"}:
|
|
73
|
+
return f"{event.type.replace('_', ' ')}: {event.content or _bounded(event.metadata)}"
|
|
74
|
+
if event.type in {"stop", "session_end"}:
|
|
75
|
+
return f"Session ended: {event.content or 'No explicit summary supplied.'}"
|
|
76
|
+
return event.content or ""
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _bounded(value: Any) -> str:
|
|
80
|
+
text = str(value or "")
|
|
81
|
+
return text[:8000]
|
|
82
|
+
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Registry of benchmark-specific dataset adapters."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from trisynapse_memory.adapters.benchmarks.base import BenchmarkAdapter
|
|
6
|
+
from trisynapse_memory.adapters.benchmarks.halumem import HaluMemAdapter
|
|
7
|
+
from trisynapse_memory.adapters.benchmarks.locomo import LoCoMoAdapter
|
|
8
|
+
from trisynapse_memory.adapters.benchmarks.longmemeval import LongMemEvalAdapter
|
|
9
|
+
from trisynapse_memory.adapters.benchmarks.memorydoc import MemoryDocAdapter
|
|
10
|
+
|
|
11
|
+
_ADAPTERS: dict[str, BenchmarkAdapter] = {
|
|
12
|
+
adapter.name: adapter
|
|
13
|
+
for adapter in (LoCoMoAdapter(), LongMemEvalAdapter(), HaluMemAdapter(), MemoryDocAdapter())
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def get_adapter(name: str) -> BenchmarkAdapter:
|
|
18
|
+
try:
|
|
19
|
+
return _ADAPTERS[name]
|
|
20
|
+
except KeyError as exc:
|
|
21
|
+
supported = ", ".join(sorted(_ADAPTERS))
|
|
22
|
+
raise ValueError(f"unsupported benchmark suite '{name}'; choose one of: {supported}") from exc
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def adapter_names() -> tuple[str, ...]:
|
|
26
|
+
return tuple(sorted(_ADAPTERS))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
__all__ = ["BenchmarkAdapter", "adapter_names", "get_adapter"]
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Dataset adapter interface for production memory benchmarks."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
from collections.abc import Iterable
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any, ClassVar
|
|
10
|
+
|
|
11
|
+
from trisynapse_memory.engine import MemoryEngine
|
|
12
|
+
from trisynapse_memory.engine.models import MemoryNamespace, MemoryQueryResult
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class BenchmarkQuestion:
|
|
17
|
+
"""A normalized question produced by a dataset adapter."""
|
|
18
|
+
|
|
19
|
+
id: str
|
|
20
|
+
question: str
|
|
21
|
+
gold: str
|
|
22
|
+
evidence: Any = None
|
|
23
|
+
evidence_text: str = ""
|
|
24
|
+
metadata: dict[str, Any] = field(default_factory=dict)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True)
|
|
28
|
+
class BenchmarkCase:
|
|
29
|
+
"""One isolated memory world containing one or more questions."""
|
|
30
|
+
|
|
31
|
+
id: str
|
|
32
|
+
payload: dict[str, Any]
|
|
33
|
+
questions: tuple[BenchmarkQuestion, ...]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class PreparedCase:
|
|
38
|
+
"""The engine-facing context returned after adapter ingestion."""
|
|
39
|
+
|
|
40
|
+
episode_ids: tuple[str, ...]
|
|
41
|
+
namespace: MemoryNamespace
|
|
42
|
+
episode_prefix: str | None = None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class BenchmarkAdapter(ABC):
|
|
46
|
+
"""Translate one external dataset into the shared benchmark lifecycle.
|
|
47
|
+
|
|
48
|
+
Adapters own file/schema interpretation and evidence semantics. They may
|
|
49
|
+
ingest through public ``MemoryEngine`` methods, but they never control
|
|
50
|
+
retrieval, extraction, judging, provider selection, or artifact writing.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
name: ClassVar[str]
|
|
54
|
+
default_filename: ClassVar[str]
|
|
55
|
+
|
|
56
|
+
def resolve_dataset(self, data_root: str | Path) -> Path:
|
|
57
|
+
path = Path(data_root)
|
|
58
|
+
if path.is_dir():
|
|
59
|
+
path = path / self.default_filename
|
|
60
|
+
if not path.is_file():
|
|
61
|
+
raise FileNotFoundError(path)
|
|
62
|
+
return path
|
|
63
|
+
|
|
64
|
+
@abstractmethod
|
|
65
|
+
def load_cases(self, path: Path) -> Iterable[BenchmarkCase]:
|
|
66
|
+
"""Parse source data into isolated, normalized cases."""
|
|
67
|
+
|
|
68
|
+
@abstractmethod
|
|
69
|
+
def ingest_case(self, engine: MemoryEngine, case: BenchmarkCase) -> PreparedCase:
|
|
70
|
+
"""Ingest a case through the shipped engine and return query scope."""
|
|
71
|
+
|
|
72
|
+
def result_metadata(
|
|
73
|
+
self,
|
|
74
|
+
question: BenchmarkQuestion,
|
|
75
|
+
result: MemoryQueryResult,
|
|
76
|
+
) -> dict[str, Any]:
|
|
77
|
+
"""Return dataset-specific evidence metrics for one answer."""
|
|
78
|
+
|
|
79
|
+
return dict(question.metadata)
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""HaluMem dataset adapter."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from collections.abc import Iterable
|
|
7
|
+
from datetime import datetime, timezone
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from trisynapse_memory.adapters.benchmarks.base import (
|
|
11
|
+
BenchmarkAdapter,
|
|
12
|
+
BenchmarkCase,
|
|
13
|
+
BenchmarkQuestion,
|
|
14
|
+
PreparedCase,
|
|
15
|
+
)
|
|
16
|
+
from trisynapse_memory.engine import MemoryEngine
|
|
17
|
+
from trisynapse_memory.engine.models import MemoryNamespace
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class HaluMemAdapter(BenchmarkAdapter):
|
|
21
|
+
name = "halumem"
|
|
22
|
+
default_filename = "HaluMem-Medium.jsonl"
|
|
23
|
+
|
|
24
|
+
def load_cases(self, path: Path) -> Iterable[BenchmarkCase]:
|
|
25
|
+
with path.open(encoding="utf-8") as handle:
|
|
26
|
+
for line_index, raw in enumerate(handle):
|
|
27
|
+
if not raw.strip():
|
|
28
|
+
continue
|
|
29
|
+
user = json.loads(raw)
|
|
30
|
+
user_id = str(user.get("uuid") or line_index)
|
|
31
|
+
questions: list[BenchmarkQuestion] = []
|
|
32
|
+
for session_index, session in enumerate(user.get("sessions") or []):
|
|
33
|
+
for question_index, item in enumerate(session.get("questions") or []):
|
|
34
|
+
evidence_text = " ".join(
|
|
35
|
+
str(value.get("memory_content") if isinstance(value, dict) else value)
|
|
36
|
+
for value in item.get("evidence") or []
|
|
37
|
+
)
|
|
38
|
+
questions.append(BenchmarkQuestion(
|
|
39
|
+
id=f"{user_id}:s{session_index}:q{question_index}",
|
|
40
|
+
question=str(item.get("question") or ""),
|
|
41
|
+
gold=str(item.get("answer") or ""),
|
|
42
|
+
evidence=item.get("evidence") or [],
|
|
43
|
+
evidence_text=evidence_text,
|
|
44
|
+
))
|
|
45
|
+
yield BenchmarkCase(id=user_id, payload=user, questions=tuple(questions))
|
|
46
|
+
|
|
47
|
+
def ingest_case(self, engine: MemoryEngine, case: BenchmarkCase) -> PreparedCase:
|
|
48
|
+
namespace = MemoryNamespace(project_id=f"benchmark:halumem:{case.id}")
|
|
49
|
+
episode_ids: list[str] = []
|
|
50
|
+
for session_index, session in enumerate(case.payload.get("sessions") or []):
|
|
51
|
+
episode_id = f"halu:{case.id}:s{session_index:04d}"
|
|
52
|
+
episode_ids.append(episode_id)
|
|
53
|
+
messages = [
|
|
54
|
+
{
|
|
55
|
+
"id": f"{session_index}:{turn_index}",
|
|
56
|
+
"role": turn.get("role", "speaker"),
|
|
57
|
+
"content": turn.get("content", ""),
|
|
58
|
+
"timestamp": _normalize_timestamp(turn.get("timestamp")),
|
|
59
|
+
}
|
|
60
|
+
for turn_index, turn in enumerate(session.get("dialogue") or [])
|
|
61
|
+
]
|
|
62
|
+
if messages:
|
|
63
|
+
engine.ingest_messages(messages, episode_id=episode_id, namespace=namespace)
|
|
64
|
+
return PreparedCase(tuple(episode_ids), namespace, episode_prefix=f"halu:{case.id}:")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _normalize_timestamp(value: object) -> str | None:
|
|
68
|
+
if value is None or not str(value).strip():
|
|
69
|
+
return None
|
|
70
|
+
text = str(value).strip()
|
|
71
|
+
try:
|
|
72
|
+
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
|
73
|
+
except ValueError:
|
|
74
|
+
try:
|
|
75
|
+
parsed = datetime.strptime(text, "%b %d, %Y, %H:%M:%S").replace(tzinfo=timezone.utc)
|
|
76
|
+
except ValueError as exc:
|
|
77
|
+
raise ValueError(f"unsupported HaluMem timestamp: {text}") from exc
|
|
78
|
+
if parsed.tzinfo is None:
|
|
79
|
+
parsed = parsed.replace(tzinfo=timezone.utc)
|
|
80
|
+
return parsed.isoformat()
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""LoCoMo dataset adapter."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from collections.abc import Iterable
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from trisynapse_memory.adapters.benchmarks.base import (
|
|
11
|
+
BenchmarkAdapter,
|
|
12
|
+
BenchmarkCase,
|
|
13
|
+
BenchmarkQuestion,
|
|
14
|
+
PreparedCase,
|
|
15
|
+
)
|
|
16
|
+
from trisynapse_memory.engine import MemoryEngine
|
|
17
|
+
from trisynapse_memory.engine.models import MemoryNamespace, MemoryQueryResult
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class LoCoMoAdapter(BenchmarkAdapter):
|
|
21
|
+
name = "locomo"
|
|
22
|
+
default_filename = "locomo10.json"
|
|
23
|
+
|
|
24
|
+
def load_cases(self, path: Path) -> Iterable[BenchmarkCase]:
|
|
25
|
+
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
26
|
+
if not isinstance(payload, list):
|
|
27
|
+
raise ValueError("LoCoMo dataset root must be a JSON list")
|
|
28
|
+
for sample_index, sample in enumerate(payload):
|
|
29
|
+
sample_id = str(sample.get("sample_id") or sample_index)
|
|
30
|
+
questions = tuple(
|
|
31
|
+
BenchmarkQuestion(
|
|
32
|
+
id=f"{sample_id}:q{index}",
|
|
33
|
+
question=str(item.get("question") or ""),
|
|
34
|
+
gold=str(item.get("answer") or ""),
|
|
35
|
+
evidence=tuple(str(value) for value in item.get("evidence") or []),
|
|
36
|
+
metadata={"category": item.get("category")},
|
|
37
|
+
)
|
|
38
|
+
for index, item in enumerate(sample.get("qa") or [])
|
|
39
|
+
)
|
|
40
|
+
yield BenchmarkCase(id=sample_id, payload=sample, questions=questions)
|
|
41
|
+
|
|
42
|
+
def ingest_case(self, engine: MemoryEngine, case: BenchmarkCase) -> PreparedCase:
|
|
43
|
+
namespace = MemoryNamespace(project_id=f"benchmark:locomo:{case.id}")
|
|
44
|
+
conversation = case.payload.get("conversation") or {}
|
|
45
|
+
episode_ids: list[str] = []
|
|
46
|
+
for key, turns in conversation.items():
|
|
47
|
+
if not key.startswith("session_") or key.endswith("_date_time") or not isinstance(turns, list):
|
|
48
|
+
continue
|
|
49
|
+
episode_id = f"locomo:{case.id}:{key}"
|
|
50
|
+
episode_ids.append(episode_id)
|
|
51
|
+
observed = conversation.get(f"{key}_date_time")
|
|
52
|
+
for index, turn in enumerate(turns):
|
|
53
|
+
text = f"{turn.get('speaker', 'speaker')}: {turn.get('text', '')}"
|
|
54
|
+
if observed:
|
|
55
|
+
text = f"[{observed}] {text}"
|
|
56
|
+
engine.ingest_observation(
|
|
57
|
+
text,
|
|
58
|
+
episode_id=episode_id,
|
|
59
|
+
source_ref={"type": "locomo", "sample_id": case.id},
|
|
60
|
+
locator={"dia_id": turn.get("dia_id"), "turn_index": index},
|
|
61
|
+
external_key=f"locomo:{case.id}:{turn.get('dia_id') or key + ':' + str(index)}",
|
|
62
|
+
namespace=namespace,
|
|
63
|
+
process=False,
|
|
64
|
+
schedule=False,
|
|
65
|
+
)
|
|
66
|
+
return PreparedCase(tuple(episode_ids), namespace)
|
|
67
|
+
|
|
68
|
+
def result_metadata(
|
|
69
|
+
self, question: BenchmarkQuestion, result: MemoryQueryResult
|
|
70
|
+
) -> dict[str, Any]:
|
|
71
|
+
cited = {
|
|
72
|
+
str(citation.locator.get("dia_id"))
|
|
73
|
+
for citation in result.citations
|
|
74
|
+
if isinstance(citation.locator, dict) and citation.locator.get("dia_id")
|
|
75
|
+
}
|
|
76
|
+
evidence = set(question.evidence or ())
|
|
77
|
+
return {
|
|
78
|
+
**question.metadata,
|
|
79
|
+
"evidence_hit": bool(evidence & cited),
|
|
80
|
+
"cited_ids": sorted(cited),
|
|
81
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""LongMemEval dataset adapter."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from collections.abc import Iterable
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from trisynapse_memory.adapters.benchmarks.base import (
|
|
11
|
+
BenchmarkAdapter,
|
|
12
|
+
BenchmarkCase,
|
|
13
|
+
BenchmarkQuestion,
|
|
14
|
+
PreparedCase,
|
|
15
|
+
)
|
|
16
|
+
from trisynapse_memory.engine import MemoryEngine
|
|
17
|
+
from trisynapse_memory.engine.models import MemoryNamespace, MemoryQueryResult
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class LongMemEvalAdapter(BenchmarkAdapter):
|
|
21
|
+
name = "longmemeval"
|
|
22
|
+
default_filename = "longmemeval_s_cleaned.json"
|
|
23
|
+
|
|
24
|
+
def load_cases(self, path: Path) -> Iterable[BenchmarkCase]:
|
|
25
|
+
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
26
|
+
if not isinstance(payload, list):
|
|
27
|
+
raise ValueError("LongMemEval dataset root must be a JSON list")
|
|
28
|
+
for index, item in enumerate(payload):
|
|
29
|
+
question_id = str(item.get("question_id") or index)
|
|
30
|
+
question = BenchmarkQuestion(
|
|
31
|
+
id=question_id,
|
|
32
|
+
question=str(item.get("question") or ""),
|
|
33
|
+
gold=str(item.get("answer") or ""),
|
|
34
|
+
evidence=tuple(str(value) for value in item.get("answer_session_ids") or []),
|
|
35
|
+
evidence_text=str(item.get("answer") or ""),
|
|
36
|
+
metadata={"question_type": item.get("question_type")},
|
|
37
|
+
)
|
|
38
|
+
yield BenchmarkCase(id=question_id, payload=item, questions=(question,))
|
|
39
|
+
|
|
40
|
+
def ingest_case(self, engine: MemoryEngine, case: BenchmarkCase) -> PreparedCase:
|
|
41
|
+
namespace = MemoryNamespace(project_id=f"benchmark:longmemeval:{case.id}")
|
|
42
|
+
episode_ids: list[str] = []
|
|
43
|
+
session_ids = case.payload.get("haystack_session_ids") or []
|
|
44
|
+
dates = case.payload.get("haystack_dates") or []
|
|
45
|
+
for session_index, turns in enumerate(case.payload.get("haystack_sessions") or []):
|
|
46
|
+
session_id = str(session_ids[session_index] if session_index < len(session_ids) else session_index)
|
|
47
|
+
episode_id = f"lme:{case.id}:{session_id}"
|
|
48
|
+
episode_ids.append(episode_id)
|
|
49
|
+
date = dates[session_index] if session_index < len(dates) else None
|
|
50
|
+
messages = []
|
|
51
|
+
for turn_index, turn in enumerate(turns):
|
|
52
|
+
content = str(turn.get("content") or "")
|
|
53
|
+
if date:
|
|
54
|
+
content = f"[{date}] {content}"
|
|
55
|
+
messages.append({
|
|
56
|
+
"id": f"{session_id}:{turn_index}",
|
|
57
|
+
"role": turn.get("role"),
|
|
58
|
+
"content": content,
|
|
59
|
+
})
|
|
60
|
+
engine.ingest_messages(messages, episode_id=episode_id, namespace=namespace)
|
|
61
|
+
return PreparedCase(tuple(episode_ids), namespace, episode_prefix=f"lme:{case.id}:")
|
|
62
|
+
|
|
63
|
+
def result_metadata(
|
|
64
|
+
self, question: BenchmarkQuestion, result: MemoryQueryResult
|
|
65
|
+
) -> dict[str, Any]:
|
|
66
|
+
prefix = f"lme:{question.id}:"
|
|
67
|
+
cited_sessions = {
|
|
68
|
+
str(citation.source_ref.get("id", "")).removeprefix(prefix)
|
|
69
|
+
for citation in result.citations
|
|
70
|
+
if isinstance(citation.source_ref, dict)
|
|
71
|
+
}
|
|
72
|
+
return {
|
|
73
|
+
**question.metadata,
|
|
74
|
+
"evidence_hit": bool(set(question.evidence or ()) & cited_sessions),
|
|
75
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""MemoryDoc micro-world dataset adapter."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from collections.abc import Iterable
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from trisynapse_memory.adapters.benchmarks.base import (
|
|
10
|
+
BenchmarkAdapter,
|
|
11
|
+
BenchmarkCase,
|
|
12
|
+
BenchmarkQuestion,
|
|
13
|
+
PreparedCase,
|
|
14
|
+
)
|
|
15
|
+
from trisynapse_memory.engine import MemoryEngine
|
|
16
|
+
from trisynapse_memory.engine.formation import chunk_document
|
|
17
|
+
from trisynapse_memory.engine.models import MemoryNamespace
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class MemoryDocAdapter(BenchmarkAdapter):
|
|
21
|
+
name = "memorydoc"
|
|
22
|
+
default_filename = "fixtures/smoke_micro_world.json"
|
|
23
|
+
|
|
24
|
+
def load_cases(self, path: Path) -> Iterable[BenchmarkCase]:
|
|
25
|
+
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
26
|
+
worlds = payload if isinstance(payload, list) else payload.get("micro_worlds") or payload.get("worlds") or [payload]
|
|
27
|
+
for world_index, world in enumerate(worlds):
|
|
28
|
+
world_id = str(world.get("micro_world_id") or world.get("world_id") or world.get("id") or world_index)
|
|
29
|
+
questions = []
|
|
30
|
+
for index, item in enumerate(world.get("qa_pairs") or world.get("questions") or []):
|
|
31
|
+
evidence = item.get("evidence_references") or item.get("evidence") or []
|
|
32
|
+
evidence_text = " ".join(
|
|
33
|
+
str(ref.get("passage_span") or ref.get("excerpt") or "")
|
|
34
|
+
if isinstance(ref, dict) else str(ref)
|
|
35
|
+
for ref in evidence
|
|
36
|
+
)
|
|
37
|
+
questions.append(BenchmarkQuestion(
|
|
38
|
+
id=f"{world_id}:q{index}",
|
|
39
|
+
question=str(item.get("question") or ""),
|
|
40
|
+
gold=str(item.get("gold_answer") or item.get("answer") or ""),
|
|
41
|
+
evidence=evidence,
|
|
42
|
+
evidence_text=evidence_text,
|
|
43
|
+
))
|
|
44
|
+
yield BenchmarkCase(id=world_id, payload=world, questions=tuple(questions))
|
|
45
|
+
|
|
46
|
+
def ingest_case(self, engine: MemoryEngine, case: BenchmarkCase) -> PreparedCase:
|
|
47
|
+
namespace = MemoryNamespace(project_id=f"benchmark:memorydoc:{case.id}")
|
|
48
|
+
prefix = f"mwd:{case.id}:"
|
|
49
|
+
episode_ids: list[str] = []
|
|
50
|
+
for document in case.payload.get("documents") or []:
|
|
51
|
+
document_id = str(document.get("document_id") or document.get("id"))
|
|
52
|
+
episode_id = f"{prefix}doc:{document_id}"
|
|
53
|
+
episode_ids.append(episode_id)
|
|
54
|
+
text = str(document.get("text") or document.get("content") or "")
|
|
55
|
+
for chunk_index, chunk in enumerate(chunk_document(text, chunk_chars=3500)):
|
|
56
|
+
engine.ingest_observation(
|
|
57
|
+
chunk,
|
|
58
|
+
episode_id=episode_id,
|
|
59
|
+
source_ref={"type": "document", "id": document_id},
|
|
60
|
+
locator={"kind": "chunk", "index": chunk_index},
|
|
61
|
+
external_key=f"mwd:{case.id}:doc:{document_id}:{chunk_index}",
|
|
62
|
+
namespace=namespace,
|
|
63
|
+
process=False,
|
|
64
|
+
schedule=False,
|
|
65
|
+
)
|
|
66
|
+
for session in case.payload.get("conversations") or case.payload.get("sessions") or []:
|
|
67
|
+
session_id = str(session.get("session_id") or session.get("event_id") or session.get("id"))
|
|
68
|
+
episode_id = f"{prefix}chat:{session_id}"
|
|
69
|
+
episode_ids.append(episode_id)
|
|
70
|
+
utterances = session.get("utterances") or session.get("messages") or []
|
|
71
|
+
engine.ingest_messages(
|
|
72
|
+
[
|
|
73
|
+
{
|
|
74
|
+
"id": utterance.get("utterance_id") or utterance.get("id") or index,
|
|
75
|
+
"role": utterance.get("speaker") or utterance.get("persona") or utterance.get("role") or "speaker",
|
|
76
|
+
"content": utterance.get("text") or utterance.get("content") or "",
|
|
77
|
+
}
|
|
78
|
+
for index, utterance in enumerate(utterances)
|
|
79
|
+
],
|
|
80
|
+
episode_id=episode_id,
|
|
81
|
+
namespace=namespace,
|
|
82
|
+
)
|
|
83
|
+
return PreparedCase(tuple(episode_ids), namespace, episode_prefix=prefix)
|