portable-memory 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- portable_memory/__init__.py +78 -0
- portable_memory/_codec.py +189 -0
- portable_memory/adapters/__init__.py +15 -0
- portable_memory/adapters/claude.py +162 -0
- portable_memory/adapters/mem0.py +247 -0
- portable_memory/adapters/openai.py +227 -0
- portable_memory/exporter.py +360 -0
- portable_memory/format.py +135 -0
- portable_memory/hashing.py +28 -0
- portable_memory/importer.py +420 -0
- portable_memory/interop.py +51 -0
- portable_memory/records.py +199 -0
- portable_memory/signing.py +182 -0
- portable_memory/store.py +249 -0
- portable_memory/tombstone.py +70 -0
- portable_memory/validator.py +204 -0
- portable_memory-0.1.2.dist-info/METADATA +256 -0
- portable_memory-0.1.2.dist-info/RECORD +20 -0
- portable_memory-0.1.2.dist-info/WHEEL +4 -0
- portable_memory-0.1.2.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Portable Memory — an open, vendor-neutral format for AI memory portability.
|
|
2
|
+
|
|
3
|
+
Python reference SDK. The on-disk format is defined by ``Spec/portable-memory-spec.md``
|
|
4
|
+
and the language-neutral ``Schemas/``; this package is byte-for-byte interoperable with
|
|
5
|
+
the Swift reference SDK (github.com/MacPaw/portable-memory-swift).
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from ._codec import (
|
|
10
|
+
canonical_json,
|
|
11
|
+
format_timestamp,
|
|
12
|
+
from_wire,
|
|
13
|
+
parse_timestamp,
|
|
14
|
+
to_line,
|
|
15
|
+
to_wire,
|
|
16
|
+
)
|
|
17
|
+
from .format import (
|
|
18
|
+
IMPORT_ORDER,
|
|
19
|
+
BundlePath,
|
|
20
|
+
ConformanceLevel,
|
|
21
|
+
ExportMode,
|
|
22
|
+
MemFileEntry,
|
|
23
|
+
MemFormat,
|
|
24
|
+
MemKind,
|
|
25
|
+
MemLimits,
|
|
26
|
+
MemManifest,
|
|
27
|
+
)
|
|
28
|
+
from .hashing import embedding_cache_key, sha256_hex, sha256_hex_str
|
|
29
|
+
from .interop import KNOWN_EPISODE_KEYS, extract_episode_ext, merge_episode_ext
|
|
30
|
+
from .records import (
|
|
31
|
+
MemImportReport,
|
|
32
|
+
PortableCategory,
|
|
33
|
+
PortableChunk,
|
|
34
|
+
PortableCommunity,
|
|
35
|
+
PortableContext,
|
|
36
|
+
PortableCore,
|
|
37
|
+
PortableEdge,
|
|
38
|
+
PortableEntity,
|
|
39
|
+
PortableEpisode,
|
|
40
|
+
PortableEpisodeLink,
|
|
41
|
+
PortableFact,
|
|
42
|
+
PortableFactLink,
|
|
43
|
+
PortablePreference,
|
|
44
|
+
PortableProcedure,
|
|
45
|
+
PortableResource,
|
|
46
|
+
PortableSecretRef,
|
|
47
|
+
)
|
|
48
|
+
from .tombstone import DerivedRefs, PortableAuditRecord, Tombstone, TombstoneOp
|
|
49
|
+
|
|
50
|
+
__version__ = "0.1.0"
|
|
51
|
+
|
|
52
|
+
# Logic modules.
|
|
53
|
+
from .store import PortableMemoryStore, StoreInfo # noqa: E402
|
|
54
|
+
from .signing import ( # noqa: E402
|
|
55
|
+
PortableSigning,
|
|
56
|
+
PortableSigningKey,
|
|
57
|
+
PortableVerifyingKey,
|
|
58
|
+
)
|
|
59
|
+
from .validator import BundleValidator, ValidationResult # noqa: E402
|
|
60
|
+
from .exporter import BundleExporter # noqa: E402
|
|
61
|
+
from .importer import BundleImporter, MemImportError # noqa: E402
|
|
62
|
+
|
|
63
|
+
__all__ = [
|
|
64
|
+
"canonical_json", "format_timestamp", "parse_timestamp", "to_wire", "from_wire", "to_line",
|
|
65
|
+
"MemFormat", "MemKind", "IMPORT_ORDER", "ExportMode", "ConformanceLevel",
|
|
66
|
+
"MemFileEntry", "MemManifest", "MemLimits", "BundlePath",
|
|
67
|
+
"sha256_hex", "sha256_hex_str", "embedding_cache_key",
|
|
68
|
+
"KNOWN_EPISODE_KEYS", "extract_episode_ext", "merge_episode_ext",
|
|
69
|
+
"PortableEpisode", "PortableEntity", "PortableEdge", "PortableFact", "PortableFactLink",
|
|
70
|
+
"PortableEpisodeLink", "PortableResource", "PortableChunk", "PortableCore",
|
|
71
|
+
"PortableProcedure", "PortableContext", "PortableCommunity", "PortableCategory",
|
|
72
|
+
"PortablePreference", "PortableSecretRef", "MemImportReport",
|
|
73
|
+
"DerivedRefs", "TombstoneOp", "Tombstone", "PortableAuditRecord",
|
|
74
|
+
"PortableMemoryStore", "StoreInfo",
|
|
75
|
+
"PortableSigning", "PortableSigningKey", "PortableVerifyingKey",
|
|
76
|
+
"BundleValidator", "ValidationResult",
|
|
77
|
+
"BundleExporter", "BundleImporter", "MemImportError",
|
|
78
|
+
]
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"""Canonical JSON + dataclass <-> wire-dict codec.
|
|
2
|
+
|
|
3
|
+
The canonical form is byte-identical to the Swift reference SDK's ``MemCodec`` (spec
|
|
4
|
+
§1.1): UTF-8, no BOM, object keys sorted recursively, no insignificant whitespace,
|
|
5
|
+
``/`` unescaped, non-ASCII emitted raw (not ``\\u``), shortest round-tripping numbers,
|
|
6
|
+
and whole-second UTC ``Z`` timestamps. That is what lets a bundle written by one
|
|
7
|
+
implementation validate (matching checksums) in the other.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import types
|
|
13
|
+
from dataclasses import fields, is_dataclass
|
|
14
|
+
from datetime import datetime, timezone
|
|
15
|
+
from enum import Enum
|
|
16
|
+
from typing import Any, Union, get_args, get_origin, get_type_hints
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
# Integral doubles below this bound serialize as plain integers ("1", not "1.0" —
|
|
20
|
+
# spec §1.1); at/above it both reference encoders switch to exponent form ("1e+16"),
|
|
21
|
+
# which Python's repr already produces. Probed against the Swift encoder: the two
|
|
22
|
+
# agree byte-for-byte on either side of this boundary.
|
|
23
|
+
_INTEGRAL_FLOAT_BOUND = 1e16
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _canon_numbers(obj: Any) -> Any:
|
|
27
|
+
"""Normalize floats to the canonical number form (spec §1.1) before dumping.
|
|
28
|
+
|
|
29
|
+
``json.dumps`` renders ``1.0`` as ``"1.0"``, but Canonical JSON (ECMAScript /
|
|
30
|
+
JCS shortest-round-trip rule) requires integral values to emit with no decimal
|
|
31
|
+
point — ``"1"``. Convert integral floats (including ``-0.0`` → ``0``) to ``int``;
|
|
32
|
+
``bool`` is untouched (it is an ``int`` subclass, never a ``float``).
|
|
33
|
+
"""
|
|
34
|
+
if isinstance(obj, float):
|
|
35
|
+
if obj.is_integer() and abs(obj) < _INTEGRAL_FLOAT_BOUND:
|
|
36
|
+
return int(obj)
|
|
37
|
+
return obj
|
|
38
|
+
if isinstance(obj, dict):
|
|
39
|
+
return {k: _canon_numbers(v) for k, v in obj.items()}
|
|
40
|
+
if isinstance(obj, (list, tuple)):
|
|
41
|
+
return [_canon_numbers(v) for v in obj]
|
|
42
|
+
return obj
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def canonical_json(obj: Any) -> str:
|
|
46
|
+
"""Serialize one value to a single canonical JSON line (no trailing newline)."""
|
|
47
|
+
# sort_keys -> sorted keys; separators -> no whitespace; ensure_ascii=False -> raw
|
|
48
|
+
# UTF-8 (and Python never escapes '/'); floats use repr = shortest round-trip;
|
|
49
|
+
# allow_nan=False -> NaN/Infinity raise instead of emitting invalid RFC 8259 JSON.
|
|
50
|
+
return json.dumps(_canon_numbers(obj), ensure_ascii=False, sort_keys=True,
|
|
51
|
+
separators=(",", ":"), allow_nan=False)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def format_timestamp(dt: datetime) -> str:
|
|
55
|
+
"""Whole-second UTC RFC 3339 with a literal ``Z`` (canonical output, spec §1.1)."""
|
|
56
|
+
if dt.tzinfo is None:
|
|
57
|
+
dt = dt.replace(tzinfo=timezone.utc)
|
|
58
|
+
dt = dt.astimezone(timezone.utc).replace(microsecond=0)
|
|
59
|
+
return dt.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def parse_timestamp(s: str) -> datetime:
|
|
63
|
+
"""Lenient RFC 3339 parse: accepts ``Z``, numeric offsets, and fractional seconds.
|
|
64
|
+
|
|
65
|
+
Canonical output is always whole-second UTC, but a foreign bundle may emit other
|
|
66
|
+
forms — a reader accepts them and re-emits canonically on the next export.
|
|
67
|
+
"""
|
|
68
|
+
raw = s
|
|
69
|
+
if s[-1:] in ("Z", "z"):
|
|
70
|
+
s = s[:-1] + "+00:00"
|
|
71
|
+
try:
|
|
72
|
+
dt = datetime.fromisoformat(s)
|
|
73
|
+
except ValueError as exc:
|
|
74
|
+
raise ValueError(f"not a valid RFC 3339 timestamp: {raw}") from exc
|
|
75
|
+
if dt.tzinfo is None:
|
|
76
|
+
dt = dt.replace(tzinfo=timezone.utc)
|
|
77
|
+
return dt.astimezone(timezone.utc)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _json_key(field) -> str:
|
|
81
|
+
return field.metadata.get("json", field.name)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _unwrap_optional(hint: Any) -> Any:
|
|
85
|
+
"""T from Optional[T] / T | None; hint unchanged otherwise."""
|
|
86
|
+
origin = get_origin(hint)
|
|
87
|
+
if origin is Union or origin is types.UnionType:
|
|
88
|
+
non_none = [a for a in get_args(hint) if a is not type(None)]
|
|
89
|
+
if len(non_none) == 1:
|
|
90
|
+
return non_none[0]
|
|
91
|
+
return hint
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _encode_value(value: Any) -> Any:
|
|
95
|
+
if value is None:
|
|
96
|
+
return None
|
|
97
|
+
if isinstance(value, datetime):
|
|
98
|
+
return format_timestamp(value)
|
|
99
|
+
if isinstance(value, Enum):
|
|
100
|
+
return value.value
|
|
101
|
+
if is_dataclass(value):
|
|
102
|
+
return to_wire(value)
|
|
103
|
+
if isinstance(value, (list, tuple)):
|
|
104
|
+
return [_encode_value(v) for v in value]
|
|
105
|
+
if isinstance(value, dict):
|
|
106
|
+
return {k: _encode_value(v) for k, v in value.items()}
|
|
107
|
+
return value # str / int / float / bool
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def to_wire(obj: Any) -> dict:
|
|
111
|
+
"""Dataclass -> wire dict (camelCase keys; absent optionals omitted, never null)."""
|
|
112
|
+
out: dict[str, Any] = {}
|
|
113
|
+
for f in fields(obj):
|
|
114
|
+
val = getattr(obj, f.name)
|
|
115
|
+
if val is None:
|
|
116
|
+
continue
|
|
117
|
+
out[_json_key(f)] = _encode_value(val)
|
|
118
|
+
return out
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _decode_value(value: Any, hint: Any) -> Any:
|
|
122
|
+
inner = _unwrap_optional(hint)
|
|
123
|
+
origin = get_origin(inner)
|
|
124
|
+
if inner is datetime:
|
|
125
|
+
if not isinstance(value, str):
|
|
126
|
+
raise TypeError(f"expected RFC 3339 string, got {type(value).__name__}")
|
|
127
|
+
return parse_timestamp(value)
|
|
128
|
+
if isinstance(inner, type) and issubclass(inner, Enum):
|
|
129
|
+
return inner(value)
|
|
130
|
+
if is_dataclass(inner):
|
|
131
|
+
return from_wire(inner, value)
|
|
132
|
+
if origin in (list, tuple):
|
|
133
|
+
if not isinstance(value, list):
|
|
134
|
+
raise TypeError(f"expected array, got {type(value).__name__}")
|
|
135
|
+
args = get_args(inner)
|
|
136
|
+
elem = args[0] if args else Any
|
|
137
|
+
return [_decode_value(v, elem) for v in value]
|
|
138
|
+
if origin is dict:
|
|
139
|
+
if not isinstance(value, dict):
|
|
140
|
+
raise TypeError(f"expected object, got {type(value).__name__}")
|
|
141
|
+
args = get_args(inner)
|
|
142
|
+
val_hint = args[1] if len(args) == 2 else Any
|
|
143
|
+
return {k: _decode_value(v, val_hint) for k, v in value.items()}
|
|
144
|
+
# Strict scalars — a wrong-typed field must fail the decode (as Swift's Codable
|
|
145
|
+
# does), not be silently absorbed and re-emitted through the canonical exporter.
|
|
146
|
+
if inner is str:
|
|
147
|
+
if not isinstance(value, str):
|
|
148
|
+
raise TypeError(f"expected string, got {type(value).__name__}")
|
|
149
|
+
return value
|
|
150
|
+
if inner is bool:
|
|
151
|
+
if not isinstance(value, bool):
|
|
152
|
+
raise TypeError(f"expected boolean, got {type(value).__name__}")
|
|
153
|
+
return value
|
|
154
|
+
if inner is int:
|
|
155
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
156
|
+
raise TypeError(f"expected integer, got {type(value).__name__}")
|
|
157
|
+
# Integral floats are accepted for integer fields (Swift's decoder does the
|
|
158
|
+
# same): canonical form writes 2.0 as "2", so a reader must take "2.0" too.
|
|
159
|
+
if isinstance(value, float):
|
|
160
|
+
if not value.is_integer():
|
|
161
|
+
raise TypeError("expected integer, got non-integral number")
|
|
162
|
+
return int(value)
|
|
163
|
+
return value
|
|
164
|
+
if inner is float:
|
|
165
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
166
|
+
raise TypeError(f"expected number, got {type(value).__name__}")
|
|
167
|
+
# Canonical form writes 1.0 as "1", so integer tokens must decode into
|
|
168
|
+
# float fields.
|
|
169
|
+
return float(value)
|
|
170
|
+
return value
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def from_wire(cls: type, data: dict) -> Any:
|
|
174
|
+
"""Wire dict -> dataclass. Absent keys fall back to the field default."""
|
|
175
|
+
hints = get_type_hints(cls)
|
|
176
|
+
kwargs = {}
|
|
177
|
+
for f in fields(cls):
|
|
178
|
+
key = _json_key(f)
|
|
179
|
+
if key in data and data[key] is not None:
|
|
180
|
+
try:
|
|
181
|
+
kwargs[f.name] = _decode_value(data[key], hints[f.name])
|
|
182
|
+
except (TypeError, ValueError) as exc:
|
|
183
|
+
raise type(exc)(f"{cls.__name__}.{key}: {exc}") from exc
|
|
184
|
+
return cls(**kwargs)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def to_line(obj: Any) -> str:
|
|
188
|
+
"""Dataclass (or plain value) -> one canonical JSON line."""
|
|
189
|
+
return canonical_json(to_wire(obj) if is_dataclass(obj) else obj)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Vendor ingest adapters (cross-provider import).
|
|
2
|
+
|
|
3
|
+
Each adapter maps a foreign export onto portable episodes; whatever it doesn't model is
|
|
4
|
+
preserved in ``metadata`` (namespaced) so a later ``.mem`` export stays lossless. Every
|
|
5
|
+
adapter exposes ``parse_episodes(...)`` (input shape varies by source) returning
|
|
6
|
+
``list[PortableEpisode]``.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from .claude import ClaudeAdapter
|
|
11
|
+
from .mem0 import Mem0Adapter
|
|
12
|
+
from .mem0 import parse_episodes # back-compat: the bare name is the original mem0 adapter
|
|
13
|
+
from .openai import OpenAIAdapter
|
|
14
|
+
|
|
15
|
+
__all__ = ["Mem0Adapter", "OpenAIAdapter", "ClaudeAdapter", "parse_episodes"]
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""Anthropic / Claude memory ingest adapter.
|
|
2
|
+
|
|
3
|
+
Claude's memory is a *directory of files* — a ``MEMORY.md`` entrypoint plus topic files
|
|
4
|
+
(markdown, commonly with YAML frontmatter), as used by the Claude Code auto-memory and the
|
|
5
|
+
API memory tool. This adapter maps each memory file onto one portable episode: the file
|
|
6
|
+
body becomes the episode text, and the frontmatter (``name``, ``description``,
|
|
7
|
+
``metadata.type``) plus the file path are preserved into ``metadata`` (namespaced
|
|
8
|
+
``claude_*``) so a later ``.mem`` export stays lossless.
|
|
9
|
+
|
|
10
|
+
Input is the memory files themselves (not JSON), since Claude leaves the storage format to
|
|
11
|
+
the host. Accepted shapes (tolerant):
|
|
12
|
+
* ``list`` of ``{"path": str, "content": str}`` (``name``/``text``/``body`` also accepted);
|
|
13
|
+
* ``list`` of ``(path, content)`` tuples;
|
|
14
|
+
* ``dict`` mapping ``path -> content``.
|
|
15
|
+
|
|
16
|
+
Adapters are pure: files in, ``list[PortableEpisode]`` out. No host, store, or I/O.
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import os
|
|
21
|
+
import uuid
|
|
22
|
+
from datetime import datetime, timezone
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
from ..records import PortableEpisode
|
|
26
|
+
|
|
27
|
+
__all__ = ["ClaudeAdapter", "parse_episodes"]
|
|
28
|
+
|
|
29
|
+
_INDEX_FILES = {"MEMORY.md", "CLAUDE.md"}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class ClaudeAdapter:
|
|
33
|
+
"""Maps a set of Claude memory files onto portable episodes."""
|
|
34
|
+
|
|
35
|
+
@staticmethod
|
|
36
|
+
def parse_episodes(files: Any) -> list[PortableEpisode]:
|
|
37
|
+
episodes: list[PortableEpisode] = []
|
|
38
|
+
for path, content in _iter_files(files):
|
|
39
|
+
ep = _map_file(path, content)
|
|
40
|
+
if ep is not None:
|
|
41
|
+
episodes.append(ep)
|
|
42
|
+
return episodes
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _iter_files(files: Any):
|
|
46
|
+
"""Yield (path, content) from any accepted input shape."""
|
|
47
|
+
if isinstance(files, dict):
|
|
48
|
+
for path, content in files.items():
|
|
49
|
+
if isinstance(path, str) and isinstance(content, str):
|
|
50
|
+
yield path, content
|
|
51
|
+
return
|
|
52
|
+
if isinstance(files, (list, tuple)):
|
|
53
|
+
for f in files:
|
|
54
|
+
if isinstance(f, dict):
|
|
55
|
+
path = _first_str(f.get("path"), f.get("name")) or ""
|
|
56
|
+
content = _first_str(f.get("content"), f.get("text"), f.get("body"))
|
|
57
|
+
if content is not None:
|
|
58
|
+
yield path, content
|
|
59
|
+
elif isinstance(f, (list, tuple)) and len(f) == 2 and isinstance(f[1], str):
|
|
60
|
+
yield (f[0] if isinstance(f[0], str) else ""), f[1]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _map_file(path: str, content: str) -> PortableEpisode | None:
|
|
64
|
+
frontmatter, body = _split_frontmatter(content)
|
|
65
|
+
text = (body or content).strip()
|
|
66
|
+
if not text:
|
|
67
|
+
return None
|
|
68
|
+
|
|
69
|
+
base = os.path.basename(path) if path else ""
|
|
70
|
+
name = frontmatter.get("name") or (base[:-3] if base.endswith(".md") else base) or None
|
|
71
|
+
description = frontmatter.get("description")
|
|
72
|
+
ctype = frontmatter.get("metadata.type") or frontmatter.get("type")
|
|
73
|
+
|
|
74
|
+
# Summary: the description, else the first non-empty line, else the name.
|
|
75
|
+
first_line = next((ln.strip() for ln in text.splitlines() if ln.strip()), "")
|
|
76
|
+
summary = (description or first_line or name or "")[:120]
|
|
77
|
+
|
|
78
|
+
meta: dict[str, str] = {}
|
|
79
|
+
_put(meta, "claude_path", path or None)
|
|
80
|
+
_put(meta, "claude_name", name)
|
|
81
|
+
_put(meta, "claude_type", ctype)
|
|
82
|
+
_put(meta, "claude_description", description)
|
|
83
|
+
if base in _INDEX_FILES:
|
|
84
|
+
meta["claude_role"] = "index"
|
|
85
|
+
|
|
86
|
+
now = datetime.now(timezone.utc)
|
|
87
|
+
return PortableEpisode(
|
|
88
|
+
id=name or (base[:-3] if base.endswith(".md") else base) or ("ep_" + uuid.uuid4().hex[:16]),
|
|
89
|
+
event_time=now,
|
|
90
|
+
mention_time=now,
|
|
91
|
+
ingestion_time=now,
|
|
92
|
+
source_type="note",
|
|
93
|
+
source_id=path or None,
|
|
94
|
+
actors=[],
|
|
95
|
+
summary=summary,
|
|
96
|
+
details=text,
|
|
97
|
+
sensitivity="low",
|
|
98
|
+
deleted_at=None,
|
|
99
|
+
metadata=meta,
|
|
100
|
+
context_id=None,
|
|
101
|
+
categories=[ctype] if ctype else [],
|
|
102
|
+
importance=0.5,
|
|
103
|
+
confidence=0.7,
|
|
104
|
+
lifecycle_state="HOT",
|
|
105
|
+
extraction_state="done",
|
|
106
|
+
last_accessed=None,
|
|
107
|
+
access_count=0,
|
|
108
|
+
pinned=False,
|
|
109
|
+
expiration_date=None,
|
|
110
|
+
vault_refs=[],
|
|
111
|
+
speaker=None,
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _split_frontmatter(content: str) -> tuple[dict[str, str], str]:
|
|
116
|
+
"""Parse a leading ``---`` YAML frontmatter block into a flat dict (nested keys as
|
|
117
|
+
``parent.child``) + the remaining body. Returns ({}, content) when there is none.
|
|
118
|
+
|
|
119
|
+
A minimal, stdlib-only parser for the ``key: value`` subset the memory format uses
|
|
120
|
+
(Claude leaves the format to the host; the core stays dependency-free). Anything it
|
|
121
|
+
can't parse is left in the body, never dropped.
|
|
122
|
+
"""
|
|
123
|
+
lines = content.splitlines()
|
|
124
|
+
if not lines or lines[0].strip() != "---":
|
|
125
|
+
return {}, content
|
|
126
|
+
end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None)
|
|
127
|
+
if end is None:
|
|
128
|
+
return {}, content
|
|
129
|
+
|
|
130
|
+
fm: dict[str, str] = {}
|
|
131
|
+
parent: str | None = None
|
|
132
|
+
for raw in lines[1:end]:
|
|
133
|
+
if not raw.strip() or raw.lstrip().startswith("#"):
|
|
134
|
+
continue
|
|
135
|
+
indented = raw[:1] in (" ", "\t")
|
|
136
|
+
key, sep, val = raw.strip().partition(":")
|
|
137
|
+
if not sep:
|
|
138
|
+
continue
|
|
139
|
+
key, val = key.strip(), val.strip()
|
|
140
|
+
if indented and parent is not None:
|
|
141
|
+
fm[f"{parent}.{key}"] = val
|
|
142
|
+
elif val == "":
|
|
143
|
+
parent = key # a nested block follows
|
|
144
|
+
else:
|
|
145
|
+
fm[key] = val
|
|
146
|
+
parent = None
|
|
147
|
+
return fm, "\n".join(lines[end + 1:])
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _first_str(*values: Any) -> str | None:
|
|
151
|
+
for v in values:
|
|
152
|
+
if isinstance(v, str):
|
|
153
|
+
return v
|
|
154
|
+
return None
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _put(meta: dict[str, str], key: str, value: str | None) -> None:
|
|
158
|
+
if value:
|
|
159
|
+
meta[key] = value
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
parse_episodes = ClaudeAdapter.parse_episodes
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"""mem0 (https://github.com/mem0ai/mem0) ingest adapter.
|
|
2
|
+
|
|
3
|
+
Mirrors the Swift ``Mem0Adapter`` (Sources/PortableMemory/Adapters/Mem0Adapter.swift):
|
|
4
|
+
maps a mem0 export — a bare JSON array, or ``{"results":[...]}`` / ``{"memories":[...]}``
|
|
5
|
+
/ ``{"data":[...]}`` (the platform's paginated envelope also wraps ``results``) — onto
|
|
6
|
+
portable episodes. The format is a superset container, so whatever mem0 models that the
|
|
7
|
+
portable schema doesn't is preserved in ``metadata`` (namespaced ``mem0_*``) rather than
|
|
8
|
+
dropped; a later ``.mem`` export stays lossless. That includes a generic sweep of any
|
|
9
|
+
top-level key the mapping doesn't recognize (``score``, ``immutable``, ``memory_type``,
|
|
10
|
+
future platform fields, …).
|
|
11
|
+
|
|
12
|
+
mem0's promoted per-memory keys (OSS ``promoted_payload_keys``) are ``user_id``,
|
|
13
|
+
``agent_id``, ``run_id``, ``actor_id``, ``role``, ``attributed_to``, and
|
|
14
|
+
``expiration_date`` — all read here; ``expiration_date`` (normalized ``YYYY-MM-DD``)
|
|
15
|
+
additionally maps onto the episode's own ``expiration_date``. Graph ``relations``
|
|
16
|
+
(returned alongside ``results`` when graph memory is enabled) are NOT mapped in v1 —
|
|
17
|
+
adapters return episodes only; entity/edge promotion is a possible follow-up.
|
|
18
|
+
|
|
19
|
+
Adapters are pure: they take foreign bytes/JSON and return ``list[PortableEpisode]``
|
|
20
|
+
that a host then imports. There is no host, store, or I/O here.
|
|
21
|
+
"""
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import json
|
|
25
|
+
import uuid
|
|
26
|
+
from datetime import datetime, timezone
|
|
27
|
+
from typing import Any, Callable
|
|
28
|
+
|
|
29
|
+
from .._codec import canonical_json, parse_timestamp
|
|
30
|
+
from ..records import PortableEpisode
|
|
31
|
+
|
|
32
|
+
__all__ = ["Mem0Adapter", "parse_episodes"]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class Mem0Adapter:
|
|
36
|
+
"""Maps a mem0 export onto portable episodes.
|
|
37
|
+
|
|
38
|
+
Missing/extra fields are handled gracefully; the original ``id``, ``hash``, role,
|
|
39
|
+
and the various actor ids are preserved into episode ``metadata``.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
@staticmethod
|
|
43
|
+
def parse_episodes(data: bytes | str | dict) -> list[PortableEpisode]:
|
|
44
|
+
"""Parse a mem0 export into portable episodes.
|
|
45
|
+
|
|
46
|
+
Accepts the already-decoded ``dict`` (or ``list``) as well as raw ``bytes`` /
|
|
47
|
+
``str`` JSON — the Swift API takes ``Data`` only, but Python callers routinely
|
|
48
|
+
hold a parsed object, so we accept both without changing the mapping.
|
|
49
|
+
"""
|
|
50
|
+
if isinstance(data, (bytes, bytearray, str)):
|
|
51
|
+
root: Any = json.loads(data)
|
|
52
|
+
else:
|
|
53
|
+
root = data
|
|
54
|
+
|
|
55
|
+
# A bare array, or one of the known envelope keys wrapping the array. Anything
|
|
56
|
+
# else (or a missing/mistyped envelope) yields no items rather than raising —
|
|
57
|
+
# mirrors the Swift adapter's tolerant shape handling.
|
|
58
|
+
if isinstance(root, list):
|
|
59
|
+
items = root
|
|
60
|
+
elif isinstance(root, dict):
|
|
61
|
+
items = (
|
|
62
|
+
_as_object_list(root.get("results"))
|
|
63
|
+
or _as_object_list(root.get("memories"))
|
|
64
|
+
or _as_object_list(root.get("data"))
|
|
65
|
+
)
|
|
66
|
+
else:
|
|
67
|
+
items = []
|
|
68
|
+
|
|
69
|
+
# Swift builds the ISO-8601 parsers once per call for large exports; the
|
|
70
|
+
# foundation's ``parse_timestamp`` is cheap, so we just wrap it in a lenient
|
|
71
|
+
# closure that returns ``None`` for missing/unparseable input (Swift returns nil
|
|
72
|
+
# from the failed-both-formatters path, which the call sites already tolerate).
|
|
73
|
+
def parse(s: Any) -> datetime | None:
|
|
74
|
+
if not isinstance(s, str) or not s:
|
|
75
|
+
return None
|
|
76
|
+
try:
|
|
77
|
+
return parse_timestamp(s)
|
|
78
|
+
except ValueError:
|
|
79
|
+
return None
|
|
80
|
+
|
|
81
|
+
episodes: list[PortableEpisode] = []
|
|
82
|
+
for item in items:
|
|
83
|
+
if not isinstance(item, dict):
|
|
84
|
+
continue # compactMap drops non-object entries
|
|
85
|
+
ep = _map_one(item, parse)
|
|
86
|
+
if ep is not None:
|
|
87
|
+
episodes.append(ep)
|
|
88
|
+
return episodes
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _as_object_list(value: Any) -> list[dict]:
|
|
92
|
+
"""Return ``value`` when it's a list of dicts, else ``[]`` (mirrors the Swift
|
|
93
|
+
``as? [[String: Any]]`` casts, which fail-to-nil on a mistyped envelope)."""
|
|
94
|
+
if isinstance(value, list) and all(isinstance(e, dict) for e in value):
|
|
95
|
+
return value
|
|
96
|
+
return []
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
#: Top-level mem0 keys the mapping reads explicitly. Anything else is swept into
|
|
100
|
+
#: ``mem0_<key>`` metadata so no field a mem0 version emits is ever dropped.
|
|
101
|
+
_HANDLED_KEYS = frozenset({
|
|
102
|
+
"memory", "text", "data", "role", "user_id", "agent_id", "actor_id", "run_id",
|
|
103
|
+
"created_at", "updated_at", "metadata", "categories", "id", "hash",
|
|
104
|
+
"attributed_to", "expiration_date",
|
|
105
|
+
})
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _map_one(
|
|
109
|
+
o: dict,
|
|
110
|
+
parse: Callable[[Any], "datetime | None"],
|
|
111
|
+
) -> PortableEpisode | None:
|
|
112
|
+
"""Map one mem0 record onto a portable episode, or ``None`` when it carries no text."""
|
|
113
|
+
text = _first_str(o.get("memory"), o.get("text"), o.get("data")) or ""
|
|
114
|
+
if not text.strip():
|
|
115
|
+
return None # nothing to remember
|
|
116
|
+
|
|
117
|
+
role = _opt_str(o.get("role"))
|
|
118
|
+
user_id = _opt_str(o.get("user_id"))
|
|
119
|
+
agent_id = _opt_str(o.get("agent_id"))
|
|
120
|
+
actor_id = _opt_str(o.get("actor_id"))
|
|
121
|
+
run_id = _opt_str(o.get("run_id"))
|
|
122
|
+
|
|
123
|
+
created = parse(o.get("created_at")) or datetime.now(timezone.utc)
|
|
124
|
+
updated = parse(o.get("updated_at"))
|
|
125
|
+
# mem0 normalizes expiration_date to date-only "YYYY-MM-DD"; the lenient parser
|
|
126
|
+
# accepts it (midnight UTC). The raw string is also preserved in metadata below.
|
|
127
|
+
expiration = parse(o.get("expiration_date"))
|
|
128
|
+
|
|
129
|
+
# Actors in mem0's own precedence: user, then agent, then explicit actor. Only
|
|
130
|
+
# non-empty ids are carried.
|
|
131
|
+
actors = [a for a in (user_id, agent_id, actor_id) if a]
|
|
132
|
+
|
|
133
|
+
# Foreign metadata first (stringified), then mem0 provenance overlaid. Provenance
|
|
134
|
+
# keys only land when the value is a non-empty string, matching Swift.
|
|
135
|
+
meta: dict[str, str] = {}
|
|
136
|
+
raw_meta = o.get("metadata")
|
|
137
|
+
if isinstance(raw_meta, dict):
|
|
138
|
+
for k, v in raw_meta.items():
|
|
139
|
+
meta[k] = _stringify(v)
|
|
140
|
+
provenance = {
|
|
141
|
+
"mem0_id": o.get("id"),
|
|
142
|
+
"mem0_hash": o.get("hash"),
|
|
143
|
+
"mem0_role": role,
|
|
144
|
+
"mem0_user_id": user_id,
|
|
145
|
+
"mem0_agent_id": agent_id,
|
|
146
|
+
"mem0_actor_id": actor_id,
|
|
147
|
+
"mem0_run_id": run_id,
|
|
148
|
+
"mem0_created_at": o.get("created_at"),
|
|
149
|
+
"mem0_updated_at": o.get("updated_at"),
|
|
150
|
+
"mem0_attributed_to": o.get("attributed_to"),
|
|
151
|
+
"mem0_expiration_date": o.get("expiration_date"),
|
|
152
|
+
}
|
|
153
|
+
for k, v in provenance.items():
|
|
154
|
+
if isinstance(v, str) and v:
|
|
155
|
+
meta[k] = v
|
|
156
|
+
# Lossless sweep: any top-level key the mapping doesn't recognize (score, immutable,
|
|
157
|
+
# memory_type, future platform fields, ...) is preserved as mem0_<key>. Nulls are
|
|
158
|
+
# skipped — mem0 emits e.g. "expiration_date": null. setdefault so user metadata or
|
|
159
|
+
# provenance that already claimed a key is never overwritten.
|
|
160
|
+
for k, v in o.items():
|
|
161
|
+
if k not in _HANDLED_KEYS and v is not None:
|
|
162
|
+
meta.setdefault(f"mem0_{k}", _stringify(v))
|
|
163
|
+
|
|
164
|
+
# Mirror Swift's `as? [String]`: a non-list (e.g. a bare string — which Python
|
|
165
|
+
# would happily iterate char-by-char) or a list with non-string members maps to [].
|
|
166
|
+
raw_categories = o.get("categories", [])
|
|
167
|
+
categories = (
|
|
168
|
+
raw_categories
|
|
169
|
+
if isinstance(raw_categories, list) and all(isinstance(c, str) for c in raw_categories)
|
|
170
|
+
else []
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
raw_id = _opt_str(o.get("id"))
|
|
174
|
+
episode_id = raw_id or ("ep_" + uuid.uuid4().hex[:16])
|
|
175
|
+
|
|
176
|
+
return PortableEpisode(
|
|
177
|
+
id=episode_id,
|
|
178
|
+
event_time=created,
|
|
179
|
+
mention_time=updated or created,
|
|
180
|
+
ingestion_time=created,
|
|
181
|
+
# A message with a role came from a conversation; anything else is free text.
|
|
182
|
+
source_type="chat" if role is not None else "text",
|
|
183
|
+
source_id=raw_id,
|
|
184
|
+
actors=actors,
|
|
185
|
+
summary=text[:120],
|
|
186
|
+
details=text,
|
|
187
|
+
sensitivity="low",
|
|
188
|
+
deleted_at=None,
|
|
189
|
+
metadata=meta,
|
|
190
|
+
context_id=run_id or None,
|
|
191
|
+
categories=categories,
|
|
192
|
+
importance=0.5,
|
|
193
|
+
confidence=0.7,
|
|
194
|
+
lifecycle_state="HOT",
|
|
195
|
+
extraction_state="done",
|
|
196
|
+
last_accessed=None,
|
|
197
|
+
access_count=0,
|
|
198
|
+
pinned=False,
|
|
199
|
+
expiration_date=expiration,
|
|
200
|
+
vault_refs=[],
|
|
201
|
+
# Only chat-shaped roles map to a speaker; a bare "role" like "system" does not.
|
|
202
|
+
speaker=role if role in ("user", "assistant") else None,
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _first_str(*values: Any) -> str | None:
|
|
207
|
+
"""First value that is a ``str`` (mirrors Swift's ``as? String`` cast chain)."""
|
|
208
|
+
for v in values:
|
|
209
|
+
if isinstance(v, str):
|
|
210
|
+
return v
|
|
211
|
+
return None
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _opt_str(value: Any) -> str | None:
|
|
215
|
+
"""A ``str`` value, or ``None`` (Swift ``o[k] as? String``)."""
|
|
216
|
+
return value if isinstance(value, str) else None
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _stringify(v: Any) -> str:
|
|
220
|
+
"""Render a metadata value as a string, mirroring Swift's ``stringify``.
|
|
221
|
+
|
|
222
|
+
- ``str`` passes through.
|
|
223
|
+
- ``bool`` becomes ``"1"``/``"0"`` — ``JSONSerialization`` decodes JSON booleans to
|
|
224
|
+
``NSNumber``, whose ``stringValue`` is ``"1"``/``"0"``. Checked before ``int``
|
|
225
|
+
because ``bool`` is an ``int`` subclass in Python.
|
|
226
|
+
- other numbers use their string form (``NSNumber.stringValue``).
|
|
227
|
+
- containers use canonical JSON (Swift uses ``JSONSerialization`` with sorted keys;
|
|
228
|
+
the foundation's ``canonical_json`` is the byte-compatible equivalent).
|
|
229
|
+
"""
|
|
230
|
+
if isinstance(v, str):
|
|
231
|
+
return v
|
|
232
|
+
if isinstance(v, bool):
|
|
233
|
+
return "1" if v else "0"
|
|
234
|
+
if isinstance(v, int):
|
|
235
|
+
return str(v)
|
|
236
|
+
if isinstance(v, float):
|
|
237
|
+
# ``canonical_json`` yields the shortest round-tripping form, matching Swift's
|
|
238
|
+
# ``NSNumber.stringValue`` for the numeric cases mem0 produces.
|
|
239
|
+
return canonical_json(v)
|
|
240
|
+
if isinstance(v, (dict, list)):
|
|
241
|
+
return canonical_json(v)
|
|
242
|
+
return str(v)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
#: Module-level alias so callers can ``from portable_memory.adapters.mem0 import
|
|
246
|
+
#: parse_episodes`` without going through the class.
|
|
247
|
+
parse_episodes = Mem0Adapter.parse_episodes
|