memgres 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- memgres/__init__.py +34 -0
- memgres/blame.py +143 -0
- memgres/config.py +111 -0
- memgres/diffing.py +130 -0
- memgres/embeddings.py +144 -0
- memgres/mcp_server.py +114 -0
- memgres/migrations/0001_core.sql +72 -0
- memgres/qdrant_backend.py +82 -0
- memgres/schema.py +133 -0
- memgres/search.py +125 -0
- memgres/server.py +222 -0
- memgres/store.py +402 -0
- memgres-0.1.0.dist-info/METADATA +231 -0
- memgres-0.1.0.dist-info/RECORD +18 -0
- memgres-0.1.0.dist-info/WHEEL +5 -0
- memgres-0.1.0.dist-info/entry_points.txt +3 -0
- memgres-0.1.0.dist-info/licenses/LICENSE +21 -0
- memgres-0.1.0.dist-info/top_level.txt +1 -0
memgres/__init__.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""memgres — versioned document memory for AI agents, backed by one Postgres.
|
|
2
|
+
|
|
3
|
+
Quick start::
|
|
4
|
+
|
|
5
|
+
from memgres import Store, load_config, migrate
|
|
6
|
+
import psycopg
|
|
7
|
+
|
|
8
|
+
cfg = load_config() # reads MEMGRES_* env
|
|
9
|
+
conn = psycopg.connect(cfg.database_url)
|
|
10
|
+
migrate(conn, cfg) # idempotent; stamps embed model/dim
|
|
11
|
+
|
|
12
|
+
store = Store(cfg, conn=conn)
|
|
13
|
+
m = store.write(body="remember this\n", tags=["note"], source="me")
|
|
14
|
+
store.write(id=m.id, diff=patch, base_hash=m.content_hash) # authored edit
|
|
15
|
+
hits = store.recall(None, "what did I remember?") # lexical or semantic
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from .config import Config, load as load_config
|
|
19
|
+
from .diffing import apply_diff, content_hash, make_diff, DiffConflict
|
|
20
|
+
from .embeddings import Embedder, get_embedder
|
|
21
|
+
from .schema import migrate, SchemaMismatch, SCHEMA_VERSION
|
|
22
|
+
from .search import Hit, recall
|
|
23
|
+
from .blame import annotate, annotate_grouped, reconstruct, replay
|
|
24
|
+
from .store import Store, Memory, Conflict, NotFound, TooLarge, NoParent
|
|
25
|
+
|
|
26
|
+
__all__ = [
|
|
27
|
+
"Config", "load_config",
|
|
28
|
+
"Store", "Memory", "Conflict", "NotFound", "TooLarge", "NoParent",
|
|
29
|
+
"make_diff", "apply_diff", "content_hash", "DiffConflict",
|
|
30
|
+
"Embedder", "get_embedder",
|
|
31
|
+
"migrate", "SchemaMismatch", "SCHEMA_VERSION",
|
|
32
|
+
"Hit", "recall",
|
|
33
|
+
"annotate", "annotate_grouped", "reconstruct", "replay",
|
|
34
|
+
]
|
memgres/blame.py
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""Turn the diff history into an annotated document (git-blame) and reconstruct
|
|
2
|
+
any past version — so callers never replay diffs themselves.
|
|
3
|
+
|
|
4
|
+
Because the store keeps a canonical diff for *every* body change (create is a
|
|
5
|
+
diff-from-empty), the history is a self-contained chain empty → current. We
|
|
6
|
+
replay it forward line by line, carrying an attribution with each surviving line:
|
|
7
|
+
|
|
8
|
+
* an added line is attributed to the change that introduced it,
|
|
9
|
+
* a context (unchanged) line keeps the attribution it already had,
|
|
10
|
+
* a removed line drops out.
|
|
11
|
+
|
|
12
|
+
That is exactly blame semantics — a modified line reads as remove+add, so the new
|
|
13
|
+
text is credited to the change that wrote it. Metadata-only rows (move/retag)
|
|
14
|
+
carry no diff and don't affect line attribution.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from typing import Iterable, List, Optional, Tuple
|
|
20
|
+
|
|
21
|
+
from .diffing import _HUNK
|
|
22
|
+
|
|
23
|
+
_ATTR_KEYS = ("seq", "op", "source", "reason", "created_at")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _apply_attributed(src: List[Tuple[str, dict]], patch: str,
|
|
27
|
+
attrib: dict) -> List[Tuple[str, dict]]:
|
|
28
|
+
"""Apply one diff to a list of (line_text, attribution) pairs. Added lines get
|
|
29
|
+
`attrib`; context/removed lines are matched against src text exactly (the base
|
|
30
|
+
is exact, so hunks line up — a mismatch is a real bug, surfaced loudly)."""
|
|
31
|
+
out: List[Tuple[str, dict]] = []
|
|
32
|
+
i = 0
|
|
33
|
+
lines = patch.split("\n")
|
|
34
|
+
n = len(lines)
|
|
35
|
+
p = 0
|
|
36
|
+
|
|
37
|
+
def marker_next(idx: int) -> bool:
|
|
38
|
+
return idx + 1 < n and lines[idx + 1].startswith("\\ No newline")
|
|
39
|
+
|
|
40
|
+
while p < n:
|
|
41
|
+
line = lines[p]
|
|
42
|
+
if line.startswith(("--- ", "+++ ")):
|
|
43
|
+
p += 1
|
|
44
|
+
continue
|
|
45
|
+
m = _HUNK.match(line)
|
|
46
|
+
if not m:
|
|
47
|
+
p += 1
|
|
48
|
+
continue
|
|
49
|
+
old_start = int(m.group(1))
|
|
50
|
+
target = old_start - 1 if old_start > 0 else 0
|
|
51
|
+
out.extend(src[i:target])
|
|
52
|
+
i = target
|
|
53
|
+
p += 1
|
|
54
|
+
while p < n and not lines[p].startswith("@@"):
|
|
55
|
+
hl = lines[p]
|
|
56
|
+
if hl.startswith("\\ No newline") or hl.startswith(("--- ", "+++ ")) or hl == "":
|
|
57
|
+
p += 1
|
|
58
|
+
continue
|
|
59
|
+
tag, text = hl[0], hl[1:]
|
|
60
|
+
if tag == " ":
|
|
61
|
+
out.append(src[i]); i += 1
|
|
62
|
+
elif tag == "-":
|
|
63
|
+
i += 1
|
|
64
|
+
elif tag == "+":
|
|
65
|
+
out.append((text if marker_next(p) else text + "\n", attrib))
|
|
66
|
+
p += 1
|
|
67
|
+
out.extend(src[i:])
|
|
68
|
+
return out
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def replay(history: List[dict], upto_seq: Optional[int] = None) -> List[Tuple[str, dict]]:
|
|
72
|
+
"""Forward-replay history (ordered by seq) into a list of (line, attribution).
|
|
73
|
+
`upto_seq` stops after that version (default: all → current body)."""
|
|
74
|
+
lines: List[Tuple[str, dict]] = []
|
|
75
|
+
for row in history:
|
|
76
|
+
if upto_seq is not None and row["seq"] > upto_seq:
|
|
77
|
+
break
|
|
78
|
+
diff = row.get("diff")
|
|
79
|
+
if not diff:
|
|
80
|
+
continue # metadata-only change: no line touched
|
|
81
|
+
attrib = {k: row.get(k) for k in _ATTR_KEYS}
|
|
82
|
+
lines = _apply_attributed(lines, diff, attrib)
|
|
83
|
+
return lines
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def reconstruct(history: List[dict], upto_seq: Optional[int] = None) -> str:
|
|
87
|
+
"""The exact body text at a version (default current)."""
|
|
88
|
+
return "".join(text for text, _ in replay(history, upto_seq))
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def annotate(history: List[dict], upto_seq: Optional[int] = None,
|
|
92
|
+
lines: Optional[Iterable[int]] = None) -> List[dict]:
|
|
93
|
+
"""Blame view: one entry per line of the (reconstructed) body, each tagged
|
|
94
|
+
with the change that last wrote it.
|
|
95
|
+
|
|
96
|
+
Attribution always needs the full replay, but `lines` (an iterable of 1-based
|
|
97
|
+
line numbers) restricts the *returned* entries — so you can blame one line or
|
|
98
|
+
a slice without shipping the whole document. Out-of-range numbers are ignored.
|
|
99
|
+
"""
|
|
100
|
+
want = set(lines) if lines is not None else None
|
|
101
|
+
result = []
|
|
102
|
+
for lineno, (text, attrib) in enumerate(replay(history, upto_seq), start=1):
|
|
103
|
+
if want is not None and lineno not in want:
|
|
104
|
+
continue
|
|
105
|
+
result.append({"line": lineno, "text": text, **attrib})
|
|
106
|
+
return result
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def annotate_grouped(history: List[dict], upto_seq: Optional[int] = None,
|
|
110
|
+
include_text: bool = True) -> List[dict]:
|
|
111
|
+
"""Blame as runs: consecutive lines with the same attribution collapse into one
|
|
112
|
+
block ``{start, end, lines, seq, op, source, reason, created_at[, text]}``.
|
|
113
|
+
|
|
114
|
+
A 5000-line document edited by two authors returns a handful of blocks, not
|
|
115
|
+
5000 rows. Set ``include_text=False`` for a pure ownership map (ranges only,
|
|
116
|
+
no body) — tiny even for huge documents.
|
|
117
|
+
"""
|
|
118
|
+
blocks: List[dict] = []
|
|
119
|
+
cur: Optional[dict] = None
|
|
120
|
+
cur_texts: List[str] = []
|
|
121
|
+
for lineno, (text, attrib) in enumerate(replay(history, upto_seq), start=1):
|
|
122
|
+
key = (attrib.get("seq"), attrib.get("source"), attrib.get("reason"),
|
|
123
|
+
attrib.get("op"))
|
|
124
|
+
if cur is not None and key == cur["_key"]:
|
|
125
|
+
cur["end"] = lineno
|
|
126
|
+
cur["lines"] += 1
|
|
127
|
+
cur_texts.append(text)
|
|
128
|
+
else:
|
|
129
|
+
if cur is not None:
|
|
130
|
+
_finish_block(cur, cur_texts, include_text, blocks)
|
|
131
|
+
cur = {"_key": key, "start": lineno, "end": lineno, "lines": 1, **attrib}
|
|
132
|
+
cur_texts = [text]
|
|
133
|
+
if cur is not None:
|
|
134
|
+
_finish_block(cur, cur_texts, include_text, blocks)
|
|
135
|
+
return blocks
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _finish_block(block: dict, texts: List[str], include_text: bool,
|
|
139
|
+
out: List[dict]) -> None:
|
|
140
|
+
block.pop("_key", None)
|
|
141
|
+
if include_text:
|
|
142
|
+
block["text"] = "".join(texts)
|
|
143
|
+
out.append(block)
|
memgres/config.py
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"""All memgres settings, read from the environment.
|
|
2
|
+
|
|
3
|
+
Every operational limit lives here so the same code serves two very different
|
|
4
|
+
deployments from env alone:
|
|
5
|
+
|
|
6
|
+
* single-user / embedded — no namespaces, unlimited retention, write whole
|
|
7
|
+
bodies directly;
|
|
8
|
+
* multi-tenant / metered — token namespaces required, retention
|
|
9
|
+
capped, single writes capped small so large memories accrue over many
|
|
10
|
+
(paid) diffs.
|
|
11
|
+
|
|
12
|
+
Nothing here is specific to a language or a project. The default embedding
|
|
13
|
+
provider is ``none`` (lexical search still works); turn on a local or cloud
|
|
14
|
+
model when you want semantic recall.
|
|
15
|
+
|
|
16
|
+
Env is read in :func:`load`, not at import time, so a process that sets env
|
|
17
|
+
after importing this module still gets the right values, and tests can vary it.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import os
|
|
23
|
+
from dataclasses import dataclass
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _int(name: str, default: int) -> int:
|
|
27
|
+
raw = os.environ.get(name)
|
|
28
|
+
return int(raw) if raw not in (None, "") else default
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _bool(name: str, default: bool) -> bool:
|
|
32
|
+
raw = os.environ.get(name)
|
|
33
|
+
if raw in (None, ""):
|
|
34
|
+
return default
|
|
35
|
+
return raw.strip().lower() in ("1", "true", "yes", "on")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _str(name: str, default: str) -> str:
|
|
39
|
+
raw = os.environ.get(name)
|
|
40
|
+
return raw if raw not in (None, "") else default
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class Config:
|
|
45
|
+
# storage limits (bytes)
|
|
46
|
+
max_body_bytes: int # whole-record ceiling; grows to this via diffs
|
|
47
|
+
max_write_bytes: int # one write/diff payload ceiling (<= max_body)
|
|
48
|
+
# retention
|
|
49
|
+
retention_days: int # 0 = forever; >0 = expire N days after last touch
|
|
50
|
+
renew_on_read: bool # a read pushes the expiry clock forward
|
|
51
|
+
# multi-tenant isolation
|
|
52
|
+
namespaces_enabled: bool # False = single space; True = secret-token namespaces
|
|
53
|
+
# organization
|
|
54
|
+
tree_enabled: bool # ltree path column + GiST index for fast subtree selection
|
|
55
|
+
require_parent: bool # False = sparse paths (create food.apple with no food row);
|
|
56
|
+
# True = a node's parent path must already exist as a memory
|
|
57
|
+
# history
|
|
58
|
+
history_enabled: bool # keep hash-chained diff history (deleted with record)
|
|
59
|
+
# search
|
|
60
|
+
fts_language: str # Postgres FTS dict: simple | english | russian | …
|
|
61
|
+
vector_backend: str # pgvector (default) | qdrant
|
|
62
|
+
# embeddings
|
|
63
|
+
embed_provider: str # none | local | jina | openai | openai-compatible
|
|
64
|
+
embed_model: str
|
|
65
|
+
embed_dim: int # 0 = infer from provider
|
|
66
|
+
embed_api_key: str
|
|
67
|
+
embed_api_base: str
|
|
68
|
+
# database
|
|
69
|
+
database_url: str
|
|
70
|
+
|
|
71
|
+
def validate(self) -> None:
|
|
72
|
+
if self.max_write_bytes > self.max_body_bytes:
|
|
73
|
+
raise ValueError(
|
|
74
|
+
"MEMGRES_MAX_WRITE_BYTES must be <= MEMGRES_MAX_BODY_BYTES"
|
|
75
|
+
)
|
|
76
|
+
if self.embed_provider not in (
|
|
77
|
+
"none", "local", "jina", "openai",
|
|
78
|
+
"openai-compatible", "compatible", "custom"):
|
|
79
|
+
raise ValueError(f"unknown MEMGRES_EMBED_PROVIDER: {self.embed_provider}")
|
|
80
|
+
if self.vector_backend not in ("pgvector", "qdrant"):
|
|
81
|
+
raise ValueError(f"unknown MEMGRES_VECTOR_BACKEND: {self.vector_backend}")
|
|
82
|
+
if self.embed_provider != "none" and self.vector_backend == "pgvector" \
|
|
83
|
+
and self.embed_dim <= 0:
|
|
84
|
+
raise ValueError(
|
|
85
|
+
"semantic search needs a vector dimension: set MEMGRES_EMBED_DIM "
|
|
86
|
+
"(or it is inferred once the provider loads)"
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def load() -> Config:
|
|
91
|
+
"""Build Config from the current environment and validate it."""
|
|
92
|
+
cfg = Config(
|
|
93
|
+
max_body_bytes=_int("MEMGRES_MAX_BODY_BYTES", 262_144), # 256 KB
|
|
94
|
+
max_write_bytes=_int("MEMGRES_MAX_WRITE_BYTES", 16_384), # 16 KB
|
|
95
|
+
retention_days=_int("MEMGRES_RETENTION_DAYS", 0),
|
|
96
|
+
renew_on_read=_bool("MEMGRES_RENEW_ON_READ", True),
|
|
97
|
+
namespaces_enabled=_bool("MEMGRES_NAMESPACES", False),
|
|
98
|
+
tree_enabled=_bool("MEMGRES_TREE", True),
|
|
99
|
+
require_parent=_bool("MEMGRES_REQUIRE_PARENT", False),
|
|
100
|
+
history_enabled=_bool("MEMGRES_HISTORY", True),
|
|
101
|
+
fts_language=_str("MEMGRES_FTS_LANGUAGE", "simple"),
|
|
102
|
+
vector_backend=_str("MEMGRES_VECTOR_BACKEND", "pgvector"),
|
|
103
|
+
embed_provider=_str("MEMGRES_EMBED_PROVIDER", "none"),
|
|
104
|
+
embed_model=_str("MEMGRES_EMBED_MODEL", ""),
|
|
105
|
+
embed_dim=_int("MEMGRES_EMBED_DIM", 0),
|
|
106
|
+
embed_api_key=_str("MEMGRES_EMBED_API_KEY", ""),
|
|
107
|
+
embed_api_base=_str("MEMGRES_EMBED_API_BASE", ""),
|
|
108
|
+
database_url=_str("MEMGRES_DATABASE_URL", ""),
|
|
109
|
+
)
|
|
110
|
+
cfg.validate()
|
|
111
|
+
return cfg
|
memgres/diffing.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Content hashing and unified-diff make/apply.
|
|
2
|
+
|
|
3
|
+
A write can arrive two ways: the whole new body, or a *diff* (a unified diff, the
|
|
4
|
+
same text ``git diff`` prints). Either way we store a unified diff in history, so
|
|
5
|
+
every change is replayable and attributable.
|
|
6
|
+
|
|
7
|
+
Concurrency is guarded by content hash, not locks: the caller passes the hash of
|
|
8
|
+
the body it started from; the store applies the change only if the current body
|
|
9
|
+
still hashes to that value (optimistic locking). Because the base is thus exact,
|
|
10
|
+
diff application never needs fuzz — a hunk that does not line up is a real
|
|
11
|
+
conflict and raises, rather than corrupting the body silently.
|
|
12
|
+
|
|
13
|
+
Trailing newlines are preserved exactly via the ``\``
|
|
14
|
+
marker, the way git does it; without it a body that does/doesn't end in a
|
|
15
|
+
newline would not survive a make→apply round-trip.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import difflib
|
|
21
|
+
import hashlib
|
|
22
|
+
import re
|
|
23
|
+
|
|
24
|
+
_HUNK = re.compile(r"^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@")
|
|
25
|
+
_NO_NL = "\"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def content_hash(body: str) -> str:
|
|
29
|
+
"""Stable identifier of a body's exact content. Used for optimistic
|
|
30
|
+
locking and as the link in the history hash-chain."""
|
|
31
|
+
return hashlib.sha256(body.encode("utf-8")).hexdigest()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def make_diff(old: str, new: str) -> str:
|
|
35
|
+
"""Unified diff turning ``old`` into ``new``. Empty string if identical."""
|
|
36
|
+
if old == new:
|
|
37
|
+
return ""
|
|
38
|
+
a = old.splitlines(keepends=True)
|
|
39
|
+
b = new.splitlines(keepends=True)
|
|
40
|
+
out: list[str] = []
|
|
41
|
+
for line in difflib.unified_diff(a, b, lineterm=""):
|
|
42
|
+
if line.startswith(("--- ", "+++ ", "@@")):
|
|
43
|
+
out.append(line + "\n")
|
|
44
|
+
elif line.endswith("\n"):
|
|
45
|
+
out.append(line)
|
|
46
|
+
else: # a content line with no trailing newline = last line of file
|
|
47
|
+
out.append(line + "\n")
|
|
48
|
+
out.append(_NO_NL + "\n")
|
|
49
|
+
return "".join(out)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class DiffConflict(Exception):
|
|
53
|
+
"""A patch hunk did not match the body it was applied to."""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def apply_diff(old: str, patch: str) -> str:
|
|
57
|
+
"""Apply a unified diff to ``old`` and return the new body.
|
|
58
|
+
|
|
59
|
+
Raises DiffConflict if a hunk's context/removed lines do not match — with
|
|
60
|
+
an exact base (enforced by the hash check upstream) this only happens on a
|
|
61
|
+
malformed or stale patch, never as normal fuzz.
|
|
62
|
+
"""
|
|
63
|
+
if patch.strip() == "":
|
|
64
|
+
return old
|
|
65
|
+
|
|
66
|
+
src = old.splitlines(keepends=True)
|
|
67
|
+
out: list[str] = []
|
|
68
|
+
i = 0 # index into src (0-based)
|
|
69
|
+
lines = patch.split("\n")
|
|
70
|
+
n = len(lines)
|
|
71
|
+
p = 0
|
|
72
|
+
|
|
73
|
+
def marker_next(idx: int) -> bool:
|
|
74
|
+
return idx + 1 < n and lines[idx + 1].startswith("\\ No newline")
|
|
75
|
+
|
|
76
|
+
while p < n:
|
|
77
|
+
line = lines[p]
|
|
78
|
+
if line.startswith("--- ") or line.startswith("+++ "):
|
|
79
|
+
p += 1
|
|
80
|
+
continue
|
|
81
|
+
m = _HUNK.match(line)
|
|
82
|
+
if not m:
|
|
83
|
+
p += 1
|
|
84
|
+
continue
|
|
85
|
+
|
|
86
|
+
old_start = int(m.group(1))
|
|
87
|
+
# @@ lines are 1-based; a hunk adding to an empty side uses -0 → 0.
|
|
88
|
+
target = old_start - 1 if old_start > 0 else 0
|
|
89
|
+
if target < i:
|
|
90
|
+
raise DiffConflict(f"hunk out of order at source line {old_start}")
|
|
91
|
+
out.extend(src[i:target])
|
|
92
|
+
i = target
|
|
93
|
+
p += 1
|
|
94
|
+
|
|
95
|
+
while p < n and not lines[p].startswith("@@"):
|
|
96
|
+
hl = lines[p]
|
|
97
|
+
if hl.startswith("\\ No newline") or hl.startswith(("--- ", "+++ ")):
|
|
98
|
+
p += 1
|
|
99
|
+
continue
|
|
100
|
+
if hl == "": # trailing split artifact
|
|
101
|
+
p += 1
|
|
102
|
+
continue
|
|
103
|
+
tag, text = hl[0], hl[1:]
|
|
104
|
+
if tag == " ": # context: copy exact source bytes
|
|
105
|
+
_expect(src, i, text)
|
|
106
|
+
out.append(src[i]); i += 1
|
|
107
|
+
elif tag == "-": # removal: verify then drop
|
|
108
|
+
_expect(src, i, text)
|
|
109
|
+
i += 1
|
|
110
|
+
elif tag == "+": # addition: marker decides newline
|
|
111
|
+
out.append(text if marker_next(p) else text + "\n")
|
|
112
|
+
else:
|
|
113
|
+
raise DiffConflict(f"unexpected patch line: {hl!r}")
|
|
114
|
+
p += 1
|
|
115
|
+
|
|
116
|
+
out.extend(src[i:])
|
|
117
|
+
return "".join(out)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _expect(src: list[str], i: int, text: str) -> None:
|
|
121
|
+
if i >= len(src):
|
|
122
|
+
raise DiffConflict(f"patch expects a line past end of body: {text!r}")
|
|
123
|
+
if src[i].rstrip("\n") != text.rstrip("\n"):
|
|
124
|
+
raise DiffConflict(f"context mismatch at line {i + 1}: "
|
|
125
|
+
f"have {src[i]!r}, patch expects {text!r}")
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def byte_len(text: str) -> int:
|
|
129
|
+
"""UTF-8 byte length — what limits are measured in."""
|
|
130
|
+
return len(text.encode("utf-8"))
|
memgres/embeddings.py
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"""Pluggable embedding providers, selected by env.
|
|
2
|
+
|
|
3
|
+
MEMGRES_EMBED_PROVIDER = none | local | jina | openai
|
|
4
|
+
|
|
5
|
+
``none`` disables semantic recall entirely (lexical FTS still works, no model,
|
|
6
|
+
no API, no GPU). ``local`` runs a sentence-transformers model in-process.
|
|
7
|
+
``jina`` / ``openai`` call a hosted API. All are behind one tiny interface so
|
|
8
|
+
the store never cares which is configured::
|
|
9
|
+
|
|
10
|
+
emb = get_embedder(cfg) # -> Embedder | None
|
|
11
|
+
if emb:
|
|
12
|
+
vecs = emb.embed_documents(["…"]) # for writes
|
|
13
|
+
q = emb.embed_query("…") # for recall
|
|
14
|
+
dim = emb.dim
|
|
15
|
+
|
|
16
|
+
The document/query split matters: several models (and Jina) want an explicit
|
|
17
|
+
"is this a passage or a search query?" hint, and mixing them up quietly hurts
|
|
18
|
+
recall. Cloud calls use stdlib urllib so the base install stays dependency-light
|
|
19
|
+
(only ``local`` pulls in sentence-transformers, via the ``[local]`` extra).
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import json
|
|
25
|
+
import urllib.request
|
|
26
|
+
from typing import List, Optional, Sequence
|
|
27
|
+
|
|
28
|
+
from .config import Config
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class Embedder:
|
|
32
|
+
"""Interface: implementations return unit-normalized float lists."""
|
|
33
|
+
|
|
34
|
+
dim: int
|
|
35
|
+
|
|
36
|
+
def embed_documents(self, texts: Sequence[str]) -> List[List[float]]:
|
|
37
|
+
raise NotImplementedError
|
|
38
|
+
|
|
39
|
+
def embed_query(self, text: str) -> List[float]:
|
|
40
|
+
raise NotImplementedError
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
# ─── local: sentence-transformers ────────────────────────────────────────────
|
|
44
|
+
class _LocalEmbedder(Embedder):
|
|
45
|
+
def __init__(self, model_name: str, want_dim: int):
|
|
46
|
+
from sentence_transformers import SentenceTransformer # lazy: heavy import
|
|
47
|
+
|
|
48
|
+
if not model_name:
|
|
49
|
+
raise ValueError("MEMGRES_EMBED_MODEL is required for the local provider")
|
|
50
|
+
self._model = SentenceTransformer(model_name, device="cpu")
|
|
51
|
+
self.dim = self._model.get_sentence_embedding_dimension()
|
|
52
|
+
if want_dim and want_dim != self.dim:
|
|
53
|
+
raise ValueError(
|
|
54
|
+
f"MEMGRES_EMBED_DIM={want_dim} but '{model_name}' emits {self.dim}"
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
def embed_documents(self, texts: Sequence[str]) -> List[List[float]]:
|
|
58
|
+
vecs = self._model.encode(
|
|
59
|
+
list(texts), normalize_embeddings=True, convert_to_numpy=True
|
|
60
|
+
)
|
|
61
|
+
return [v.tolist() for v in vecs]
|
|
62
|
+
|
|
63
|
+
def embed_query(self, text: str) -> List[float]:
|
|
64
|
+
return self.embed_documents([text])[0]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
# ─── cloud: Jina and OpenAI share an OpenAI-shaped /embeddings endpoint ───────
|
|
68
|
+
class _HttpEmbedder(Embedder):
|
|
69
|
+
def __init__(self, model: str, dim: int, api_key: str, api_base: str,
|
|
70
|
+
query_task: Optional[str] = None, passage_task: Optional[str] = None,
|
|
71
|
+
require_key: bool = True):
|
|
72
|
+
if not model:
|
|
73
|
+
raise ValueError("MEMGRES_EMBED_MODEL is required for HTTP providers")
|
|
74
|
+
if not api_base:
|
|
75
|
+
raise ValueError("MEMGRES_EMBED_API_BASE (server URL) is required")
|
|
76
|
+
if require_key and not api_key:
|
|
77
|
+
raise ValueError("MEMGRES_EMBED_API_KEY is required for this provider")
|
|
78
|
+
if dim <= 0:
|
|
79
|
+
raise ValueError("MEMGRES_EMBED_DIM must be set for HTTP providers")
|
|
80
|
+
self.dim = dim
|
|
81
|
+
self._model = model
|
|
82
|
+
self._key = api_key
|
|
83
|
+
self._base = api_base.rstrip("/")
|
|
84
|
+
self._query_task = query_task
|
|
85
|
+
self._passage_task = passage_task
|
|
86
|
+
|
|
87
|
+
def _post(self, inputs: Sequence[str], task: Optional[str]) -> List[List[float]]:
|
|
88
|
+
payload = {"model": self._model, "input": list(inputs)}
|
|
89
|
+
if task: # Jina uses task to distinguish passage vs query; OpenAI ignores it
|
|
90
|
+
payload["task"] = task
|
|
91
|
+
headers = {"Content-Type": "application/json"}
|
|
92
|
+
if self._key: # local servers (LM Studio/Ollama) often need no key
|
|
93
|
+
headers["Authorization"] = f"Bearer {self._key}"
|
|
94
|
+
req = urllib.request.Request(
|
|
95
|
+
f"{self._base}/embeddings",
|
|
96
|
+
data=json.dumps(payload).encode(),
|
|
97
|
+
headers=headers,
|
|
98
|
+
method="POST",
|
|
99
|
+
)
|
|
100
|
+
with urllib.request.urlopen(req, timeout=60) as resp:
|
|
101
|
+
body = json.loads(resp.read())
|
|
102
|
+
rows = sorted(body["data"], key=lambda d: d.get("index", 0))
|
|
103
|
+
out = [r["embedding"] for r in rows]
|
|
104
|
+
for v in out:
|
|
105
|
+
if len(v) != self.dim:
|
|
106
|
+
raise ValueError(
|
|
107
|
+
f"provider returned dim {len(v)}, config says {self.dim} — "
|
|
108
|
+
f"fix MEMGRES_EMBED_DIM to match the model"
|
|
109
|
+
)
|
|
110
|
+
return out
|
|
111
|
+
|
|
112
|
+
def embed_documents(self, texts: Sequence[str]) -> List[List[float]]:
|
|
113
|
+
return self._post(texts, self._passage_task)
|
|
114
|
+
|
|
115
|
+
def embed_query(self, text: str) -> List[float]:
|
|
116
|
+
return self._post([text], self._query_task)[0]
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def get_embedder(cfg: Config) -> Optional[Embedder]:
|
|
120
|
+
"""Build the embedder the config asks for, or None for lexical-only."""
|
|
121
|
+
p = cfg.embed_provider
|
|
122
|
+
if p == "none":
|
|
123
|
+
return None
|
|
124
|
+
if p == "local":
|
|
125
|
+
return _LocalEmbedder(cfg.embed_model, cfg.embed_dim)
|
|
126
|
+
if p == "jina":
|
|
127
|
+
return _HttpEmbedder(
|
|
128
|
+
cfg.embed_model, cfg.embed_dim, cfg.embed_api_key,
|
|
129
|
+
cfg.embed_api_base or "https://api.jina.ai/v1",
|
|
130
|
+
query_task="retrieval.query", passage_task="retrieval.passage",
|
|
131
|
+
)
|
|
132
|
+
if p == "openai":
|
|
133
|
+
return _HttpEmbedder(
|
|
134
|
+
cfg.embed_model, cfg.embed_dim, cfg.embed_api_key,
|
|
135
|
+
cfg.embed_api_base or "https://api.openai.com/v1",
|
|
136
|
+
)
|
|
137
|
+
if p in ("openai-compatible", "compatible", "custom"):
|
|
138
|
+
# any OpenAI-shaped /embeddings server: LM Studio, Ollama, vLLM, TEI,
|
|
139
|
+
# LocalAI, … base URL required, key optional.
|
|
140
|
+
return _HttpEmbedder(
|
|
141
|
+
cfg.embed_model, cfg.embed_dim, cfg.embed_api_key, cfg.embed_api_base,
|
|
142
|
+
require_key=False,
|
|
143
|
+
)
|
|
144
|
+
raise ValueError(f"unknown embed provider: {p}") # config.validate guards this
|
memgres/mcp_server.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""MCP server exposing memgres as agent tools (stdio).
|
|
2
|
+
|
|
3
|
+
Thin wrapper over the same `Store` the HTTP layer uses — an MCP client (Claude
|
|
4
|
+
Desktop, etc.) gets write / recall / get / blame / history / move / forget as
|
|
5
|
+
tools. Run it:
|
|
6
|
+
|
|
7
|
+
memgres-mcp # after: pip install "memgres[mcp]"
|
|
8
|
+
|
|
9
|
+
Requires MEMGRES_DATABASE_URL (or libpq PG* env) pointing at a Postgres the
|
|
10
|
+
schema can migrate into; migration runs once on startup.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from typing import List, Optional
|
|
16
|
+
|
|
17
|
+
import psycopg
|
|
18
|
+
|
|
19
|
+
from .config import Config, load
|
|
20
|
+
from .embeddings import get_embedder
|
|
21
|
+
from .schema import migrate
|
|
22
|
+
from .store import Store
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _mcp(name: str):
|
|
26
|
+
# The SDK renamed FastMCP -> MCPServer; support both.
|
|
27
|
+
try:
|
|
28
|
+
from mcp.server.mcpserver import MCPServer
|
|
29
|
+
return MCPServer(name)
|
|
30
|
+
except ImportError:
|
|
31
|
+
from mcp.server.fastmcp import FastMCP
|
|
32
|
+
return FastMCP(name)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _mem(m) -> dict:
|
|
36
|
+
return {"id": m.id, "content_hash": m.content_hash, "body": m.body,
|
|
37
|
+
"tags": m.tags, "path": m.path, "seq": m.seq,
|
|
38
|
+
"created_at": str(m.created_at), "updated_at": str(m.updated_at),
|
|
39
|
+
"expires_at": str(m.expires_at) if m.expires_at else None}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def build_server(cfg: Optional[Config] = None):
|
|
43
|
+
cfg = cfg or load()
|
|
44
|
+
conn = psycopg.connect(cfg.database_url or "")
|
|
45
|
+
migrate(conn, cfg)
|
|
46
|
+
store = Store(cfg, embedder=get_embedder(cfg), conn=conn)
|
|
47
|
+
mcp = _mcp("memgres")
|
|
48
|
+
|
|
49
|
+
@mcp.tool()
|
|
50
|
+
def memory_write(body: Optional[str] = None, id: Optional[str] = None,
|
|
51
|
+
diff: Optional[str] = None, base_hash: Optional[str] = None,
|
|
52
|
+
path: Optional[str] = None, tags: Optional[List[str]] = None,
|
|
53
|
+
source: Optional[str] = None, reason: Optional[str] = None,
|
|
54
|
+
ttl_days: Optional[int] = None,
|
|
55
|
+
token: Optional[str] = None) -> dict:
|
|
56
|
+
"""Create or edit a memory. Omit `id` to create (needs `body`). To edit,
|
|
57
|
+
pass `id` plus either a whole new `body` or a unified `diff` with the
|
|
58
|
+
`base_hash` it was cut from (stale base -> conflict). `path`/`tags` set
|
|
59
|
+
the tree position and labels; `source`/`reason` record provenance."""
|
|
60
|
+
return _mem(store.write(token, id=id or None, body=body, diff=diff,
|
|
61
|
+
base_hash=base_hash, path=path, tags=tags,
|
|
62
|
+
source=source, reason=reason, ttl_days=ttl_days))
|
|
63
|
+
|
|
64
|
+
@mcp.tool()
|
|
65
|
+
def memory_get(id: str, token: Optional[str] = None) -> dict:
|
|
66
|
+
"""Fetch one memory by id (renews its TTL)."""
|
|
67
|
+
return _mem(store.get(token, id))
|
|
68
|
+
|
|
69
|
+
@mcp.tool()
|
|
70
|
+
def memory_recall(query: str, k: int = 10, mode: str = "auto",
|
|
71
|
+
tags: Optional[List[str]] = None,
|
|
72
|
+
path_prefix: Optional[str] = None,
|
|
73
|
+
token: Optional[str] = None) -> List[dict]:
|
|
74
|
+
"""Search memories. `mode`: lexical | semantic | hybrid | auto. Optionally
|
|
75
|
+
scope to a tag set (`tags`) or a subtree (`path_prefix`, e.g. 'ops.postgres')."""
|
|
76
|
+
return [{"id": h.id, "body": h.body, "tags": h.tags, "path": h.path,
|
|
77
|
+
"score": h.score}
|
|
78
|
+
for h in store.recall(token, query, k=k, tags=tags,
|
|
79
|
+
path_prefix=path_prefix, mode=mode)]
|
|
80
|
+
|
|
81
|
+
@mcp.tool()
|
|
82
|
+
def memory_blame(id: str, grouped: bool = True,
|
|
83
|
+
token: Optional[str] = None) -> List[dict]:
|
|
84
|
+
"""Who last changed each line. Grouped into author-blocks by default;
|
|
85
|
+
set grouped=false for per-line attribution."""
|
|
86
|
+
if grouped:
|
|
87
|
+
return store.annotate_grouped(token, id)
|
|
88
|
+
return store.annotate(token, id)
|
|
89
|
+
|
|
90
|
+
@mcp.tool()
|
|
91
|
+
def memory_history(id: str, token: Optional[str] = None) -> List[dict]:
|
|
92
|
+
"""The full change chain (diffs, provenance, hashes) for a memory."""
|
|
93
|
+
return store.history(token, id)
|
|
94
|
+
|
|
95
|
+
@mcp.tool()
|
|
96
|
+
def memory_move(id: str, new_path: str, reason: Optional[str] = None,
|
|
97
|
+
token: Optional[str] = None) -> dict:
|
|
98
|
+
"""Move a memory to a new tree path (cascades its subtree)."""
|
|
99
|
+
return _mem(store.move(token, id, new_path, reason=reason))
|
|
100
|
+
|
|
101
|
+
@mcp.tool()
|
|
102
|
+
def memory_forget(id: str, token: Optional[str] = None) -> dict:
|
|
103
|
+
"""Permanently delete a memory and its history (GDPR erasure)."""
|
|
104
|
+
return {"forgotten": store.forget(token, id)}
|
|
105
|
+
|
|
106
|
+
return mcp
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def main(): # pragma: no cover - entrypoint
|
|
110
|
+
build_server().run()
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
if __name__ == "__main__": # pragma: no cover
|
|
114
|
+
main()
|