memgres 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- memgres/__init__.py +34 -0
- memgres/blame.py +143 -0
- memgres/config.py +111 -0
- memgres/diffing.py +130 -0
- memgres/embeddings.py +144 -0
- memgres/mcp_server.py +114 -0
- memgres/migrations/0001_core.sql +72 -0
- memgres/qdrant_backend.py +82 -0
- memgres/schema.py +133 -0
- memgres/search.py +125 -0
- memgres/server.py +222 -0
- memgres/store.py +402 -0
- memgres-0.1.0.dist-info/METADATA +231 -0
- memgres-0.1.0.dist-info/RECORD +18 -0
- memgres-0.1.0.dist-info/WHEEL +5 -0
- memgres-0.1.0.dist-info/entry_points.txt +3 -0
- memgres-0.1.0.dist-info/licenses/LICENSE +21 -0
- memgres-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
-- memgres core schema (config-independent parts).
|
|
2
|
+
--
|
|
3
|
+
-- This file is always applied. Parts that depend on runtime config are added by
|
|
4
|
+
-- memgres/schema.py after this runs:
|
|
5
|
+
-- * ltree `path` column + GiST index — only when MEMGRES_TREE=true
|
|
6
|
+
-- * pgvector `embedding` column + HNSW — only when embeddings are enabled
|
|
7
|
+
-- * the meta row is stamped/verified there — hard-fail on model/dim drift
|
|
8
|
+
--
|
|
9
|
+
-- Nothing here needs a contrib extension: tags (text[]+GIN), FTS (tsvector+GIN)
|
|
10
|
+
-- and gen_random_uuid() are all core Postgres (>=13). ltree/vector are pulled in
|
|
11
|
+
-- conditionally by schema.py so a deployment without them still migrates.
|
|
12
|
+
|
|
13
|
+
-- ─── one memory = one mutable body ───────────────────────────────────────────
|
|
14
|
+
CREATE TABLE IF NOT EXISTS memory (
|
|
15
|
+
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
16
|
+
namespace text NOT NULL DEFAULT '', -- '' = single space; else hash(token)
|
|
17
|
+
body text NOT NULL,
|
|
18
|
+
content_hash text NOT NULL, -- sha256(body) hex; OCC + dedup
|
|
19
|
+
tags text[] NOT NULL DEFAULT '{}', -- cross-cutting labels
|
|
20
|
+
fts tsvector, -- store fills with the configured dict
|
|
21
|
+
seq integer NOT NULL DEFAULT 0, -- current version; == latest history.seq
|
|
22
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
23
|
+
updated_at timestamptz NOT NULL DEFAULT now(),
|
|
24
|
+
expires_at timestamptz -- NULL = keep forever
|
|
25
|
+
);
|
|
26
|
+
|
|
27
|
+
CREATE INDEX IF NOT EXISTS memory_ns_idx ON memory (namespace);
|
|
28
|
+
CREATE INDEX IF NOT EXISTS memory_tags_idx ON memory USING gin (tags);
|
|
29
|
+
CREATE INDEX IF NOT EXISTS memory_fts_idx ON memory USING gin (fts);
|
|
30
|
+
CREATE INDEX IF NOT EXISTS memory_expires_idx ON memory (expires_at) WHERE expires_at IS NOT NULL;
|
|
31
|
+
|
|
32
|
+
-- ─── hash-chained, deletable history (git-like provenance, GDPR-erasable) ─────
|
|
33
|
+
-- Deleted with the memory (ON DELETE CASCADE) → real erasure, not "hidden".
|
|
34
|
+
-- row_hash chains over the *history* rows so tampering is detectable; the chain
|
|
35
|
+
-- is for audit, it does not prevent deletion (that is the whole point vs git).
|
|
36
|
+
CREATE TABLE IF NOT EXISTS memory_history (
|
|
37
|
+
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
|
38
|
+
memory_id uuid NOT NULL REFERENCES memory(id) ON DELETE CASCADE,
|
|
39
|
+
seq integer NOT NULL, -- 1,2,3… per memory
|
|
40
|
+
op text NOT NULL, -- create|diff|replace|move|retag|delete
|
|
41
|
+
diff text, -- unified diff of body (NULL = metadata-only)
|
|
42
|
+
hash_before text, -- body content_hash before (NULL on create)
|
|
43
|
+
hash_after text, -- body content_hash after (NULL on delete)
|
|
44
|
+
path_before text, -- ltree as text; records a move
|
|
45
|
+
path_after text,
|
|
46
|
+
tags_before text[],
|
|
47
|
+
tags_after text[],
|
|
48
|
+
source text, -- provenance: where this change came from
|
|
49
|
+
reason text, -- provenance: why
|
|
50
|
+
prev_row_hash text, -- previous row_hash (NULL for seq=1)
|
|
51
|
+
row_hash text NOT NULL, -- H(prev_row_hash‖memory_id‖seq‖op‖…)
|
|
52
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
53
|
+
UNIQUE (memory_id, seq)
|
|
54
|
+
);
|
|
55
|
+
|
|
56
|
+
CREATE INDEX IF NOT EXISTS memory_history_mid_idx ON memory_history (memory_id, seq);
|
|
57
|
+
|
|
58
|
+
-- ─── single-row meta: stamps what the collection was built with ──────────────
|
|
59
|
+
-- schema.py verifies config against this and HARD-FAILS on drift (indexing with
|
|
60
|
+
-- one embedding model and querying with another silently returns garbage —
|
|
61
|
+
-- a documented silent-failure class this guard exists to prevent).
|
|
62
|
+
CREATE TABLE IF NOT EXISTS memgres_meta (
|
|
63
|
+
only_row boolean PRIMARY KEY DEFAULT true CHECK (only_row),
|
|
64
|
+
schema_version integer NOT NULL,
|
|
65
|
+
embed_provider text NOT NULL,
|
|
66
|
+
embed_model text NOT NULL DEFAULT '',
|
|
67
|
+
embed_dim integer NOT NULL DEFAULT 0,
|
|
68
|
+
fts_language text NOT NULL,
|
|
69
|
+
tree_enabled boolean NOT NULL,
|
|
70
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
71
|
+
updated_at timestamptz NOT NULL DEFAULT now()
|
|
72
|
+
);
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""Optional Qdrant vector backend (MEMGRES_VECTOR_BACKEND=qdrant).
|
|
2
|
+
|
|
3
|
+
pgvector keeps vectors in the same Postgres and is the default. Qdrant is for
|
|
4
|
+
when you already run it or want a dedicated ANN service. The split of labor:
|
|
5
|
+
|
|
6
|
+
* Qdrant holds only the vector + the `namespace` (so ranking never crosses
|
|
7
|
+
tenants) and ranks by cosine similarity.
|
|
8
|
+
* Postgres remains the source of truth for bodies and for every other filter
|
|
9
|
+
(tags, subtree, expiry) — semantic recall ranks in Qdrant, then fetches and
|
|
10
|
+
filters the candidates in Postgres.
|
|
11
|
+
|
|
12
|
+
So tag/tree/TTL changes never need to touch Qdrant — only a body change re-embeds
|
|
13
|
+
(upsert) and `forget` deletes the point. Config for the connection:
|
|
14
|
+
|
|
15
|
+
QDRANT_URL (default http://localhost:6333) · QDRANT_API_KEY ·
|
|
16
|
+
MEMGRES_QDRANT_COLLECTION (default "memgres")
|
|
17
|
+
|
|
18
|
+
Needs the `[qdrant]` extra (qdrant-client).
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import os
|
|
24
|
+
from typing import List, Optional, Sequence, Tuple
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class QdrantIndex:
|
|
28
|
+
def __init__(self, dim: int, url: Optional[str] = None,
|
|
29
|
+
api_key: Optional[str] = None, collection: Optional[str] = None):
|
|
30
|
+
from qdrant_client import QdrantClient
|
|
31
|
+
|
|
32
|
+
self.dim = dim
|
|
33
|
+
self.collection = collection or os.environ.get("MEMGRES_QDRANT_COLLECTION", "memgres")
|
|
34
|
+
url = url or os.environ.get("QDRANT_URL", "http://localhost:6333")
|
|
35
|
+
api_key = api_key or os.environ.get("QDRANT_API_KEY") or None
|
|
36
|
+
self.client = QdrantClient(url=url, api_key=api_key)
|
|
37
|
+
self._ensure()
|
|
38
|
+
|
|
39
|
+
def _ensure(self) -> None:
|
|
40
|
+
from qdrant_client.models import Distance, VectorParams
|
|
41
|
+
|
|
42
|
+
if not self.client.collection_exists(self.collection):
|
|
43
|
+
self.client.create_collection(
|
|
44
|
+
self.collection,
|
|
45
|
+
vectors_config=VectorParams(size=self.dim, distance=Distance.COSINE),
|
|
46
|
+
)
|
|
47
|
+
return
|
|
48
|
+
info = self.client.get_collection(self.collection)
|
|
49
|
+
size = info.config.params.vectors.size
|
|
50
|
+
if size != self.dim:
|
|
51
|
+
from .schema import SchemaMismatch
|
|
52
|
+
raise SchemaMismatch(
|
|
53
|
+
f"Qdrant collection '{self.collection}' has dim {size}, model emits "
|
|
54
|
+
f"{self.dim} — re-embed into a fresh collection, don't mix models."
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
def upsert(self, id: str, vector: Sequence[float], namespace: str) -> None:
|
|
58
|
+
from qdrant_client.models import PointStruct
|
|
59
|
+
|
|
60
|
+
self.client.upsert(
|
|
61
|
+
self.collection,
|
|
62
|
+
points=[PointStruct(id=id, vector=list(vector),
|
|
63
|
+
payload={"namespace": namespace})],
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
def delete(self, id: str) -> None:
|
|
67
|
+
from qdrant_client.models import PointIdsList
|
|
68
|
+
|
|
69
|
+
self.client.delete(self.collection, points_selector=PointIdsList(points=[id]))
|
|
70
|
+
|
|
71
|
+
def query(self, vector: Sequence[float], k: int,
|
|
72
|
+
namespace: str) -> List[Tuple[str, float]]:
|
|
73
|
+
"""Top-k point ids by cosine similarity, scoped to one namespace."""
|
|
74
|
+
from qdrant_client.models import FieldCondition, Filter, MatchValue
|
|
75
|
+
|
|
76
|
+
flt = Filter(must=[FieldCondition(key="namespace",
|
|
77
|
+
match=MatchValue(value=namespace))])
|
|
78
|
+
res = self.client.query_points(
|
|
79
|
+
self.collection, query=list(vector), limit=k,
|
|
80
|
+
query_filter=flt, with_payload=False,
|
|
81
|
+
).points
|
|
82
|
+
return [(str(p.id), float(p.score)) for p in res]
|
memgres/schema.py
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""Apply the schema and reconcile it with the running config.
|
|
2
|
+
|
|
3
|
+
`migrate(conn, cfg)` is idempotent: it applies the core SQL, then adds the
|
|
4
|
+
config-dependent pieces (ltree tree, pgvector column) only when they're turned
|
|
5
|
+
on, and finally *stamps* the collection with the embedding model/dim + FTS dict
|
|
6
|
+
it was built with.
|
|
7
|
+
|
|
8
|
+
The stamp is the safety rail. Indexing with one embedding model and querying
|
|
9
|
+
with another returns meaningless results with no error — a silent-failure class
|
|
10
|
+
we refuse to allow. On a mismatch `migrate` raises :class:`SchemaMismatch`
|
|
11
|
+
instead of quietly corrupting recall.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from .config import Config
|
|
19
|
+
|
|
20
|
+
SCHEMA_VERSION = 1
|
|
21
|
+
|
|
22
|
+
# Dev layout: repo/migrations next to the package. When packaged, migrations are
|
|
23
|
+
# shipped inside the package (see pyproject) and this still resolves.
|
|
24
|
+
_HERE = Path(__file__).resolve().parent
|
|
25
|
+
for _cand in (_HERE / "migrations", _HERE.parent / "migrations"):
|
|
26
|
+
if _cand.is_dir():
|
|
27
|
+
MIGRATIONS_DIR = _cand
|
|
28
|
+
break
|
|
29
|
+
else: # pragma: no cover - packaging guarantees one of the above
|
|
30
|
+
MIGRATIONS_DIR = _HERE.parent / "migrations"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class SchemaMismatch(RuntimeError):
|
|
34
|
+
"""The DB was built with settings incompatible with the current config."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _core_sql() -> str:
|
|
38
|
+
return (MIGRATIONS_DIR / "0001_core.sql").read_text(encoding="utf-8")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def migrate(conn, cfg: Config) -> None:
|
|
42
|
+
"""Bring the database at `conn` to the schema `cfg` describes.
|
|
43
|
+
|
|
44
|
+
`conn` is a psycopg connection. Runs in one transaction; raises
|
|
45
|
+
:class:`SchemaMismatch` (and rolls back) on an incompatible existing stamp.
|
|
46
|
+
"""
|
|
47
|
+
with conn.transaction():
|
|
48
|
+
with conn.cursor() as cur:
|
|
49
|
+
cur.execute(_core_sql())
|
|
50
|
+
_apply_tree(cur, cfg)
|
|
51
|
+
_apply_vector(cur, cfg)
|
|
52
|
+
_stamp(cur, cfg)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _apply_tree(cur, cfg: Config) -> None:
|
|
56
|
+
if not cfg.tree_enabled:
|
|
57
|
+
return
|
|
58
|
+
cur.execute("CREATE EXTENSION IF NOT EXISTS ltree")
|
|
59
|
+
# Add the path column + its indexes only once; ALTER … ADD COLUMN IF NOT
|
|
60
|
+
# EXISTS keeps this idempotent across restarts.
|
|
61
|
+
cur.execute("ALTER TABLE memory ADD COLUMN IF NOT EXISTS path ltree")
|
|
62
|
+
cur.execute(
|
|
63
|
+
"CREATE INDEX IF NOT EXISTS memory_path_gist ON memory "
|
|
64
|
+
"USING gist (path) WHERE path IS NOT NULL"
|
|
65
|
+
)
|
|
66
|
+
# A path is a node's address: unique within a namespace when present.
|
|
67
|
+
cur.execute(
|
|
68
|
+
"CREATE UNIQUE INDEX IF NOT EXISTS memory_ns_path_uniq ON memory "
|
|
69
|
+
"(namespace, path) WHERE path IS NOT NULL"
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _apply_vector(cur, cfg: Config) -> None:
|
|
74
|
+
if cfg.embed_provider == "none":
|
|
75
|
+
return
|
|
76
|
+
if cfg.vector_backend != "pgvector":
|
|
77
|
+
return # qdrant holds vectors out-of-band; nothing to add here
|
|
78
|
+
if cfg.embed_dim <= 0:
|
|
79
|
+
# config.validate() already guards this; belt and suspenders.
|
|
80
|
+
raise SchemaMismatch("pgvector semantic search needs MEMGRES_EMBED_DIM > 0")
|
|
81
|
+
cur.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
|
82
|
+
cur.execute(
|
|
83
|
+
f"ALTER TABLE memory ADD COLUMN IF NOT EXISTS embedding vector({cfg.embed_dim})"
|
|
84
|
+
)
|
|
85
|
+
cur.execute(
|
|
86
|
+
"CREATE INDEX IF NOT EXISTS memory_embedding_hnsw ON memory "
|
|
87
|
+
"USING hnsw (embedding vector_cosine_ops)"
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _stamp(cur, cfg: Config) -> None:
|
|
92
|
+
"""Insert the meta row, or verify the existing one and hard-fail on drift."""
|
|
93
|
+
cur.execute(
|
|
94
|
+
"SELECT embed_provider, embed_model, embed_dim, fts_language, tree_enabled "
|
|
95
|
+
"FROM memgres_meta WHERE only_row"
|
|
96
|
+
)
|
|
97
|
+
row = cur.fetchone()
|
|
98
|
+
if row is None:
|
|
99
|
+
cur.execute(
|
|
100
|
+
"INSERT INTO memgres_meta "
|
|
101
|
+
"(only_row, schema_version, embed_provider, embed_model, embed_dim, "
|
|
102
|
+
" fts_language, tree_enabled) VALUES (true, %s, %s, %s, %s, %s, %s)",
|
|
103
|
+
(SCHEMA_VERSION, cfg.embed_provider, cfg.embed_model, cfg.embed_dim,
|
|
104
|
+
cfg.fts_language, cfg.tree_enabled),
|
|
105
|
+
)
|
|
106
|
+
return
|
|
107
|
+
|
|
108
|
+
provider, model, dim, fts_lang, tree = row
|
|
109
|
+
|
|
110
|
+
if fts_lang != cfg.fts_language:
|
|
111
|
+
raise SchemaMismatch(
|
|
112
|
+
f"FTS dictionary changed: collection built with '{fts_lang}', config "
|
|
113
|
+
f"says '{cfg.fts_language}'. The stored tsvectors were computed with the "
|
|
114
|
+
f"old dictionary; recompute them (reindex) before switching."
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
# Embeddings: going none → enabled is fine (adopt the new stamp). But changing
|
|
118
|
+
# an *existing* model or dimension silently invalidates every stored vector.
|
|
119
|
+
had_vectors = dim > 0
|
|
120
|
+
if had_vectors and (model != cfg.embed_model or dim != cfg.embed_dim):
|
|
121
|
+
raise SchemaMismatch(
|
|
122
|
+
f"embedding model/dim changed: collection built with "
|
|
123
|
+
f"'{model}' (dim {dim}), config says '{cfg.embed_model}' (dim "
|
|
124
|
+
f"{cfg.embed_dim}). Existing vectors are meaningless under the new "
|
|
125
|
+
f"model — re-embed the corpus, don't mix models."
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
if not had_vectors and cfg.embed_provider != "none":
|
|
129
|
+
cur.execute(
|
|
130
|
+
"UPDATE memgres_meta SET embed_provider=%s, embed_model=%s, "
|
|
131
|
+
"embed_dim=%s, updated_at=now() WHERE only_row",
|
|
132
|
+
(cfg.embed_provider, cfg.embed_model, cfg.embed_dim),
|
|
133
|
+
)
|
memgres/search.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Recall: lexical (Postgres FTS), semantic (pgvector), or hybrid (RRF).
|
|
2
|
+
|
|
3
|
+
All three share the same filters — namespace, tags (``@>`` contains-all), tree
|
|
4
|
+
subtree (``path <@ prefix``), and not-expired — so you can scope any query to a
|
|
5
|
+
branch of the tree or a set of tags. ``mode='auto'`` picks semantic when an
|
|
6
|
+
embedder is configured, else lexical.
|
|
7
|
+
|
|
8
|
+
Hybrid fuses the two ranked lists with Reciprocal Rank Fusion (RRF): exact
|
|
9
|
+
identifiers that dense vectors fumble come in via lexical, meaning-based matches
|
|
10
|
+
via vectors, and neither backend needs to know about the other.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from typing import List, Optional, Sequence
|
|
17
|
+
|
|
18
|
+
RRF_K = 60 # standard RRF damping constant
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class Hit:
|
|
23
|
+
id: str
|
|
24
|
+
body: str
|
|
25
|
+
tags: List[str]
|
|
26
|
+
path: Optional[str]
|
|
27
|
+
score: float
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _filters(ns: str, tags: Optional[Sequence[str]], path_prefix: Optional[str]):
|
|
31
|
+
"""Return (sql_fragment, params) for the shared WHERE tail."""
|
|
32
|
+
sql = ["namespace = %s", "(expires_at IS NULL OR expires_at > now())"]
|
|
33
|
+
params: list = [ns]
|
|
34
|
+
if tags:
|
|
35
|
+
sql.append("tags @> %s") # row must contain all requested tags
|
|
36
|
+
params.append(list(tags))
|
|
37
|
+
if path_prefix:
|
|
38
|
+
sql.append("path <@ %s::ltree") # subtree of the prefix
|
|
39
|
+
params.append(path_prefix)
|
|
40
|
+
return " AND ".join(sql), params
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _lexical(conn, cfg, ns, query, k, tags, path_prefix) -> List[Hit]:
|
|
44
|
+
where, params = _filters(ns, tags, path_prefix)
|
|
45
|
+
sql = (
|
|
46
|
+
"SELECT id, body, tags, path::text, "
|
|
47
|
+
"ts_rank(fts, plainto_tsquery(%s::regconfig, %s)) AS score "
|
|
48
|
+
f"FROM memory WHERE {where} "
|
|
49
|
+
"AND fts @@ plainto_tsquery(%s::regconfig, %s) "
|
|
50
|
+
"ORDER BY score DESC LIMIT %s"
|
|
51
|
+
)
|
|
52
|
+
args = [cfg.fts_language, query] + params + [cfg.fts_language, query, k]
|
|
53
|
+
with conn.cursor() as cur:
|
|
54
|
+
cur.execute(sql, args)
|
|
55
|
+
return [Hit(str(r[0]), r[1], list(r[2]), r[3], float(r[4]))
|
|
56
|
+
for r in cur.fetchall()]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _semantic(conn, cfg, embedder, ns, query, k, tags, path_prefix, qdrant=None) -> List[Hit]:
|
|
60
|
+
if not embedder:
|
|
61
|
+
raise RuntimeError("semantic recall needs an embedder (MEMGRES_EMBED_PROVIDER)")
|
|
62
|
+
if qdrant is not None:
|
|
63
|
+
return _semantic_qdrant(conn, cfg, embedder, ns, query, k, tags, path_prefix, qdrant)
|
|
64
|
+
qv = "[" + ",".join(repr(float(x)) for x in embedder.embed_query(query)) + "]"
|
|
65
|
+
where, params = _filters(ns, tags, path_prefix)
|
|
66
|
+
sql = (
|
|
67
|
+
"SELECT id, body, tags, path::text, "
|
|
68
|
+
"1 - (embedding <=> %s::vector) AS score "
|
|
69
|
+
f"FROM memory WHERE {where} AND embedding IS NOT NULL "
|
|
70
|
+
"ORDER BY embedding <=> %s::vector ASC LIMIT %s"
|
|
71
|
+
)
|
|
72
|
+
args = [qv] + params + [qv, k]
|
|
73
|
+
with conn.cursor() as cur:
|
|
74
|
+
cur.execute(sql, args)
|
|
75
|
+
return [Hit(str(r[0]), r[1], list(r[2]), r[3], float(r[4]))
|
|
76
|
+
for r in cur.fetchall()]
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _semantic_qdrant(conn, cfg, embedder, ns, query, k, tags, path_prefix, qdrant) -> List[Hit]:
|
|
80
|
+
"""Rank in Qdrant (namespace-scoped), then fetch + filter bodies in Postgres.
|
|
81
|
+
Over-fetch when tag/subtree filters apply, since those are enforced in PG."""
|
|
82
|
+
qv = embedder.embed_query(query)
|
|
83
|
+
overfetch = k if not (tags or path_prefix) else min(max(k * 10, k), 500)
|
|
84
|
+
pairs = qdrant.query(qv, overfetch, ns) # [(id, score)] cosine similarity
|
|
85
|
+
if not pairs:
|
|
86
|
+
return []
|
|
87
|
+
score = {pid: s for pid, s in pairs}
|
|
88
|
+
where, params = _filters(ns, tags, path_prefix) # namespace, expiry, tags, subtree
|
|
89
|
+
sql = (f"SELECT id, body, tags, path::text FROM memory "
|
|
90
|
+
f"WHERE {where} AND id = ANY(%s)")
|
|
91
|
+
with conn.cursor() as cur:
|
|
92
|
+
cur.execute(sql, params + [list(score.keys())])
|
|
93
|
+
hits = [Hit(str(r[0]), r[1], list(r[2]), r[3], score[str(r[0])])
|
|
94
|
+
for r in cur.fetchall()]
|
|
95
|
+
hits.sort(key=lambda h: h.score, reverse=True) # Qdrant order, minus PG-filtered
|
|
96
|
+
return hits[:k]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _rrf(lists: Sequence[List[Hit]], k: int) -> List[Hit]:
|
|
100
|
+
scores: dict = {}
|
|
101
|
+
keep: dict = {}
|
|
102
|
+
for hits in lists:
|
|
103
|
+
for rank, h in enumerate(hits):
|
|
104
|
+
scores[h.id] = scores.get(h.id, 0.0) + 1.0 / (RRF_K + rank + 1)
|
|
105
|
+
keep[h.id] = h
|
|
106
|
+
fused = sorted(keep.values(), key=lambda h: scores[h.id], reverse=True)
|
|
107
|
+
for h in fused:
|
|
108
|
+
h.score = scores[h.id]
|
|
109
|
+
return fused[:k]
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def recall(conn, cfg, embedder, ns: str, query: str, *, k: int = 10,
|
|
113
|
+
tags: Optional[Sequence[str]] = None, path_prefix: Optional[str] = None,
|
|
114
|
+
mode: str = "auto", qdrant=None) -> List[Hit]:
|
|
115
|
+
if mode == "auto":
|
|
116
|
+
mode = "semantic" if embedder else "lexical"
|
|
117
|
+
if mode == "lexical":
|
|
118
|
+
return _lexical(conn, cfg, ns, query, k, tags, path_prefix)
|
|
119
|
+
if mode == "semantic":
|
|
120
|
+
return _semantic(conn, cfg, embedder, ns, query, k, tags, path_prefix, qdrant)
|
|
121
|
+
if mode == "hybrid":
|
|
122
|
+
lex = _lexical(conn, cfg, ns, query, k, tags, path_prefix)
|
|
123
|
+
sem = _semantic(conn, cfg, embedder, ns, query, k, tags, path_prefix, qdrant)
|
|
124
|
+
return _rrf([sem, lex], k)
|
|
125
|
+
raise ValueError(f"unknown recall mode: {mode!r} (lexical|semantic|hybrid|auto)")
|
memgres/server.py
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"""Thin HTTP layer over the store (FastAPI).
|
|
2
|
+
|
|
3
|
+
Deliberately a straight mapping from store operations to REST, so an auth or
|
|
4
|
+
billing layer can wrap these routes without touching store logic:
|
|
5
|
+
|
|
6
|
+
POST /memories create
|
|
7
|
+
GET /memories/{id} read (renews TTL)
|
|
8
|
+
PATCH /memories/{id} edit: whole body OR diff+base_hash; move; retag
|
|
9
|
+
POST /memories/{id}/move convenience reparent
|
|
10
|
+
DELETE /memories/{id} forget (hard-erase + history)
|
|
11
|
+
GET /memories/{id}/history provenance chain
|
|
12
|
+
GET /recall lexical / semantic / hybrid recall
|
|
13
|
+
GET /healthz liveness
|
|
14
|
+
|
|
15
|
+
The namespace token (when MEMGRES_NAMESPACES is on) comes in as a bearer token or
|
|
16
|
+
`X-Memgres-Token` header; recall/read are the cheap ops, writes the expensive ones
|
|
17
|
+
— the natural place for per-route pricing/metering later.
|
|
18
|
+
|
|
19
|
+
Concurrency: a psycopg_pool hands each request its own connection; the embedder
|
|
20
|
+
is built once and shared. Requires the `[server]` extra (fastapi, uvicorn,
|
|
21
|
+
psycopg_pool).
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from typing import List, Optional
|
|
25
|
+
|
|
26
|
+
from .config import Config, load
|
|
27
|
+
from .diffing import DiffConflict
|
|
28
|
+
from .embeddings import get_embedder
|
|
29
|
+
from .schema import migrate
|
|
30
|
+
from .store import Conflict, NoParent, NotFound, Store, TooLarge
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _parse_lines(spec: Optional[str]) -> Optional[List[int]]:
|
|
34
|
+
"""Parse a line selector like '2' or '1,3-5' into [2] / [1,3,4,5]. None → all."""
|
|
35
|
+
if not spec:
|
|
36
|
+
return None
|
|
37
|
+
out: List[int] = []
|
|
38
|
+
for part in spec.split(","):
|
|
39
|
+
part = part.strip()
|
|
40
|
+
if not part:
|
|
41
|
+
continue
|
|
42
|
+
if "-" in part:
|
|
43
|
+
a, b = part.split("-", 1)
|
|
44
|
+
out.extend(range(int(a), int(b) + 1))
|
|
45
|
+
else:
|
|
46
|
+
out.append(int(part))
|
|
47
|
+
return out or None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def create_app(cfg: Optional[Config] = None):
|
|
51
|
+
from contextlib import asynccontextmanager
|
|
52
|
+
|
|
53
|
+
from fastapi import Depends, FastAPI, Header, HTTPException, Query
|
|
54
|
+
from psycopg_pool import ConnectionPool
|
|
55
|
+
from pydantic import BaseModel
|
|
56
|
+
|
|
57
|
+
cfg = cfg or load()
|
|
58
|
+
embedder = get_embedder(cfg)
|
|
59
|
+
pool = ConnectionPool(cfg.database_url or "", min_size=1, open=False)
|
|
60
|
+
|
|
61
|
+
@asynccontextmanager
|
|
62
|
+
async def lifespan(app):
|
|
63
|
+
pool.open()
|
|
64
|
+
with pool.connection() as conn:
|
|
65
|
+
migrate(conn, cfg) # idempotent; stamps embed model/dim
|
|
66
|
+
yield
|
|
67
|
+
pool.close()
|
|
68
|
+
|
|
69
|
+
app = FastAPI(title="memgres", version="0.1.0", lifespan=lifespan)
|
|
70
|
+
|
|
71
|
+
# ─── request bodies ─────────────────────────────────────────────────────
|
|
72
|
+
class CreateBody(BaseModel):
|
|
73
|
+
body: str
|
|
74
|
+
path: Optional[str] = None
|
|
75
|
+
tags: Optional[List[str]] = None
|
|
76
|
+
source: Optional[str] = None
|
|
77
|
+
reason: Optional[str] = None
|
|
78
|
+
ttl_days: Optional[int] = None
|
|
79
|
+
|
|
80
|
+
class EditBody(BaseModel):
|
|
81
|
+
body: Optional[str] = None
|
|
82
|
+
diff: Optional[str] = None
|
|
83
|
+
base_hash: Optional[str] = None
|
|
84
|
+
path: Optional[str] = None
|
|
85
|
+
tags: Optional[List[str]] = None
|
|
86
|
+
source: Optional[str] = None
|
|
87
|
+
reason: Optional[str] = None
|
|
88
|
+
ttl_days: Optional[int] = None
|
|
89
|
+
|
|
90
|
+
class MoveBody(BaseModel):
|
|
91
|
+
path: str
|
|
92
|
+
source: Optional[str] = None
|
|
93
|
+
reason: Optional[str] = None
|
|
94
|
+
|
|
95
|
+
# ─── auth: extract the namespace token ──────────────────────────────────
|
|
96
|
+
def token(authorization: Optional[str] = Header(None),
|
|
97
|
+
x_memgres_token: Optional[str] = Header(None)) -> Optional[str]:
|
|
98
|
+
tok = x_memgres_token
|
|
99
|
+
if not tok and authorization and authorization.lower().startswith("bearer "):
|
|
100
|
+
tok = authorization[7:]
|
|
101
|
+
if cfg.namespaces_enabled and not tok:
|
|
102
|
+
raise HTTPException(401, "namespace token required")
|
|
103
|
+
return tok
|
|
104
|
+
|
|
105
|
+
def _mem(m) -> dict:
|
|
106
|
+
return {"id": m.id, "content_hash": m.content_hash, "body": m.body,
|
|
107
|
+
"tags": m.tags, "path": m.path, "seq": m.seq,
|
|
108
|
+
"created_at": m.created_at, "updated_at": m.updated_at,
|
|
109
|
+
"expires_at": m.expires_at}
|
|
110
|
+
|
|
111
|
+
def _store(conn):
|
|
112
|
+
return Store(cfg, embedder=embedder, conn=conn)
|
|
113
|
+
|
|
114
|
+
def _guard(fn):
|
|
115
|
+
"""Run a store call, mapping store exceptions to HTTP codes."""
|
|
116
|
+
try:
|
|
117
|
+
return fn()
|
|
118
|
+
except NotFound:
|
|
119
|
+
raise HTTPException(404, "not found")
|
|
120
|
+
except (Conflict, DiffConflict) as e:
|
|
121
|
+
raise HTTPException(409, str(e))
|
|
122
|
+
except NoParent as e:
|
|
123
|
+
raise HTTPException(409, str(e))
|
|
124
|
+
except TooLarge as e:
|
|
125
|
+
raise HTTPException(413, str(e))
|
|
126
|
+
except PermissionError as e:
|
|
127
|
+
raise HTTPException(401, str(e))
|
|
128
|
+
except ValueError as e:
|
|
129
|
+
raise HTTPException(422, str(e))
|
|
130
|
+
|
|
131
|
+
# ─── routes ─────────────────────────────────────────────────────────────
|
|
132
|
+
@app.get("/healthz")
|
|
133
|
+
def healthz():
|
|
134
|
+
return {"ok": True}
|
|
135
|
+
|
|
136
|
+
@app.post("/memories", status_code=201)
|
|
137
|
+
def create(req: CreateBody, tok: Optional[str] = Depends(token)):
|
|
138
|
+
with pool.connection() as conn:
|
|
139
|
+
m = _guard(lambda: _store(conn).write(
|
|
140
|
+
tok, body=req.body, path=req.path, tags=req.tags,
|
|
141
|
+
source=req.source, reason=req.reason, ttl_days=req.ttl_days))
|
|
142
|
+
return _mem(m)
|
|
143
|
+
|
|
144
|
+
@app.get("/memories/{mid}")
|
|
145
|
+
def read(mid: str, tok: Optional[str] = Depends(token)):
|
|
146
|
+
with pool.connection() as conn:
|
|
147
|
+
return _mem(_guard(lambda: _store(conn).get(tok, mid)))
|
|
148
|
+
|
|
149
|
+
@app.patch("/memories/{mid}")
|
|
150
|
+
def edit(mid: str, req: EditBody, tok: Optional[str] = Depends(token)):
|
|
151
|
+
with pool.connection() as conn:
|
|
152
|
+
m = _guard(lambda: _store(conn).write(
|
|
153
|
+
tok, id=mid, body=req.body, diff=req.diff, base_hash=req.base_hash,
|
|
154
|
+
path=req.path, tags=req.tags, source=req.source, reason=req.reason,
|
|
155
|
+
ttl_days=req.ttl_days))
|
|
156
|
+
return _mem(m)
|
|
157
|
+
|
|
158
|
+
@app.post("/memories/{mid}/move")
|
|
159
|
+
def move(mid: str, req: MoveBody, tok: Optional[str] = Depends(token)):
|
|
160
|
+
with pool.connection() as conn:
|
|
161
|
+
m = _guard(lambda: _store(conn).move(
|
|
162
|
+
tok, mid, req.path, source=req.source, reason=req.reason))
|
|
163
|
+
return _mem(m)
|
|
164
|
+
|
|
165
|
+
@app.delete("/memories/{mid}", status_code=204)
|
|
166
|
+
def forget(mid: str, tok: Optional[str] = Depends(token)):
|
|
167
|
+
with pool.connection() as conn:
|
|
168
|
+
ok = _guard(lambda: _store(conn).forget(tok, mid))
|
|
169
|
+
if not ok:
|
|
170
|
+
raise HTTPException(404, "not found")
|
|
171
|
+
|
|
172
|
+
@app.get("/memories/{mid}/history")
|
|
173
|
+
def history(mid: str, tok: Optional[str] = Depends(token)):
|
|
174
|
+
with pool.connection() as conn:
|
|
175
|
+
return _guard(lambda: _store(conn).history(tok, mid))
|
|
176
|
+
|
|
177
|
+
@app.get("/memories/{mid}/blame")
|
|
178
|
+
def blame(mid: str, upto_seq: Optional[int] = None,
|
|
179
|
+
group: bool = True, text: bool = True,
|
|
180
|
+
lines: Optional[str] = Query(None, description="e.g. '2' or '1,3-5'"),
|
|
181
|
+
tok: Optional[str] = Depends(token)):
|
|
182
|
+
"""Who last changed each line. Grouped into author-blocks by default
|
|
183
|
+
(`group=false` for per-line); `text=false` drops bodies for a pure
|
|
184
|
+
ownership map; `lines` selects specific 1-based lines/ranges (per-line)."""
|
|
185
|
+
want = _parse_lines(lines)
|
|
186
|
+
with pool.connection() as conn:
|
|
187
|
+
s = _store(conn)
|
|
188
|
+
if want is not None or not group:
|
|
189
|
+
return _guard(lambda: s.annotate(tok, mid, upto_seq, want))
|
|
190
|
+
return _guard(lambda: s.annotate_grouped(tok, mid, upto_seq, text))
|
|
191
|
+
|
|
192
|
+
@app.get("/memories/{mid}/at/{seq}")
|
|
193
|
+
def at_version(mid: str, seq: int, tok: Optional[str] = Depends(token)):
|
|
194
|
+
"""The exact body as it was at version `seq` (reconstructed from history)."""
|
|
195
|
+
with pool.connection() as conn:
|
|
196
|
+
body = _guard(lambda: _store(conn).reconstruct(tok, mid, seq))
|
|
197
|
+
return {"seq": seq, "body": body}
|
|
198
|
+
|
|
199
|
+
@app.get("/recall")
|
|
200
|
+
def recall(q: str, k: int = 10, mode: str = "auto",
|
|
201
|
+
tags: Optional[str] = Query(None, description="comma-separated"),
|
|
202
|
+
path_prefix: Optional[str] = None,
|
|
203
|
+
tok: Optional[str] = Depends(token)):
|
|
204
|
+
taglist = [t for t in (tags.split(",") if tags else []) if t]
|
|
205
|
+
with pool.connection() as conn:
|
|
206
|
+
hits = _guard(lambda: _store(conn).recall(
|
|
207
|
+
tok, q, k=k, tags=taglist or None, path_prefix=path_prefix, mode=mode))
|
|
208
|
+
return [{"id": h.id, "body": h.body, "tags": h.tags,
|
|
209
|
+
"path": h.path, "score": h.score} for h in hits]
|
|
210
|
+
|
|
211
|
+
return app
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def main(): # pragma: no cover - entrypoint
|
|
215
|
+
import os
|
|
216
|
+
import uvicorn
|
|
217
|
+
uvicorn.run(create_app(), host=os.environ.get("MEMGRES_HTTP_HOST", "0.0.0.0"),
|
|
218
|
+
port=int(os.environ.get("MEMGRES_HTTP_PORT", "8080")))
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
if __name__ == "__main__": # pragma: no cover
|
|
222
|
+
main()
|