memgres 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {memgres-0.1.0 → memgres-0.2.0}/PKG-INFO +135 -26
- {memgres-0.1.0 → memgres-0.2.0}/README.md +133 -25
- {memgres-0.1.0 → memgres-0.2.0}/memgres/__init__.py +12 -0
- {memgres-0.1.0 → memgres-0.2.0}/memgres/config.py +14 -2
- memgres-0.2.0/memgres/identity.py +453 -0
- memgres-0.2.0/memgres/mcp_server.py +323 -0
- memgres-0.2.0/memgres/migrations/0002_identity.sql +90 -0
- {memgres-0.1.0 → memgres-0.2.0}/memgres/schema.py +6 -4
- memgres-0.2.0/memgres/server.py +416 -0
- {memgres-0.1.0 → memgres-0.2.0}/memgres/store.py +64 -26
- {memgres-0.1.0 → memgres-0.2.0}/memgres.egg-info/PKG-INFO +135 -26
- {memgres-0.1.0 → memgres-0.2.0}/memgres.egg-info/SOURCES.txt +4 -0
- {memgres-0.1.0 → memgres-0.2.0}/memgres.egg-info/requires.txt +1 -0
- {memgres-0.1.0 → memgres-0.2.0}/pyproject.toml +2 -2
- {memgres-0.1.0 → memgres-0.2.0}/tests/test_config.py +1 -1
- memgres-0.2.0/tests/test_identity_integration.py +325 -0
- memgres-0.2.0/tests/test_security_integration.py +263 -0
- {memgres-0.1.0 → memgres-0.2.0}/tests/test_server_integration.py +84 -9
- {memgres-0.1.0 → memgres-0.2.0}/tests/test_store_integration.py +14 -14
- memgres-0.1.0/memgres/mcp_server.py +0 -114
- memgres-0.1.0/memgres/server.py +0 -222
- {memgres-0.1.0 → memgres-0.2.0}/LICENSE +0 -0
- {memgres-0.1.0 → memgres-0.2.0}/memgres/blame.py +0 -0
- {memgres-0.1.0 → memgres-0.2.0}/memgres/diffing.py +0 -0
- {memgres-0.1.0 → memgres-0.2.0}/memgres/embeddings.py +0 -0
- {memgres-0.1.0 → memgres-0.2.0}/memgres/migrations/0001_core.sql +0 -0
- {memgres-0.1.0 → memgres-0.2.0}/memgres/qdrant_backend.py +0 -0
- {memgres-0.1.0 → memgres-0.2.0}/memgres/search.py +0 -0
- {memgres-0.1.0 → memgres-0.2.0}/memgres.egg-info/dependency_links.txt +0 -0
- {memgres-0.1.0 → memgres-0.2.0}/memgres.egg-info/entry_points.txt +0 -0
- {memgres-0.1.0 → memgres-0.2.0}/memgres.egg-info/top_level.txt +0 -0
- {memgres-0.1.0 → memgres-0.2.0}/setup.cfg +0 -0
- {memgres-0.1.0 → memgres-0.2.0}/tests/test_blame_integration.py +0 -0
- {memgres-0.1.0 → memgres-0.2.0}/tests/test_diffing.py +0 -0
- {memgres-0.1.0 → memgres-0.2.0}/tests/test_embeddings.py +0 -0
- {memgres-0.1.0 → memgres-0.2.0}/tests/test_qdrant_integration.py +0 -0
- {memgres-0.1.0 → memgres-0.2.0}/tests/test_search_integration.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: memgres
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Drop-in memory for AI agents: one Postgres, lexical + semantic recall, diff-versioned history, GDPR-erasable.
|
|
5
5
|
Author: mozgsml
|
|
6
6
|
License-Expression: MIT
|
|
@@ -28,6 +28,7 @@ Requires-Dist: uvicorn>=0.29; extra == "server"
|
|
|
28
28
|
Requires-Dist: psycopg-pool>=3.2; extra == "server"
|
|
29
29
|
Provides-Extra: mcp
|
|
30
30
|
Requires-Dist: mcp>=1.2; extra == "mcp"
|
|
31
|
+
Requires-Dist: psycopg-pool>=3.2; extra == "mcp"
|
|
31
32
|
Provides-Extra: dev
|
|
32
33
|
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
33
34
|
Dynamic: license-file
|
|
@@ -39,7 +40,7 @@ Dynamic: license-file
|
|
|
39
40
|
|
|
40
41
|
**Versioned document memory for AI agents — one Postgres, lexical *or* semantic recall, diff-based history, GDPR-erasable.**
|
|
41
42
|
|
|
42
|
-
> Status:
|
|
43
|
+
> Status: v0.2.0 — on [PyPI](https://pypi.org/project/memgres/) (`pip install memgres`) and [GHCR](https://ghcr.io/mozgsml/memgres). Core library, search (pgvector/Qdrant), multi-tenant identity (users / namespaces / scoped tokens), HTTP API and MCP server, all tested in CI against live Postgres + Qdrant.
|
|
43
44
|
|
|
44
45
|
memgres is a lightweight, drop-in memory layer — a Python library plus an optional HTTP/MCP service — backed by a single PostgreSQL database. You store **documents** (bodies of text an agent owns and edits), not facts an LLM guessed at. Every change is an authored diff with provenance, kept in a tamper-evident history that you can still delete when the law says you must.
|
|
45
46
|
|
|
@@ -71,7 +72,7 @@ Reach for memgres when you want **auditable, authored, versioned text memory**.
|
|
|
71
72
|
| **Lexical *and* semantic (hybrid)** | Exact identifiers/codes go to lexical (where [dense retrieval alone stumbles](https://tianpan.co/blog/2026-04-12-hybrid-search-production-bm25-dense-embeddings)); meaning-based queries go to vectors; hybrid fuses both with RRF. |
|
|
72
73
|
| **Embedding-model safety by construction** | The model id + dimension are stamped into the schema; a mismatch **hard-fails** instead of silently returning garbage. |
|
|
73
74
|
| **TTL renewed on read** | Active memory persists because it's used; abandoned memory expires itself. Storage self-cleans instead of growing forever. |
|
|
74
|
-
| **Optional
|
|
75
|
+
| **Optional multi-tenant identity** | Users, namespaces and rotatable scoped tokens when you need isolation; nothing to configure for single-user. |
|
|
75
76
|
| **Fast subtree recall via `ltree`** | Memories form a real tree; `path <@ 'a.b'` pulls a whole subtree in one GiST index scan, no recursive walk that degrades with depth. |
|
|
76
77
|
| **Git-blame + version reconstruct** | Every line carries who last changed it (grouped into author-blocks); any past version reconstructs from history — no replaying diffs yourself. |
|
|
77
78
|
| **One Postgres, one backup** | The whole thing is `pg_dump`-able; the vector index rebuilds from the source of truth. No second datastore to run or back up. |
|
|
@@ -89,7 +90,7 @@ memgres (core library, no HTTP dependency)
|
|
|
89
90
|
├─ organize tags (text[] + GIN) · tree (ltree path + GiST, fast subtree select)
|
|
90
91
|
├─ search lexical (Postgres FTS) + semantic (pgvector or Qdrant) + hybrid
|
|
91
92
|
├─ embeddings provider via env: none | local (sentence-transformers) | cloud (Jina/OpenAI)
|
|
92
|
-
└─ config every limit via env (body/write size, TTL,
|
|
93
|
+
└─ config every limit via env (body/write size, TTL, key mode, …)
|
|
93
94
|
|
|
94
95
|
optional layers on top of the same core:
|
|
95
96
|
├─ HTTP API (FastAPI) REST + OpenAPI
|
|
@@ -98,18 +99,35 @@ optional layers on top of the same core:
|
|
|
98
99
|
|
|
99
100
|
**Record model:** one memory = one mutable body (up to a configurable ceiling, default 256 KB) plus metadata — `tags` (cross-cutting labels, `text[]` + GIN), a `path` (its place in an `ltree` tree), timestamps, and per-diff provenance (`source`/`reason`, kept in history). A single write/diff is capped smaller (default 16 KB), so large bodies accrue over many authored diffs. **Organization is two orthogonal axes:** the tree is *where a memory lives* (one place, subtree-selectable); tags are *what it's about* (many, overlapping). Both filter either search — narrow a semantic query to a subtree, or list a tag across the tree.
|
|
100
101
|
|
|
101
|
-
**Isolation:** optional
|
|
102
|
+
**Isolation:** optional multi-tenant identity keeps tenants from seeing each other's memories — users own *namespaces*, and rotatable, permission-scoped *tokens* authenticate as a user (turned on with `MEMGRES_KEY_MODE=open|managed`; see [docs/TENANCY.md](docs/TENANCY.md)). Encryption at rest is left to the deployment — Postgres/managed-PG/disk TDE stays transparent to queries, so search keeps working; memgres deliberately does **not** encrypt bodies application-side (that would make them unsearchable, which is why no comparable tool does it either). GDPR erasure is real: `forget()` hard-deletes the row, its vectors, and crypto-shreds the history chain. All limits are env-configurable, so the same code serves a single-user embed and a capped multi-tenant service.
|
|
102
103
|
|
|
103
104
|
---
|
|
104
105
|
|
|
105
106
|
## Quickstart
|
|
106
107
|
|
|
107
|
-
**
|
|
108
|
+
**Run everything with Docker** — `pgvector`, the memgres service (HTTP on `:8080`) and an MCP server (Streamable HTTP on `:8765`), schema auto-migrated on startup. Just grab the compose file (it pulls the published image, so there's nothing to build):
|
|
108
109
|
|
|
109
110
|
```bash
|
|
110
|
-
|
|
111
|
+
curl -O https://raw.githubusercontent.com/mozgsml/memgres/main/docker-compose.yml
|
|
112
|
+
docker compose up
|
|
111
113
|
```
|
|
112
114
|
|
|
115
|
+
Defaults suit a single-user setup with no auth. To change limits, the embedding provider, tokens, … drop a `.env` beside it — every `MEMGRES_*` is optional (see [Configuration](#configuration) or [.env.example](.env.example)).
|
|
116
|
+
|
|
117
|
+
**Give it to an LLM / agent — no code (MCP).** Point any URL-capable MCP client (Cursor, Cline, Claude Desktop, …) at the running server; the model gets `memory_write`, `memory_recall`, `memory_get`, `memory_blame`, `memory_history`, `memory_move`, `memory_forget` as tools:
|
|
118
|
+
|
|
119
|
+
```json
|
|
120
|
+
{
|
|
121
|
+
"mcpServers": {
|
|
122
|
+
"memgres": { "url": "http://localhost:8765/mcp" }
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Then just tell the model *"remember X"* / *"what do you know about Y?"* and it calls the tools — nothing else to run. (For stdio-only clients, semantic recall, and multi-tenant tokens, see [Use it with an LLM / agent (MCP)](#use-it-with-an-llm--agent-mcp) and [docs/TENANCY.md](docs/TENANCY.md).)
|
|
128
|
+
|
|
129
|
+
**Also a plain HTTP API** — the same service on `:8080` (`GET /healthz → {"ok":true}`), for when you drive the agent loop yourself:
|
|
130
|
+
|
|
113
131
|
```bash
|
|
114
132
|
# create a memory
|
|
115
133
|
curl -sX POST localhost:8080/memories \
|
|
@@ -131,6 +149,13 @@ curl -s localhost:8080/memories/$ID/blame
|
|
|
131
149
|
|
|
132
150
|
### As a Python library (no HTTP)
|
|
133
151
|
|
|
152
|
+
```bash
|
|
153
|
+
pip install memgres # core library
|
|
154
|
+
pip install "memgres[server]" # + HTTP API
|
|
155
|
+
pip install "memgres[mcp]" # + MCP server
|
|
156
|
+
# extras: local (sentence-transformers), qdrant (Qdrant backend)
|
|
157
|
+
```
|
|
158
|
+
|
|
134
159
|
```python
|
|
135
160
|
from memgres import Store, load_config, migrate
|
|
136
161
|
import psycopg
|
|
@@ -151,18 +176,7 @@ old = s.reconstruct(None, m.id, 1) # body as of versi
|
|
|
151
176
|
s.forget(None, m.id) # hard-erase + history
|
|
152
177
|
```
|
|
153
178
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
> Not on PyPI yet — install from git (or clone and `pip install -e .`):
|
|
157
|
-
|
|
158
|
-
```bash
|
|
159
|
-
pip install "git+https://github.com/mozgsml/memgres" # core library
|
|
160
|
-
pip install "memgres[server] @ git+https://github.com/mozgsml/memgres" # + HTTP API
|
|
161
|
-
pip install "memgres[mcp] @ git+https://github.com/mozgsml/memgres" # + MCP server
|
|
162
|
-
# extras: local (sentence-transformers), qdrant (Qdrant backend)
|
|
163
|
-
```
|
|
164
|
-
|
|
165
|
-
Or pull the container image:
|
|
179
|
+
Or pull the container image (public, no login):
|
|
166
180
|
|
|
167
181
|
```bash
|
|
168
182
|
docker pull ghcr.io/mozgsml/memgres:latest
|
|
@@ -183,11 +197,14 @@ Everything is env, all optional (defaults suit a single-user embed). Full list i
|
|
|
183
197
|
| Variable | Default | Meaning |
|
|
184
198
|
|---|---|---|
|
|
185
199
|
| `MEMGRES_DATABASE_URL` | libpq env | Postgres connection string |
|
|
200
|
+
| `MEMGRES_POOL_SIZE` | `4` | max pooled DB connections (HTTP + http-MCP servers); raise for many concurrent clients, `1` to serialize |
|
|
186
201
|
| `MEMGRES_MAX_BODY_BYTES` | `262144` | ceiling for a whole record body (256 KB) |
|
|
187
202
|
| `MEMGRES_MAX_WRITE_BYTES` | `16384` | ceiling for one write/diff payload (≤ body) |
|
|
188
203
|
| `MEMGRES_RETENTION_DAYS` | `0` | `0` = keep forever; `>0` = expire N days after last touch |
|
|
189
204
|
| `MEMGRES_RENEW_ON_READ` | `true` | a read pushes the expiry clock forward |
|
|
190
|
-
| `
|
|
205
|
+
| `MEMGRES_KEY_MODE` | `single` | `single` (no auth, one space) · `open` (bring-your-own token, self-registers) · `managed` (admin-provisioned). See [docs/TENANCY.md](docs/TENANCY.md) |
|
|
206
|
+
| `MEMGRES_ADMIN_TOKEN` | — | global admin bearer for provisioning (managed mode) |
|
|
207
|
+
| `MEMGRES_TOKEN` | — | default token used when a call passes none (single-tenant endpoints) |
|
|
191
208
|
| `MEMGRES_TREE` | `true` | `ltree` path column + GiST index (fast subtree select) |
|
|
192
209
|
| `MEMGRES_REQUIRE_PARENT` | `false` | `true` = a node's parent path must already exist |
|
|
193
210
|
| `MEMGRES_HISTORY` | `true` | keep the hash-chained diff history (deleted with the record) |
|
|
@@ -209,20 +226,112 @@ Everything is env, all optional (defaults suit a single-user embed). Full list i
|
|
|
209
226
|
| `GET` | `/memories/{id}/blame` | line attribution; `?group`, `?text`, `?lines=1,3-5` |
|
|
210
227
|
| `GET` | `/memories/{id}/at/{seq}` | body reconstructed at a version |
|
|
211
228
|
| `GET` | `/recall` | `?q=&k=&mode=&tags=&path_prefix=` |
|
|
229
|
+
| `GET` | `/spaces` | namespaces this token can reach (identity modes) |
|
|
212
230
|
| `GET` | `/healthz` | liveness |
|
|
213
231
|
|
|
214
|
-
|
|
232
|
+
Every memory/recall route also takes optional `space` (one of your namespaces by
|
|
233
|
+
name) and `space_id` (canonical id, for shared spaces). In `open`/`managed` mode
|
|
234
|
+
the token goes in `Authorization: Bearer <token>` or `X-Memgres-Token`; there are
|
|
235
|
+
also request-access and `/admin/*` provisioning routes — see
|
|
236
|
+
[docs/TENANCY.md](docs/TENANCY.md). OpenAPI/Swagger is at `/docs`. Store errors
|
|
237
|
+
map to status codes: `409` stale-hash conflict, `404` not found, `413` too large,
|
|
238
|
+
`401`/`403` auth.
|
|
239
|
+
|
|
240
|
+
## Use it with an LLM / agent (MCP)
|
|
215
241
|
|
|
216
|
-
|
|
242
|
+
memgres itself **never calls an LLM** — it's the memory, not the model. Your LLM
|
|
243
|
+
uses it one of two ways:
|
|
217
244
|
|
|
218
|
-
|
|
245
|
+
**A. Via MCP** — the model calls memgres tools directly (Cursor, Cline, Claude
|
|
246
|
+
Desktop, any MCP client). Zero code.
|
|
247
|
+
|
|
248
|
+
`docker compose up` already starts an MCP server over Streamable HTTP at
|
|
249
|
+
**`http://localhost:8765/mcp`**. Point a URL-capable MCP client at it — nothing else
|
|
250
|
+
to run:
|
|
251
|
+
|
|
252
|
+
```json
|
|
253
|
+
{
|
|
254
|
+
"mcpServers": {
|
|
255
|
+
"memgres": { "url": "http://localhost:8765/mcp" }
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
For **stdio-only** clients, install the command and let the client spawn it:
|
|
219
261
|
|
|
220
262
|
```bash
|
|
221
|
-
pip install "memgres[mcp]
|
|
222
|
-
|
|
263
|
+
pip install "memgres[mcp]"
|
|
264
|
+
```
|
|
265
|
+
```json
|
|
266
|
+
{
|
|
267
|
+
"mcpServers": {
|
|
268
|
+
"memgres": {
|
|
269
|
+
"command": "memgres-mcp",
|
|
270
|
+
"env": {
|
|
271
|
+
"MEMGRES_DATABASE_URL": "postgresql://memgres:memgres@localhost:5432/memgres",
|
|
272
|
+
"MEMGRES_KEY_MODE": "open",
|
|
273
|
+
"MEMGRES_TOKEN": "mgk_…"
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
}
|
|
223
278
|
```
|
|
224
279
|
|
|
225
|
-
|
|
280
|
+
Either way the model gets tools `memory_write`, `memory_recall`, `memory_get`,
|
|
281
|
+
`memory_blame`, `memory_history`, `memory_move`, `memory_forget`. Tell it *"remember
|
|
282
|
+
X"* / *"what do you know about Y?"* and it calls them. (For semantic recall add the
|
|
283
|
+
embedding env vars — see [docs/BACKENDS.md](docs/BACKENDS.md).)
|
|
284
|
+
|
|
285
|
+
**Isolation — pin the identity in the client config; the agent never handles the
|
|
286
|
+
token** (so the model spends nothing echoing a secret and can't switch user):
|
|
287
|
+
|
|
288
|
+
- **stdio**: set `MEMGRES_KEY_MODE=open` + `MEMGRES_TOKEN=<mgk_…>` in the client's
|
|
289
|
+
`env` block (above).
|
|
290
|
+
- **http**: send the token as a header — one shared endpoint then serves many
|
|
291
|
+
clients, each pinned to its own user:
|
|
292
|
+
```json
|
|
293
|
+
{ "mcpServers": { "memgres": {
|
|
294
|
+
"url": "http://localhost:8765/mcp",
|
|
295
|
+
"headers": { "Authorization": "Bearer mgk_…" } } } }
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
A *namespace-scoped* token also locks the agent to one space. Only a genuinely
|
|
299
|
+
multi-tenant endpoint (open/managed, **no** pinned token) exposes a `token` tool
|
|
300
|
+
argument for the model to supply — force it either way with
|
|
301
|
+
`MEMGRES_MCP_TOKEN_ARG=on|off`. Single mode needs no token. Full model in
|
|
302
|
+
[docs/TENANCY.md](docs/TENANCY.md).
|
|
303
|
+
|
|
304
|
+
**B. From your own agent code** — your loop calls the HTTP API or the `Store`
|
|
305
|
+
library after the model produces text (see the examples above). Use this when you
|
|
306
|
+
control the agent loop and decide when to write/recall.
|
|
307
|
+
|
|
308
|
+
## Tokens & auth
|
|
309
|
+
|
|
310
|
+
There is **no token for single-user / local use** — leave everything default
|
|
311
|
+
(`MEMGRES_KEY_MODE=single`) and it just works. Two token concepts exist, unrelated:
|
|
312
|
+
|
|
313
|
+
- **Embedding API key** (`MEMGRES_EMBED_API_KEY`) — only if you use a *cloud*
|
|
314
|
+
embedding provider (`openai`/`jina`) for semantic recall. Local models and
|
|
315
|
+
lexical-only need none. This is the key from your embedding provider.
|
|
316
|
+
- **Access token** (multi-tenant, `MEMGRES_KEY_MODE=open|managed`) — a bearer
|
|
317
|
+
credential of the form `mgk_` + 43 url-safe chars, authenticating *as a user*.
|
|
318
|
+
Tokens are rotatable, expirable, revocable, and restrictable (a permission
|
|
319
|
+
ceiling + optional scope to one namespace); the secret is stored only as a hash.
|
|
320
|
+
Rotating a token does **not** move you to a new empty space — many tokens can
|
|
321
|
+
back one user, and namespaces are addressed by name or id.
|
|
322
|
+
|
|
323
|
+
```bash
|
|
324
|
+
python -c "import secrets; print('mgk_'+secrets.token_urlsafe(32))" # open mode: mint your own
|
|
325
|
+
```
|
|
326
|
+
|
|
327
|
+
Sent as `Authorization: Bearer <token>` / `X-Memgres-Token` (HTTP + MCP over
|
|
328
|
+
http), the `token` argument (library), or `MEMGRES_TOKEN` in env for a
|
|
329
|
+
single-tenant endpoint. Over MCP the agent never passes it — you pin it in the
|
|
330
|
+
client config (env or headers). It's a bearer secret with **no recovery** —
|
|
331
|
+
treat it like a password.
|
|
332
|
+
|
|
333
|
+
Full model — users, namespaces, permissions, request-access, admin
|
|
334
|
+
provisioning — in **[docs/TENANCY.md](docs/TENANCY.md)**.
|
|
226
335
|
|
|
227
336
|
---
|
|
228
337
|
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
**Versioned document memory for AI agents — one Postgres, lexical *or* semantic recall, diff-based history, GDPR-erasable.**
|
|
7
7
|
|
|
8
|
-
> Status:
|
|
8
|
+
> Status: v0.2.0 — on [PyPI](https://pypi.org/project/memgres/) (`pip install memgres`) and [GHCR](https://ghcr.io/mozgsml/memgres). Core library, search (pgvector/Qdrant), multi-tenant identity (users / namespaces / scoped tokens), HTTP API and MCP server, all tested in CI against live Postgres + Qdrant.
|
|
9
9
|
|
|
10
10
|
memgres is a lightweight, drop-in memory layer — a Python library plus an optional HTTP/MCP service — backed by a single PostgreSQL database. You store **documents** (bodies of text an agent owns and edits), not facts an LLM guessed at. Every change is an authored diff with provenance, kept in a tamper-evident history that you can still delete when the law says you must.
|
|
11
11
|
|
|
@@ -37,7 +37,7 @@ Reach for memgres when you want **auditable, authored, versioned text memory**.
|
|
|
37
37
|
| **Lexical *and* semantic (hybrid)** | Exact identifiers/codes go to lexical (where [dense retrieval alone stumbles](https://tianpan.co/blog/2026-04-12-hybrid-search-production-bm25-dense-embeddings)); meaning-based queries go to vectors; hybrid fuses both with RRF. |
|
|
38
38
|
| **Embedding-model safety by construction** | The model id + dimension are stamped into the schema; a mismatch **hard-fails** instead of silently returning garbage. |
|
|
39
39
|
| **TTL renewed on read** | Active memory persists because it's used; abandoned memory expires itself. Storage self-cleans instead of growing forever. |
|
|
40
|
-
| **Optional
|
|
40
|
+
| **Optional multi-tenant identity** | Users, namespaces and rotatable scoped tokens when you need isolation; nothing to configure for single-user. |
|
|
41
41
|
| **Fast subtree recall via `ltree`** | Memories form a real tree; `path <@ 'a.b'` pulls a whole subtree in one GiST index scan, no recursive walk that degrades with depth. |
|
|
42
42
|
| **Git-blame + version reconstruct** | Every line carries who last changed it (grouped into author-blocks); any past version reconstructs from history — no replaying diffs yourself. |
|
|
43
43
|
| **One Postgres, one backup** | The whole thing is `pg_dump`-able; the vector index rebuilds from the source of truth. No second datastore to run or back up. |
|
|
@@ -55,7 +55,7 @@ memgres (core library, no HTTP dependency)
|
|
|
55
55
|
├─ organize tags (text[] + GIN) · tree (ltree path + GiST, fast subtree select)
|
|
56
56
|
├─ search lexical (Postgres FTS) + semantic (pgvector or Qdrant) + hybrid
|
|
57
57
|
├─ embeddings provider via env: none | local (sentence-transformers) | cloud (Jina/OpenAI)
|
|
58
|
-
└─ config every limit via env (body/write size, TTL,
|
|
58
|
+
└─ config every limit via env (body/write size, TTL, key mode, …)
|
|
59
59
|
|
|
60
60
|
optional layers on top of the same core:
|
|
61
61
|
├─ HTTP API (FastAPI) REST + OpenAPI
|
|
@@ -64,18 +64,35 @@ optional layers on top of the same core:
|
|
|
64
64
|
|
|
65
65
|
**Record model:** one memory = one mutable body (up to a configurable ceiling, default 256 KB) plus metadata — `tags` (cross-cutting labels, `text[]` + GIN), a `path` (its place in an `ltree` tree), timestamps, and per-diff provenance (`source`/`reason`, kept in history). A single write/diff is capped smaller (default 16 KB), so large bodies accrue over many authored diffs. **Organization is two orthogonal axes:** the tree is *where a memory lives* (one place, subtree-selectable); tags are *what it's about* (many, overlapping). Both filter either search — narrow a semantic query to a subtree, or list a tag across the tree.
|
|
66
66
|
|
|
67
|
-
**Isolation:** optional
|
|
67
|
+
**Isolation:** optional multi-tenant identity keeps tenants from seeing each other's memories — users own *namespaces*, and rotatable, permission-scoped *tokens* authenticate as a user (turned on with `MEMGRES_KEY_MODE=open|managed`; see [docs/TENANCY.md](docs/TENANCY.md)). Encryption at rest is left to the deployment — Postgres/managed-PG/disk TDE stays transparent to queries, so search keeps working; memgres deliberately does **not** encrypt bodies application-side (that would make them unsearchable, which is why no comparable tool does it either). GDPR erasure is real: `forget()` hard-deletes the row, its vectors, and crypto-shreds the history chain. All limits are env-configurable, so the same code serves a single-user embed and a capped multi-tenant service.
|
|
68
68
|
|
|
69
69
|
---
|
|
70
70
|
|
|
71
71
|
## Quickstart
|
|
72
72
|
|
|
73
|
-
**
|
|
73
|
+
**Run everything with Docker** — `pgvector`, the memgres service (HTTP on `:8080`) and an MCP server (Streamable HTTP on `:8765`), schema auto-migrated on startup. Just grab the compose file (it pulls the published image, so there's nothing to build):
|
|
74
74
|
|
|
75
75
|
```bash
|
|
76
|
-
|
|
76
|
+
curl -O https://raw.githubusercontent.com/mozgsml/memgres/main/docker-compose.yml
|
|
77
|
+
docker compose up
|
|
77
78
|
```
|
|
78
79
|
|
|
80
|
+
Defaults suit a single-user setup with no auth. To change limits, the embedding provider, tokens, … drop a `.env` beside it — every `MEMGRES_*` is optional (see [Configuration](#configuration) or [.env.example](.env.example)).
|
|
81
|
+
|
|
82
|
+
**Give it to an LLM / agent — no code (MCP).** Point any URL-capable MCP client (Cursor, Cline, Claude Desktop, …) at the running server; the model gets `memory_write`, `memory_recall`, `memory_get`, `memory_blame`, `memory_history`, `memory_move`, `memory_forget` as tools:
|
|
83
|
+
|
|
84
|
+
```json
|
|
85
|
+
{
|
|
86
|
+
"mcpServers": {
|
|
87
|
+
"memgres": { "url": "http://localhost:8765/mcp" }
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Then just tell the model *"remember X"* / *"what do you know about Y?"* and it calls the tools — nothing else to run. (For stdio-only clients, semantic recall, and multi-tenant tokens, see [Use it with an LLM / agent (MCP)](#use-it-with-an-llm--agent-mcp) and [docs/TENANCY.md](docs/TENANCY.md).)
|
|
93
|
+
|
|
94
|
+
**Also a plain HTTP API** — the same service on `:8080` (`GET /healthz → {"ok":true}`), for when you drive the agent loop yourself:
|
|
95
|
+
|
|
79
96
|
```bash
|
|
80
97
|
# create a memory
|
|
81
98
|
curl -sX POST localhost:8080/memories \
|
|
@@ -97,6 +114,13 @@ curl -s localhost:8080/memories/$ID/blame
|
|
|
97
114
|
|
|
98
115
|
### As a Python library (no HTTP)
|
|
99
116
|
|
|
117
|
+
```bash
|
|
118
|
+
pip install memgres # core library
|
|
119
|
+
pip install "memgres[server]" # + HTTP API
|
|
120
|
+
pip install "memgres[mcp]" # + MCP server
|
|
121
|
+
# extras: local (sentence-transformers), qdrant (Qdrant backend)
|
|
122
|
+
```
|
|
123
|
+
|
|
100
124
|
```python
|
|
101
125
|
from memgres import Store, load_config, migrate
|
|
102
126
|
import psycopg
|
|
@@ -117,18 +141,7 @@ old = s.reconstruct(None, m.id, 1) # body as of versi
|
|
|
117
141
|
s.forget(None, m.id) # hard-erase + history
|
|
118
142
|
```
|
|
119
143
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
> Not on PyPI yet — install from git (or clone and `pip install -e .`):
|
|
123
|
-
|
|
124
|
-
```bash
|
|
125
|
-
pip install "git+https://github.com/mozgsml/memgres" # core library
|
|
126
|
-
pip install "memgres[server] @ git+https://github.com/mozgsml/memgres" # + HTTP API
|
|
127
|
-
pip install "memgres[mcp] @ git+https://github.com/mozgsml/memgres" # + MCP server
|
|
128
|
-
# extras: local (sentence-transformers), qdrant (Qdrant backend)
|
|
129
|
-
```
|
|
130
|
-
|
|
131
|
-
Or pull the container image:
|
|
144
|
+
Or pull the container image (public, no login):
|
|
132
145
|
|
|
133
146
|
```bash
|
|
134
147
|
docker pull ghcr.io/mozgsml/memgres:latest
|
|
@@ -149,11 +162,14 @@ Everything is env, all optional (defaults suit a single-user embed). Full list i
|
|
|
149
162
|
| Variable | Default | Meaning |
|
|
150
163
|
|---|---|---|
|
|
151
164
|
| `MEMGRES_DATABASE_URL` | libpq env | Postgres connection string |
|
|
165
|
+
| `MEMGRES_POOL_SIZE` | `4` | max pooled DB connections (HTTP + http-MCP servers); raise for many concurrent clients, `1` to serialize |
|
|
152
166
|
| `MEMGRES_MAX_BODY_BYTES` | `262144` | ceiling for a whole record body (256 KB) |
|
|
153
167
|
| `MEMGRES_MAX_WRITE_BYTES` | `16384` | ceiling for one write/diff payload (≤ body) |
|
|
154
168
|
| `MEMGRES_RETENTION_DAYS` | `0` | `0` = keep forever; `>0` = expire N days after last touch |
|
|
155
169
|
| `MEMGRES_RENEW_ON_READ` | `true` | a read pushes the expiry clock forward |
|
|
156
|
-
| `
|
|
170
|
+
| `MEMGRES_KEY_MODE` | `single` | `single` (no auth, one space) · `open` (bring-your-own token, self-registers) · `managed` (admin-provisioned). See [docs/TENANCY.md](docs/TENANCY.md) |
|
|
171
|
+
| `MEMGRES_ADMIN_TOKEN` | — | global admin bearer for provisioning (managed mode) |
|
|
172
|
+
| `MEMGRES_TOKEN` | — | default token used when a call passes none (single-tenant endpoints) |
|
|
157
173
|
| `MEMGRES_TREE` | `true` | `ltree` path column + GiST index (fast subtree select) |
|
|
158
174
|
| `MEMGRES_REQUIRE_PARENT` | `false` | `true` = a node's parent path must already exist |
|
|
159
175
|
| `MEMGRES_HISTORY` | `true` | keep the hash-chained diff history (deleted with the record) |
|
|
@@ -175,20 +191,112 @@ Everything is env, all optional (defaults suit a single-user embed). Full list i
|
|
|
175
191
|
| `GET` | `/memories/{id}/blame` | line attribution; `?group`, `?text`, `?lines=1,3-5` |
|
|
176
192
|
| `GET` | `/memories/{id}/at/{seq}` | body reconstructed at a version |
|
|
177
193
|
| `GET` | `/recall` | `?q=&k=&mode=&tags=&path_prefix=` |
|
|
194
|
+
| `GET` | `/spaces` | namespaces this token can reach (identity modes) |
|
|
178
195
|
| `GET` | `/healthz` | liveness |
|
|
179
196
|
|
|
180
|
-
|
|
197
|
+
Every memory/recall route also takes optional `space` (one of your namespaces by
|
|
198
|
+
name) and `space_id` (canonical id, for shared spaces). In `open`/`managed` mode
|
|
199
|
+
the token goes in `Authorization: Bearer <token>` or `X-Memgres-Token`; there are
|
|
200
|
+
also request-access and `/admin/*` provisioning routes — see
|
|
201
|
+
[docs/TENANCY.md](docs/TENANCY.md). OpenAPI/Swagger is at `/docs`. Store errors
|
|
202
|
+
map to status codes: `409` stale-hash conflict, `404` not found, `413` too large,
|
|
203
|
+
`401`/`403` auth.
|
|
204
|
+
|
|
205
|
+
## Use it with an LLM / agent (MCP)
|
|
181
206
|
|
|
182
|
-
|
|
207
|
+
memgres itself **never calls an LLM** — it's the memory, not the model. Your LLM
|
|
208
|
+
uses it one of two ways:
|
|
183
209
|
|
|
184
|
-
|
|
210
|
+
**A. Via MCP** — the model calls memgres tools directly (Cursor, Cline, Claude
|
|
211
|
+
Desktop, any MCP client). Zero code.
|
|
212
|
+
|
|
213
|
+
`docker compose up` already starts an MCP server over Streamable HTTP at
|
|
214
|
+
**`http://localhost:8765/mcp`**. Point a URL-capable MCP client at it — nothing else
|
|
215
|
+
to run:
|
|
216
|
+
|
|
217
|
+
```json
|
|
218
|
+
{
|
|
219
|
+
"mcpServers": {
|
|
220
|
+
"memgres": { "url": "http://localhost:8765/mcp" }
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
For **stdio-only** clients, install the command and let the client spawn it:
|
|
185
226
|
|
|
186
227
|
```bash
|
|
187
|
-
pip install "memgres[mcp]
|
|
188
|
-
|
|
228
|
+
pip install "memgres[mcp]"
|
|
229
|
+
```
|
|
230
|
+
```json
|
|
231
|
+
{
|
|
232
|
+
"mcpServers": {
|
|
233
|
+
"memgres": {
|
|
234
|
+
"command": "memgres-mcp",
|
|
235
|
+
"env": {
|
|
236
|
+
"MEMGRES_DATABASE_URL": "postgresql://memgres:memgres@localhost:5432/memgres",
|
|
237
|
+
"MEMGRES_KEY_MODE": "open",
|
|
238
|
+
"MEMGRES_TOKEN": "mgk_…"
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
189
243
|
```
|
|
190
244
|
|
|
191
|
-
|
|
245
|
+
Either way the model gets tools `memory_write`, `memory_recall`, `memory_get`,
|
|
246
|
+
`memory_blame`, `memory_history`, `memory_move`, `memory_forget`. Tell it *"remember
|
|
247
|
+
X"* / *"what do you know about Y?"* and it calls them. (For semantic recall add the
|
|
248
|
+
embedding env vars — see [docs/BACKENDS.md](docs/BACKENDS.md).)
|
|
249
|
+
|
|
250
|
+
**Isolation — pin the identity in the client config; the agent never handles the
|
|
251
|
+
token** (so the model spends nothing echoing a secret and can't switch user):
|
|
252
|
+
|
|
253
|
+
- **stdio**: set `MEMGRES_KEY_MODE=open` + `MEMGRES_TOKEN=<mgk_…>` in the client's
|
|
254
|
+
`env` block (above).
|
|
255
|
+
- **http**: send the token as a header — one shared endpoint then serves many
|
|
256
|
+
clients, each pinned to its own user:
|
|
257
|
+
```json
|
|
258
|
+
{ "mcpServers": { "memgres": {
|
|
259
|
+
"url": "http://localhost:8765/mcp",
|
|
260
|
+
"headers": { "Authorization": "Bearer mgk_…" } } } }
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
A *namespace-scoped* token also locks the agent to one space. Only a genuinely
|
|
264
|
+
multi-tenant endpoint (open/managed, **no** pinned token) exposes a `token` tool
|
|
265
|
+
argument for the model to supply — force it either way with
|
|
266
|
+
`MEMGRES_MCP_TOKEN_ARG=on|off`. Single mode needs no token. Full model in
|
|
267
|
+
[docs/TENANCY.md](docs/TENANCY.md).
|
|
268
|
+
|
|
269
|
+
**B. From your own agent code** — your loop calls the HTTP API or the `Store`
|
|
270
|
+
library after the model produces text (see the examples above). Use this when you
|
|
271
|
+
control the agent loop and decide when to write/recall.
|
|
272
|
+
|
|
273
|
+
## Tokens & auth
|
|
274
|
+
|
|
275
|
+
There is **no token for single-user / local use** — leave everything default
|
|
276
|
+
(`MEMGRES_KEY_MODE=single`) and it just works. Two token concepts exist, unrelated:
|
|
277
|
+
|
|
278
|
+
- **Embedding API key** (`MEMGRES_EMBED_API_KEY`) — only if you use a *cloud*
|
|
279
|
+
embedding provider (`openai`/`jina`) for semantic recall. Local models and
|
|
280
|
+
lexical-only need none. This is the key from your embedding provider.
|
|
281
|
+
- **Access token** (multi-tenant, `MEMGRES_KEY_MODE=open|managed`) — a bearer
|
|
282
|
+
credential of the form `mgk_` + 43 url-safe chars, authenticating *as a user*.
|
|
283
|
+
Tokens are rotatable, expirable, revocable, and restrictable (a permission
|
|
284
|
+
ceiling + optional scope to one namespace); the secret is stored only as a hash.
|
|
285
|
+
Rotating a token does **not** move you to a new empty space — many tokens can
|
|
286
|
+
back one user, and namespaces are addressed by name or id.
|
|
287
|
+
|
|
288
|
+
```bash
|
|
289
|
+
python -c "import secrets; print('mgk_'+secrets.token_urlsafe(32))" # open mode: mint your own
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
Sent as `Authorization: Bearer <token>` / `X-Memgres-Token` (HTTP + MCP over
|
|
293
|
+
http), the `token` argument (library), or `MEMGRES_TOKEN` in env for a
|
|
294
|
+
single-tenant endpoint. Over MCP the agent never passes it — you pin it in the
|
|
295
|
+
client config (env or headers). It's a bearer secret with **no recovery** —
|
|
296
|
+
treat it like a password.
|
|
297
|
+
|
|
298
|
+
Full model — users, namespaces, permissions, request-access, admin
|
|
299
|
+
provisioning — in **[docs/TENANCY.md](docs/TENANCY.md)**.
|
|
192
300
|
|
|
193
301
|
---
|
|
194
302
|
|
|
@@ -22,6 +22,13 @@ from .schema import migrate, SchemaMismatch, SCHEMA_VERSION
|
|
|
22
22
|
from .search import Hit, recall
|
|
23
23
|
from .blame import annotate, annotate_grouped, reconstruct, replay
|
|
24
24
|
from .store import Store, Memory, Conflict, NotFound, TooLarge, NoParent
|
|
25
|
+
from .identity import (
|
|
26
|
+
Principal, AuthError, SpaceNotFound,
|
|
27
|
+
resolve, resolve_space, new_token, valid_format,
|
|
28
|
+
create_user, create_namespace, list_spaces,
|
|
29
|
+
issue_token, register_token, revoke_token, list_tokens,
|
|
30
|
+
request_access, approve_request, deny_request, list_requests,
|
|
31
|
+
)
|
|
25
32
|
|
|
26
33
|
__all__ = [
|
|
27
34
|
"Config", "load_config",
|
|
@@ -31,4 +38,9 @@ __all__ = [
|
|
|
31
38
|
"migrate", "SchemaMismatch", "SCHEMA_VERSION",
|
|
32
39
|
"Hit", "recall",
|
|
33
40
|
"annotate", "annotate_grouped", "reconstruct", "replay",
|
|
41
|
+
"Principal", "AuthError", "SpaceNotFound",
|
|
42
|
+
"resolve", "resolve_space", "new_token", "valid_format",
|
|
43
|
+
"create_user", "create_namespace", "list_spaces",
|
|
44
|
+
"issue_token", "register_token", "revoke_token", "list_tokens",
|
|
45
|
+
"request_access", "approve_request", "deny_request", "list_requests",
|
|
34
46
|
]
|
|
@@ -49,7 +49,11 @@ class Config:
|
|
|
49
49
|
retention_days: int # 0 = forever; >0 = expire N days after last touch
|
|
50
50
|
renew_on_read: bool # a read pushes the expiry clock forward
|
|
51
51
|
# multi-tenant isolation
|
|
52
|
-
|
|
52
|
+
token: str # default token used when a call passes none
|
|
53
|
+
# (set in MCP/env for a single-tenant deployment)
|
|
54
|
+
# identity / tenancy (see docs/TENANCY.md)
|
|
55
|
+
key_mode: str # single | open | managed (how tokens/users are minted)
|
|
56
|
+
admin_token: str # global-admin bearer: provision users/namespaces anywhere
|
|
53
57
|
# organization
|
|
54
58
|
tree_enabled: bool # ltree path column + GiST index for fast subtree selection
|
|
55
59
|
require_parent: bool # False = sparse paths (create food.apple with no food row);
|
|
@@ -67,8 +71,11 @@ class Config:
|
|
|
67
71
|
embed_api_base: str
|
|
68
72
|
# database
|
|
69
73
|
database_url: str
|
|
74
|
+
pool_size: int # max pooled connections (HTTP + http-MCP servers)
|
|
70
75
|
|
|
71
76
|
def validate(self) -> None:
|
|
77
|
+
if self.pool_size < 1:
|
|
78
|
+
raise ValueError("MEMGRES_POOL_SIZE must be >= 1")
|
|
72
79
|
if self.max_write_bytes > self.max_body_bytes:
|
|
73
80
|
raise ValueError(
|
|
74
81
|
"MEMGRES_MAX_WRITE_BYTES must be <= MEMGRES_MAX_BODY_BYTES"
|
|
@@ -79,6 +86,8 @@ class Config:
|
|
|
79
86
|
raise ValueError(f"unknown MEMGRES_EMBED_PROVIDER: {self.embed_provider}")
|
|
80
87
|
if self.vector_backend not in ("pgvector", "qdrant"):
|
|
81
88
|
raise ValueError(f"unknown MEMGRES_VECTOR_BACKEND: {self.vector_backend}")
|
|
89
|
+
if self.key_mode not in ("single", "open", "managed"):
|
|
90
|
+
raise ValueError(f"unknown MEMGRES_KEY_MODE: {self.key_mode}")
|
|
82
91
|
if self.embed_provider != "none" and self.vector_backend == "pgvector" \
|
|
83
92
|
and self.embed_dim <= 0:
|
|
84
93
|
raise ValueError(
|
|
@@ -94,7 +103,9 @@ def load() -> Config:
|
|
|
94
103
|
max_write_bytes=_int("MEMGRES_MAX_WRITE_BYTES", 16_384), # 16 KB
|
|
95
104
|
retention_days=_int("MEMGRES_RETENTION_DAYS", 0),
|
|
96
105
|
renew_on_read=_bool("MEMGRES_RENEW_ON_READ", True),
|
|
97
|
-
|
|
106
|
+
token=_str("MEMGRES_TOKEN", ""),
|
|
107
|
+
key_mode=_str("MEMGRES_KEY_MODE", "single"),
|
|
108
|
+
admin_token=_str("MEMGRES_ADMIN_TOKEN", ""),
|
|
98
109
|
tree_enabled=_bool("MEMGRES_TREE", True),
|
|
99
110
|
require_parent=_bool("MEMGRES_REQUIRE_PARENT", False),
|
|
100
111
|
history_enabled=_bool("MEMGRES_HISTORY", True),
|
|
@@ -106,6 +117,7 @@ def load() -> Config:
|
|
|
106
117
|
embed_api_key=_str("MEMGRES_EMBED_API_KEY", ""),
|
|
107
118
|
embed_api_base=_str("MEMGRES_EMBED_API_BASE", ""),
|
|
108
119
|
database_url=_str("MEMGRES_DATABASE_URL", ""),
|
|
120
|
+
pool_size=_int("MEMGRES_POOL_SIZE", 4),
|
|
109
121
|
)
|
|
110
122
|
cfg.validate()
|
|
111
123
|
return cfg
|