memoryintelligence-mcp 0.1.10__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {memoryintelligence_mcp-0.1.10 → memoryintelligence_mcp-0.2.0}/CHANGELOG.md +50 -0
- {memoryintelligence_mcp-0.1.10 → memoryintelligence_mcp-0.2.0}/PKG-INFO +13 -2
- {memoryintelligence_mcp-0.1.10 → memoryintelligence_mcp-0.2.0}/README.md +8 -1
- {memoryintelligence_mcp-0.1.10 → memoryintelligence_mcp-0.2.0}/pyproject.toml +10 -3
- {memoryintelligence_mcp-0.1.10 → memoryintelligence_mcp-0.2.0}/src/mi_mcp/__init__.py +1 -1
- {memoryintelligence_mcp-0.1.10 → memoryintelligence_mcp-0.2.0}/src/mi_mcp/__main__.py +15 -1
- {memoryintelligence_mcp-0.1.10 → memoryintelligence_mcp-0.2.0}/src/mi_mcp/cli.py +321 -7
- {memoryintelligence_mcp-0.1.10 → memoryintelligence_mcp-0.2.0}/src/mi_mcp/client.py +91 -16
- {memoryintelligence_mcp-0.1.10 → memoryintelligence_mcp-0.2.0}/src/mi_mcp/config.py +8 -0
- memoryintelligence_mcp-0.2.0/src/mi_mcp/embedder.py +84 -0
- memoryintelligence_mcp-0.2.0/src/mi_mcp/indexer.py +290 -0
- memoryintelligence_mcp-0.2.0/src/mi_mcp/keys.py +231 -0
- memoryintelligence_mcp-0.2.0/src/mi_mcp/local_index.py +190 -0
- memoryintelligence_mcp-0.2.0/src/mi_mcp/localreads.py +114 -0
- {memoryintelligence_mcp-0.1.10 → memoryintelligence_mcp-0.2.0}/src/mi_mcp/paths.py +11 -0
- memoryintelligence_mcp-0.2.0/src/mi_mcp/scrub.py +135 -0
- {memoryintelligence_mcp-0.1.10 → memoryintelligence_mcp-0.2.0}/src/mi_mcp/server.py +164 -30
- {memoryintelligence_mcp-0.1.10 → memoryintelligence_mcp-0.2.0}/src/mi_mcp/umo_format.py +8 -0
- memoryintelligence_mcp-0.1.10/src/mi_mcp/keys.py +0 -125
- {memoryintelligence_mcp-0.1.10 → memoryintelligence_mcp-0.2.0}/.gitignore +0 -0
- {memoryintelligence_mcp-0.1.10 → memoryintelligence_mcp-0.2.0}/CONTRIBUTING.md +0 -0
- {memoryintelligence_mcp-0.1.10 → memoryintelligence_mcp-0.2.0}/LICENSE +0 -0
- {memoryintelligence_mcp-0.1.10 → memoryintelligence_mcp-0.2.0}/src/mi_mcp/vault.py +0 -0
|
@@ -3,6 +3,56 @@
|
|
|
3
3
|
All notable changes to `memoryintelligence-mcp` are documented here.
|
|
4
4
|
The format follows [Keep a Changelog](https://keepachangelog.com/); this project uses [Semantic Versioning](https://semver.org/).
|
|
5
5
|
|
|
6
|
+
## [0.2.0] — 2026-07-04
|
|
7
|
+
|
|
8
|
+
### Added — the local vault (Path A), previously built on `main` but never released
|
|
9
|
+
The published `0.1.12` shipped as a thin cloud client; the entire local-vault stack
|
|
10
|
+
landed on `main` afterward **under the same version number** and was never cut into a
|
|
11
|
+
release. This release publishes it (release-hygiene fix — no new code, just a version
|
|
12
|
+
bump over what `main` already carried).
|
|
13
|
+
|
|
14
|
+
- **`backfill --execute` now writes the local vault** (`cli.py`): the cloud → local
|
|
15
|
+
migration re-embeds each memory locally, encrypts to the owner's key, and writes a
|
|
16
|
+
signed `.umo`. The prior published build's `--execute` was a dry-run stub.
|
|
17
|
+
- **Offline reads** — `local_index.py` + `localreads.py` + `indexer.py`: a flat-numpy
|
|
18
|
+
cosine index over the decrypted vault, mirroring the hosted ranking, so `mi_ask` works
|
|
19
|
+
network-free (needs the `[local]` extra: `cryptography` + `numpy`).
|
|
20
|
+
- **`mi-mcp index {build,stat,path}`** — build/inspect the local vector index.
|
|
21
|
+
- **On-device redaction** (`scrub.py`) applied on the local read path.
|
|
22
|
+
- **`embedder.py`** — local bge-small embeddings for backfill + query.
|
|
23
|
+
- Note (MI#653): `mi-mcp` still defaults its vault to `~/MemoryIntelligence`; the
|
|
24
|
+
MemorySpace Desktop vault is `~/Somewhere`. Until `wire`/`setup` sets
|
|
25
|
+
`MI_VAULT=~/Somewhere`, point it there manually so the two surfaces share one vault.
|
|
26
|
+
|
|
27
|
+
## [0.1.12] — 2026-06-16
|
|
28
|
+
|
|
29
|
+
### Fixed
|
|
30
|
+
- **The MCP client now retries on transient failures instead of failing the
|
|
31
|
+
first time the API is slow to wake.** The Railway `sdk-api` is a single small
|
|
32
|
+
instance that intermittently cold-starts or saturates (CPU-bound embedding +
|
|
33
|
+
pgvector rerank), and the client had a hard 30s budget with **no retry** — the
|
|
34
|
+
source of the "MCP keeps timing out" reports. Now:
|
|
35
|
+
- **Idempotent reads** (`mi_ask`, `mi_list`, `mi_explain`, `mi_verify`,
|
|
36
|
+
`mi_match`, account lookup) retry up to 2× on a read timeout or a transient
|
|
37
|
+
5xx (502/503/504), with exponential backoff (0.5s, 1.0s).
|
|
38
|
+
- **Writes** (`mi_capture`, `mi_upload`, `mi_forget`, batch) are **not**
|
|
39
|
+
read-retried — a timeout after the request body landed could double-apply.
|
|
40
|
+
- **Connection-level retries** (transport `retries=2`) cover the cold-start
|
|
41
|
+
`ConnectError` case for *every* verb, since no body is re-sent when the
|
|
42
|
+
connection never established.
|
|
43
|
+
|
|
44
|
+
This is a client-side reliability mitigation. The durable fix is local-vault
|
|
45
|
+
reads (no network round-trip on recall) — tracked separately.
|
|
46
|
+
|
|
47
|
+
## [0.1.11] — 2026-06-16
|
|
48
|
+
|
|
49
|
+
### Added
|
|
50
|
+
- **`mi_upload` is now part of the default tool surface** — file-capture parity
|
|
51
|
+
with the API. The MCP previously exposed only text capture; `mi_upload` now
|
|
52
|
+
ships in the visible tool set and its description covers the full capture
|
|
53
|
+
matrix: structured files (csv/tsv/xlsx/json → typed claims), documents
|
|
54
|
+
(pdf/docx), images (→ OCR), and audio/video (→ transcription).
|
|
55
|
+
|
|
6
56
|
## [0.1.10] — 2026-06-15
|
|
7
57
|
|
|
8
58
|
### Changed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: memoryintelligence-mcp
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: MCP server for MemoryIntelligence — give Claude a memory you own, set up in one command
|
|
5
5
|
Project-URL: Homepage, https://memoryintelligence.io
|
|
6
6
|
Project-URL: Repository, https://github.com/somewhere11/memoryintelligence-mcp
|
|
@@ -26,11 +26,15 @@ Requires-Dist: mcp[cli]<2.0,>=1.0.0
|
|
|
26
26
|
Requires-Dist: pydantic<3.0,>=2.5.0
|
|
27
27
|
Provides-Extra: dev
|
|
28
28
|
Requires-Dist: cryptography<46,>=42.0; extra == 'dev'
|
|
29
|
+
Requires-Dist: fastembed<1.0,>=0.3.0; extra == 'dev'
|
|
30
|
+
Requires-Dist: numpy<3,>=1.24; extra == 'dev'
|
|
29
31
|
Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
|
|
30
32
|
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
31
33
|
Requires-Dist: ruff>=0.4.0; extra == 'dev'
|
|
32
34
|
Provides-Extra: local
|
|
33
35
|
Requires-Dist: cryptography<46,>=42.0; extra == 'local'
|
|
36
|
+
Requires-Dist: fastembed<1.0,>=0.3.0; extra == 'local'
|
|
37
|
+
Requires-Dist: numpy<3,>=1.24; extra == 'local'
|
|
34
38
|
Description-Content-Type: text/markdown
|
|
35
39
|
|
|
36
40
|
# MemoryIntelligence MCP Server
|
|
@@ -77,10 +81,17 @@ API key is ever written into a config file.**
|
|
|
77
81
|
|
|
78
82
|
> Prefer no install? `uvx memoryintelligence-mcp --help` runs it via `uv` with
|
|
79
83
|
> nothing to install. (You'll still run `mi-mcp setup` once to store your key + wire.)
|
|
84
|
+
>
|
|
85
|
+
> **Note on `--help`:** the admin subcommands (`setup`, `wire`, `doctor`, `status`)
|
|
86
|
+
> are dispatched before the argument parser, so they're listed in `--help`'s footer
|
|
87
|
+
> rather than as standard subcommands — `mi-mcp setup` is real even though it isn't a
|
|
88
|
+
> `--help` subparser. For persistent use prefer `uv tool install memoryintelligence-mcp`
|
|
89
|
+
> over bare `uvx`: the launcher Claude Desktop spawns resolves the binary from `PATH` /
|
|
90
|
+
> `~/.local/bin`, not `uvx`'s ephemeral cache.
|
|
80
91
|
|
|
81
92
|
---
|
|
82
93
|
|
|
83
|
-
## ✅ What works today (0.
|
|
94
|
+
## ✅ What works today (0.2.0)
|
|
84
95
|
|
|
85
96
|
Honest status — this is beta, so here's exactly what's live:
|
|
86
97
|
|
|
@@ -42,10 +42,17 @@ API key is ever written into a config file.**
|
|
|
42
42
|
|
|
43
43
|
> Prefer no install? `uvx memoryintelligence-mcp --help` runs it via `uv` with
|
|
44
44
|
> nothing to install. (You'll still run `mi-mcp setup` once to store your key + wire.)
|
|
45
|
+
>
|
|
46
|
+
> **Note on `--help`:** the admin subcommands (`setup`, `wire`, `doctor`, `status`)
|
|
47
|
+
> are dispatched before the argument parser, so they're listed in `--help`'s footer
|
|
48
|
+
> rather than as standard subcommands — `mi-mcp setup` is real even though it isn't a
|
|
49
|
+
> `--help` subparser. For persistent use prefer `uv tool install memoryintelligence-mcp`
|
|
50
|
+
> over bare `uvx`: the launcher Claude Desktop spawns resolves the binary from `PATH` /
|
|
51
|
+
> `~/.local/bin`, not `uvx`'s ephemeral cache.
|
|
45
52
|
|
|
46
53
|
---
|
|
47
54
|
|
|
48
|
-
## ✅ What works today (0.
|
|
55
|
+
## ✅ What works today (0.2.0)
|
|
49
56
|
|
|
50
57
|
Honest status — this is beta, so here's exactly what's live:
|
|
51
58
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "memoryintelligence-mcp"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.0"
|
|
8
8
|
description = "MCP server for MemoryIntelligence — give Claude a memory you own, set up in one command"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -37,16 +37,23 @@ Changelog = "https://github.com/somewhere11/memoryintelligence-mcp/blob/main/CHA
|
|
|
37
37
|
Documentation = "https://memoryintelligence.io/docs/api-reference"
|
|
38
38
|
|
|
39
39
|
[project.optional-dependencies]
|
|
40
|
-
# Local vault (Path A): owner-side .umo encryption/decryption + signature verify
|
|
41
|
-
#
|
|
40
|
+
# Local vault (Path A): owner-side .umo encryption/decryption + signature verify,
|
|
41
|
+
# plus the local vector index (numpy) for network-free reads. Kept out of the base
|
|
42
|
+
# install so the thin server-client stays lightweight.
|
|
42
43
|
local = [
|
|
43
44
|
"cryptography>=42.0,<46",
|
|
45
|
+
"numpy>=1.24,<3",
|
|
46
|
+
# Local-first ONNX embedder (bge-small) for network-free reads. Pulls
|
|
47
|
+
# onnxruntime — real weight, hence kept out of the thin base install.
|
|
48
|
+
"fastembed>=0.3.0,<1.0",
|
|
44
49
|
]
|
|
45
50
|
dev = [
|
|
46
51
|
"pytest>=8.0",
|
|
47
52
|
"pytest-asyncio>=0.24",
|
|
48
53
|
"ruff>=0.4.0",
|
|
49
54
|
"cryptography>=42.0,<46",
|
|
55
|
+
"numpy>=1.24,<3",
|
|
56
|
+
"fastembed>=0.3.0,<1.0",
|
|
50
57
|
]
|
|
51
58
|
|
|
52
59
|
[project.scripts]
|
|
@@ -29,13 +29,27 @@ def main():
|
|
|
29
29
|
# Admin subcommands: `mi-mcp {setup|init|wire|doctor|status|memory}`. Bare
|
|
30
30
|
# `mi-mcp` (no subcommand) runs the server — that's how the MCP host spawns it.
|
|
31
31
|
argv = sys.argv[1:]
|
|
32
|
-
if argv and argv[0] in (
|
|
32
|
+
if argv and argv[0] in (
|
|
33
|
+
"setup", "init", "wire", "doctor", "status", "memory", "backfill", "index",
|
|
34
|
+
):
|
|
33
35
|
from .cli import run_admin
|
|
34
36
|
sys.exit(run_admin(argv[0], argv[1:]))
|
|
35
37
|
|
|
36
38
|
parser = argparse.ArgumentParser(
|
|
37
39
|
prog="mi-mcp",
|
|
38
40
|
description="MemoryIntelligence MCP Server",
|
|
41
|
+
# Admin subcommands are dispatched before argparse (above), so they don't
|
|
42
|
+
# appear as subparsers. List them here so `--help` isn't misleading — a new
|
|
43
|
+
# user checking `--help` for `setup` shouldn't conclude the docs are stale.
|
|
44
|
+
epilog=(
|
|
45
|
+
"setup commands (run these first, not shown above):\n"
|
|
46
|
+
" mi-mcp setup store your API key + wire + opt-in + verify (one command)\n"
|
|
47
|
+
" mi-mcp wire wire into Claude Desktop / Code / Cursor\n"
|
|
48
|
+
" mi-mcp doctor verify install, key resolution, and wiring\n"
|
|
49
|
+
" mi-mcp status show wired surfaces + opt-in allowlist\n"
|
|
50
|
+
"\nGet a key at https://memoryintelligence.io/portal, then run `mi-mcp setup`."
|
|
51
|
+
),
|
|
52
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
39
53
|
)
|
|
40
54
|
parser.add_argument(
|
|
41
55
|
"--version",
|
|
@@ -6,11 +6,14 @@ a wrapper (``~/.memoryintelligence/mcp/run-mi-mcp.sh``) that resolves
|
|
|
6
6
|
``MI_API_KEY`` at launch from the macOS Keychain, then the on-disk keyfile,
|
|
7
7
|
else fails. So a leaked config file exposes nothing.
|
|
8
8
|
|
|
9
|
-
setup
|
|
10
|
-
|
|
11
|
-
wire
|
|
12
|
-
doctor
|
|
13
|
-
status
|
|
9
|
+
setup — one command: store the key, wire hosts, opt in this dir, verify.
|
|
10
|
+
(alias: ``init``). The frictionless front door for new users.
|
|
11
|
+
wire — register the server in Claude config(s); no key in any file.
|
|
12
|
+
doctor — verify wiring + key resolvability (prints prefix only, never the key).
|
|
13
|
+
status — show which surfaces are wired + the capture opt-in allowlist.
|
|
14
|
+
memory — inspect the local .umo vault (ls|open|verify|rm|path).
|
|
15
|
+
backfill — migrate cloud memories into the local .umo vault (re-embed locally).
|
|
16
|
+
index — build/inspect the local vector index (build|path|stat).
|
|
14
17
|
|
|
15
18
|
The key is stored OUTSIDE every config — in the macOS Keychain (``security``)
|
|
16
19
|
or, on Linux/Windows (or by choice), a ``chmod 600 ~/.memoryintelligence/.env``
|
|
@@ -31,9 +34,10 @@ import shutil
|
|
|
31
34
|
import subprocess
|
|
32
35
|
import sys
|
|
33
36
|
import tempfile
|
|
37
|
+
from dataclasses import dataclass
|
|
34
38
|
from pathlib import Path
|
|
35
39
|
|
|
36
|
-
from . import paths
|
|
40
|
+
from . import __version__, paths
|
|
37
41
|
from .config import load_opt_in_paths
|
|
38
42
|
|
|
39
43
|
SERVER_KEY = "memoryintelligence"
|
|
@@ -323,7 +327,12 @@ def cmd_wire(argv: list[str]) -> int:
|
|
|
323
327
|
do_wire(home, surfaces, args.dry_run, capture_anywhere=args.capture_anywhere)
|
|
324
328
|
if not args.dry_run:
|
|
325
329
|
print("\nNext steps:")
|
|
326
|
-
|
|
330
|
+
# "opt in a directory" reads as "put a file in a folder" to a new user — it's
|
|
331
|
+
# neither. It records a project path in an allowlist; nothing is placed in the
|
|
332
|
+
# folder. Show it as a concrete cd-then-command so the mental model is right.
|
|
333
|
+
print(" 1. allow captures from a project (no file is placed — it records the path):")
|
|
334
|
+
print(" cd ~/Projects/my-app")
|
|
335
|
+
print(f" mi-mcp setup --opt-in \"$(pwd)\" # or: echo \"$(pwd)\" >> {paths.mcp_config_dir(home=home) / 'opt-in-paths'}")
|
|
327
336
|
print(" 2. restart Claude (MCP servers load at startup)")
|
|
328
337
|
print(" 3. mi-mcp doctor # verify")
|
|
329
338
|
return 0
|
|
@@ -346,6 +355,15 @@ def cmd_doctor(argv: list[str]) -> int:
|
|
|
346
355
|
bin_path = _mi_mcp_bin()
|
|
347
356
|
check("mi-mcp binary", Path(bin_path).exists(), bin_path)
|
|
348
357
|
|
|
358
|
+
# PATH self-check — the #1 post-`uv tool install` stall: the shim exists but its
|
|
359
|
+
# dir isn't on PATH, so `mi-mcp` "isn't found" until the user notices uv's warning.
|
|
360
|
+
bin_dir = str(Path(bin_path).parent)
|
|
361
|
+
path_dirs = os.environ.get("PATH", "").split(os.pathsep)
|
|
362
|
+
on_path = bin_dir in path_dirs
|
|
363
|
+
check("binary on PATH", on_path,
|
|
364
|
+
"" if on_path else f'{bin_dir} not on PATH — run `export PATH="{bin_dir}:$PATH"` (or `uv tool update-shell`) and open a new terminal',
|
|
365
|
+
critical=False)
|
|
366
|
+
|
|
349
367
|
wrapper = _wrapper_path(home)
|
|
350
368
|
check("wrapper rendered", wrapper.exists(), str(wrapper))
|
|
351
369
|
check("wrapper executable", wrapper.exists() and os.access(wrapper, os.X_OK),
|
|
@@ -361,6 +379,20 @@ def cmd_doctor(argv: list[str]) -> int:
|
|
|
361
379
|
f"{n} entries" if optin.exists() else "absent — all captures will skip",
|
|
362
380
|
critical=False)
|
|
363
381
|
|
|
382
|
+
# Vault path — surface where the local vault resolves and whether it matches the
|
|
383
|
+
# MemorySpace Desktop vault (~/Somewhere). They differ by default (#653): mi-mcp
|
|
384
|
+
# defaults to ~/MemoryIntelligence; the Desktop reads ~/Somewhere. If MI_VAULT
|
|
385
|
+
# isn't unifying them, backfilled memories won't appear in the Desktop.
|
|
386
|
+
from . import vault as _vault
|
|
387
|
+
vpath = _vault.vault_path()
|
|
388
|
+
desktop_vault = home / "Somewhere"
|
|
389
|
+
mi_vault_set = "MI_VAULT" in os.environ
|
|
390
|
+
matches = vpath.resolve() == desktop_vault.resolve()
|
|
391
|
+
check("vault path", matches or mi_vault_set,
|
|
392
|
+
str(vpath) if matches else
|
|
393
|
+
f"{vpath} — Desktop reads {desktop_vault}; set MI_VAULT={desktop_vault} so both share one vault (#653)",
|
|
394
|
+
critical=False)
|
|
395
|
+
|
|
364
396
|
for s, p in _surface_paths(home).items():
|
|
365
397
|
servers = _load_json(p).get(_surface_key(s), {})
|
|
366
398
|
wired = SERVER_KEY in servers
|
|
@@ -464,6 +496,248 @@ def cmd_memory(argv: list[str]) -> int:
|
|
|
464
496
|
return 0
|
|
465
497
|
|
|
466
498
|
|
|
499
|
+
# ---------------------------------------------------------------------------
|
|
500
|
+
# backfill — one-time cloud → local-vault migration (dry-run by default)
|
|
501
|
+
# ---------------------------------------------------------------------------
|
|
502
|
+
|
|
503
|
+
# A <PERSON>/<…> marker in the OWNER export means the content was redacted at
|
|
504
|
+
# INGEST (stored redacted). The owner export keeps person names, so a marker here
|
|
505
|
+
# flags a memory whose raw text was never stored and cannot be recovered locally.
|
|
506
|
+
_REDACTION_MARKERS = (
|
|
507
|
+
"<PERSON>", "<EMAIL>", "<PHONE>", "<SSN>", "<LOCATION>", "<ORG>",
|
|
508
|
+
"<CREDIT_CARD>", "[REDACTED]",
|
|
509
|
+
)
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
@dataclass
|
|
513
|
+
class BackfillStats:
|
|
514
|
+
total: int = 0
|
|
515
|
+
approx_bytes: int = 0
|
|
516
|
+
with_raw: int = 0
|
|
517
|
+
redacted_at_source: int = 0
|
|
518
|
+
stream_error: str | None = None
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def _analyze_export(lines) -> BackfillStats:
|
|
522
|
+
"""Tally an NDJSON export stream without writing anything.
|
|
523
|
+
|
|
524
|
+
Pure + side-effect-free so it's unit-testable with synthetic lines. Counts
|
|
525
|
+
UMOs, bytes, how many carry raw_text, and how many were already redacted at
|
|
526
|
+
ingest (a marker in the owner export = unrecoverable raw text).
|
|
527
|
+
"""
|
|
528
|
+
st = BackfillStats()
|
|
529
|
+
for raw in lines:
|
|
530
|
+
line = raw.strip()
|
|
531
|
+
if not line:
|
|
532
|
+
continue
|
|
533
|
+
st.approx_bytes += len(raw) + 1
|
|
534
|
+
try:
|
|
535
|
+
obj = json.loads(line)
|
|
536
|
+
except (json.JSONDecodeError, TypeError):
|
|
537
|
+
continue
|
|
538
|
+
if isinstance(obj, dict) and obj.get("_error"):
|
|
539
|
+
st.stream_error = str(obj.get("_error"))
|
|
540
|
+
continue
|
|
541
|
+
st.total += 1
|
|
542
|
+
if obj.get("raw_text"):
|
|
543
|
+
st.with_raw += 1
|
|
544
|
+
if any(m in line for m in _REDACTION_MARKERS):
|
|
545
|
+
st.redacted_at_source += 1
|
|
546
|
+
return st
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
# --- pure transforms (export row → .umo inputs); unit-testable without I/O -----
|
|
550
|
+
|
|
551
|
+
def _parse_export_objects(lines) -> list[dict]:
|
|
552
|
+
"""Parse export NDJSON lines into UMO dicts (skip blanks/errors/malformed)."""
|
|
553
|
+
out: list[dict] = []
|
|
554
|
+
for raw in lines:
|
|
555
|
+
line = (raw or "").strip()
|
|
556
|
+
if not line:
|
|
557
|
+
continue
|
|
558
|
+
try:
|
|
559
|
+
obj = json.loads(line)
|
|
560
|
+
except (json.JSONDecodeError, TypeError):
|
|
561
|
+
continue
|
|
562
|
+
if isinstance(obj, dict) and not obj.get("_error"):
|
|
563
|
+
out.append(obj)
|
|
564
|
+
return out
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def _text_for_embedding(row: dict) -> str:
|
|
568
|
+
"""Pick the text to (re-)embed — the normalized text, else raw, else summary.
|
|
569
|
+
|
|
570
|
+
The cloud has no exported embedding (`export.py` omits the 384-d vector), so
|
|
571
|
+
backfill must re-embed locally; this chooses the same text capture embeds.
|
|
572
|
+
"""
|
|
573
|
+
for k in ("normalized_text", "raw_text", "summary"):
|
|
574
|
+
v = row.get(k)
|
|
575
|
+
if v and str(v).strip():
|
|
576
|
+
return str(v)
|
|
577
|
+
return ""
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def _payload_for_umo(row: dict, embedding: list[float]) -> dict:
|
|
581
|
+
"""Build the encrypted ``.umo`` payload for a backfilled cloud UMO."""
|
|
582
|
+
return {
|
|
583
|
+
"embedding": embedding,
|
|
584
|
+
"summary": row.get("summary") or "",
|
|
585
|
+
"entities": row.get("entities") or [],
|
|
586
|
+
"topics": row.get("tags") or [],
|
|
587
|
+
"created_at": row.get("created_at"),
|
|
588
|
+
"normalized_text": row.get("normalized_text") or "",
|
|
589
|
+
"raw_text": row.get("raw_text"),
|
|
590
|
+
"semantic_hash": row.get("semantic_hash"),
|
|
591
|
+
"quality_score": row.get("quality_score"),
|
|
592
|
+
"source_type": row.get("source_type"),
|
|
593
|
+
# Provenance: owner-self-signed, NOT MI-attested (we re-produced it locally).
|
|
594
|
+
"origin": "backfill",
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
def _public_metadata_for_umo(row: dict, owner_did: str) -> dict:
|
|
599
|
+
"""Plaintext public metadata for a backfilled ``.umo`` (no PII; vault-readable).
|
|
600
|
+
|
|
601
|
+
Emits the same field set the desktop app writes (content_type / format_version /
|
|
602
|
+
mi_key_id / schema_version / source_count) so the two surfaces share one
|
|
603
|
+
readable home, plus an ``origin`` provenance label (the desktop ignores extras).
|
|
604
|
+
"""
|
|
605
|
+
from . import umo_format
|
|
606
|
+
return {
|
|
607
|
+
"umo_id": row.get("id"),
|
|
608
|
+
"created_at": row.get("created_at"),
|
|
609
|
+
"owner_did": owner_did,
|
|
610
|
+
"content_type": "text/plain",
|
|
611
|
+
"format_version": umo_format.FORMAT_VERSION_STR,
|
|
612
|
+
"mi_key_id": "local",
|
|
613
|
+
"schema_version": row.get("schema_version") or "1.0",
|
|
614
|
+
"source_count": 1,
|
|
615
|
+
# MCP provenance: owner-self-signed backfill, NOT MI-attested (extra field).
|
|
616
|
+
"origin": "backfill",
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def cmd_backfill(argv: list[str]) -> int:
|
|
621
|
+
ap = argparse.ArgumentParser(
|
|
622
|
+
prog="mi-mcp backfill",
|
|
623
|
+
description="One-time migration of your cloud memories into the local .umo vault. "
|
|
624
|
+
"Dry-run by default (writes nothing).",
|
|
625
|
+
)
|
|
626
|
+
ap.add_argument("--execute", action="store_true",
|
|
627
|
+
help="actually write to the local vault (default: dry-run, writes nothing)")
|
|
628
|
+
ap.add_argument("--since", default=None, metavar="ISO8601",
|
|
629
|
+
help="only backfill memories created on/after this timestamp")
|
|
630
|
+
ap.add_argument("--home", default=os.environ.get("HOME"))
|
|
631
|
+
args = ap.parse_args(argv)
|
|
632
|
+
home = Path(args.home)
|
|
633
|
+
|
|
634
|
+
key, src = _resolve_key(home)
|
|
635
|
+
if not key:
|
|
636
|
+
print("error: no MI_API_KEY resolvable — run `mi-mcp setup` first.", file=sys.stderr)
|
|
637
|
+
return 2
|
|
638
|
+
base_url = os.environ.get("MI_BASE_URL", "https://api.memoryintelligence.io").rstrip("/")
|
|
639
|
+
|
|
640
|
+
from . import vault
|
|
641
|
+
vault_dir = vault.vault_path()
|
|
642
|
+
|
|
643
|
+
mode = "EXECUTE" if args.execute else "DRY-RUN"
|
|
644
|
+
print(f"{mode}: backfill cloud → local vault")
|
|
645
|
+
print(f" source: {base_url}/v1/memories/export?include_raw=true (raw owner export · key {src})")
|
|
646
|
+
print(f" vault : {vault_dir}")
|
|
647
|
+
print(" streaming export …")
|
|
648
|
+
|
|
649
|
+
import httpx
|
|
650
|
+
params = {"include_raw": "true"}
|
|
651
|
+
if args.since:
|
|
652
|
+
params["since"] = args.since
|
|
653
|
+
headers = {"Authorization": f"Bearer {key}", "User-Agent": f"mi-mcp/{__version__}"}
|
|
654
|
+
try:
|
|
655
|
+
with httpx.stream(
|
|
656
|
+
"GET", f"{base_url}/v1/memories/export",
|
|
657
|
+
params=params, headers=headers,
|
|
658
|
+
timeout=httpx.Timeout(120.0, connect=10.0),
|
|
659
|
+
) as resp:
|
|
660
|
+
if resp.status_code >= 400:
|
|
661
|
+
resp.read()
|
|
662
|
+
print(f"error: export HTTP {resp.status_code}: {resp.text[:200]}", file=sys.stderr)
|
|
663
|
+
return 1
|
|
664
|
+
# Collect the stream once; we need it for both the stats and (on
|
|
665
|
+
# --execute) the write pass. The migration is a one-shot, so holding
|
|
666
|
+
# the (embedding-free) export in memory is acceptable.
|
|
667
|
+
lines = list(resp.iter_lines())
|
|
668
|
+
except httpx.HTTPError as exc:
|
|
669
|
+
print(f"error: export request failed: {exc}", file=sys.stderr)
|
|
670
|
+
return 1
|
|
671
|
+
|
|
672
|
+
stats = _analyze_export(lines)
|
|
673
|
+
print()
|
|
674
|
+
print(f" memories : {stats.total}")
|
|
675
|
+
print(f" approx size : {stats.approx_bytes / 1024:.1f} KB")
|
|
676
|
+
print(f" carry raw_text : {stats.with_raw}")
|
|
677
|
+
print(f" redacted at ingest : {stats.redacted_at_source} (raw never stored — unrecoverable)")
|
|
678
|
+
if stats.stream_error:
|
|
679
|
+
print(f" ⚠ stream interrupted: {stats.stream_error} — counts are partial")
|
|
680
|
+
|
|
681
|
+
if not args.execute:
|
|
682
|
+
print("\n DRY-RUN — nothing written.")
|
|
683
|
+
print(" --execute would: re-embed each locally (fastembed bge-small) → encrypt to")
|
|
684
|
+
print(f" .umo with your owner key → write to {vault_dir} → build the local index.")
|
|
685
|
+
return 0
|
|
686
|
+
|
|
687
|
+
# ---- EXECUTE: re-embed locally → encrypt to .umo → vault → build index ----
|
|
688
|
+
objects = _parse_export_objects(lines)
|
|
689
|
+
embeddable = [(o, t) for o in objects if (t := _text_for_embedding(o)) and o.get("id")]
|
|
690
|
+
skipped_no_text = sum(1 for o in objects if o.get("id") and not _text_for_embedding(o))
|
|
691
|
+
if not embeddable:
|
|
692
|
+
print("\n nothing to write (no embeddable memories in the export).")
|
|
693
|
+
return 0
|
|
694
|
+
|
|
695
|
+
try:
|
|
696
|
+
from . import embedder, indexer, keys, umo_format
|
|
697
|
+
# create=False: NEVER mint a new owner key as a side effect of a migration —
|
|
698
|
+
# that would encrypt the backfill to a key the owner doesn't hold, and reads
|
|
699
|
+
# would silently skip every file (review H3). Fail loudly if it's not resolvable.
|
|
700
|
+
owner_priv = keys.load_master_private_key(create=False)
|
|
701
|
+
owner_pub = owner_priv.public_key()
|
|
702
|
+
signing = keys.load_local_signing_key(create=False)
|
|
703
|
+
# Match the desktop's fixed owner_did so MCP- and desktop-written .umo share
|
|
704
|
+
# one consistent home (owner_did is a label, not used in decryption).
|
|
705
|
+
owner_did = umo_format.LOCAL_OWNER_DID
|
|
706
|
+
except Exception as e: # missing crypto extra / no Keychain key / etc.
|
|
707
|
+
print(f"\nerror: cannot resolve owner keys: {e}", file=sys.stderr)
|
|
708
|
+
print(" (run `mi-mcp setup`/the desktop app first so the owner key exists)", file=sys.stderr)
|
|
709
|
+
return 2
|
|
710
|
+
|
|
711
|
+
# Print the key fingerprint so you can confirm the backfill is encrypted to YOUR key.
|
|
712
|
+
print(f"\n owner key: {keys.owner_did(owner_priv)} (confirm this is yours)")
|
|
713
|
+
print(f" embedding {len(embeddable)} memories locally (bge-small)…")
|
|
714
|
+
try:
|
|
715
|
+
vectors = embedder.embed([t for _, t in embeddable])
|
|
716
|
+
except embedder.LocalEmbedderError as e:
|
|
717
|
+
print(f"\nerror: {e}", file=sys.stderr)
|
|
718
|
+
return 2
|
|
719
|
+
|
|
720
|
+
written = 0
|
|
721
|
+
for (row, _text), vec in zip(embeddable, vectors):
|
|
722
|
+
pm = _public_metadata_for_umo(row, owner_did)
|
|
723
|
+
blob = umo_format.produce_umo_for_owner(
|
|
724
|
+
_payload_for_umo(row, vec), pm, owner_pub, signing
|
|
725
|
+
)
|
|
726
|
+
vault.write_umo(pm["umo_id"], owner_did, blob)
|
|
727
|
+
written += 1
|
|
728
|
+
|
|
729
|
+
print(f" wrote {written} .umo files → {vault_dir}")
|
|
730
|
+
if skipped_no_text:
|
|
731
|
+
print(f" skipped {skipped_no_text} memories with no embeddable text")
|
|
732
|
+
|
|
733
|
+
print(" building local index …")
|
|
734
|
+
count = indexer.rebuild_and_save(owner_priv=owner_priv)
|
|
735
|
+
print(f" ✅ index built: {count} memories rankable locally → {paths.local_index_path()}")
|
|
736
|
+
print("\n Local reads are wired: set MI_MCP_LOCAL=1 to serve mi_ask/mi_list from this")
|
|
737
|
+
print(" vault. `mi-mcp index build` rebuilds the index after vault changes.")
|
|
738
|
+
return 0
|
|
739
|
+
|
|
740
|
+
|
|
467
741
|
# ---------------------------------------------------------------------------
|
|
468
742
|
# setup — the one-command front door (store key → wire → opt-in → verify)
|
|
469
743
|
# ---------------------------------------------------------------------------
|
|
@@ -627,6 +901,44 @@ def cmd_setup(argv: list[str]) -> int:
|
|
|
627
901
|
return rc
|
|
628
902
|
|
|
629
903
|
|
|
904
|
+
def cmd_index(argv: list[str]) -> int:
|
|
905
|
+
"""Build/inspect the local vector index: ``mi-mcp index {build|path|stat}``.
|
|
906
|
+
|
|
907
|
+
``build`` decrypts the vault with your master key (Keychain prompt) and writes
|
|
908
|
+
the rank sidecar; the server loads that prebuilt sidecar on the read path.
|
|
909
|
+
"""
|
|
910
|
+
ap = argparse.ArgumentParser(prog="mi-mcp index")
|
|
911
|
+
sub = ap.add_subparsers(dest="action", required=True)
|
|
912
|
+
sub.add_parser("build", help="(re)build the index from the vault (needs your master key)")
|
|
913
|
+
sub.add_parser("path", help="print the index sidecar path")
|
|
914
|
+
sub.add_parser("stat", help="show how many memories are indexed")
|
|
915
|
+
args = ap.parse_args(argv)
|
|
916
|
+
|
|
917
|
+
from . import indexer
|
|
918
|
+
|
|
919
|
+
if args.action == "path":
|
|
920
|
+
print(paths.local_index_path())
|
|
921
|
+
return 0
|
|
922
|
+
|
|
923
|
+
if args.action == "stat":
|
|
924
|
+
idx = indexer.load_index()
|
|
925
|
+
if idx is None:
|
|
926
|
+
print(f"(no index yet — run `mi-mcp index build`) [{paths.local_index_path()}]")
|
|
927
|
+
return 0
|
|
928
|
+
print(f"{len(idx)} memories indexed [{paths.local_index_path()}]")
|
|
929
|
+
return 0
|
|
930
|
+
|
|
931
|
+
# build
|
|
932
|
+
try:
|
|
933
|
+
count = indexer.rebuild_and_save()
|
|
934
|
+
except Exception as e:
|
|
935
|
+
detail = str(e) or type(e).__name__
|
|
936
|
+
print(f"cannot build index: {detail}", file=sys.stderr)
|
|
937
|
+
return 1
|
|
938
|
+
print(f"✅ indexed {count} memories → {paths.local_index_path()}")
|
|
939
|
+
return 0
|
|
940
|
+
|
|
941
|
+
|
|
630
942
|
_COMMANDS = {
|
|
631
943
|
"setup": cmd_setup,
|
|
632
944
|
"init": cmd_setup, # alias
|
|
@@ -634,6 +946,8 @@ _COMMANDS = {
|
|
|
634
946
|
"doctor": cmd_doctor,
|
|
635
947
|
"status": cmd_status,
|
|
636
948
|
"memory": cmd_memory,
|
|
949
|
+
"backfill": cmd_backfill,
|
|
950
|
+
"index": cmd_index,
|
|
637
951
|
}
|
|
638
952
|
|
|
639
953
|
|