memty 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- memoryport/__init__.py +20 -0
- memoryport/api.py +198 -0
- memoryport/cli.py +94 -0
- memoryport/embeddings.py +130 -0
- memoryport/evaluation.py +57 -0
- memoryport/extractor.py +94 -0
- memoryport/factory.py +61 -0
- memoryport/ledger.py +629 -0
- memoryport/mcp_server.py +56 -0
- memoryport/memory.py +208 -0
- memoryport/mongo_ledger.py +327 -0
- memoryport/router.py +197 -0
- memoryport/sdk.py +731 -0
- memoryport/sdk_cli.py +109 -0
- memoryport/sdk_mcp.py +81 -0
- memoryport/vault.py +145 -0
- memty/README.md +207 -0
- memty/__init__.py +36 -0
- memty/__main__.py +5 -0
- memty/api.py +8 -0
- memty/cli.py +196 -0
- memty/compiler.py +334 -0
- memty/connect.py +60 -0
- memty/daemon.py +257 -0
- memty/factory.py +27 -0
- memty/ledger.py +864 -0
- memty/legacy_mcp.py +8 -0
- memty/legacy_vault.py +8 -0
- memty/mcp_server.py +65 -0
- memty/migrate.py +114 -0
- memty/router_mcp.py +8 -0
- memty/sdk.py +38 -0
- memty/tokens.py +39 -0
- memty/vectors.py +31 -0
- memty-0.1.0.dist-info/METADATA +235 -0
- memty-0.1.0.dist-info/RECORD +40 -0
- memty-0.1.0.dist-info/WHEEL +5 -0
- memty-0.1.0.dist-info/entry_points.txt +8 -0
- memty-0.1.0.dist-info/licenses/LICENSE +21 -0
- memty-0.1.0.dist-info/top_level.txt +2 -0
memoryport/__init__.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Internal implementation modules for the Memty SDK."""
|
|
2
|
+
|
|
3
|
+
from .memory import MongoSyncedMemoryStore, PortableMemoryStore
|
|
4
|
+
from .router import ModelSpec, RouteDecision, SmartRouter
|
|
5
|
+
from .ledger import MemoryLedger
|
|
6
|
+
from .evaluation import evaluate_cases
|
|
7
|
+
from .vault import export_nxvault, import_nxvault
|
|
8
|
+
from .mongo_ledger import MongoMemoryLedger
|
|
9
|
+
from .extractor import MemoryExtractor
|
|
10
|
+
from .sdk import ChatResponse, Memty, RouteResult, ModelConfig, load_client
|
|
11
|
+
NexorAI = Memty # Keep the former Python name as a compatibility alias.
|
|
12
|
+
from .embeddings import (EmbeddingAdapter, EmbeddingConfig, EmbeddingError,
|
|
13
|
+
GeminiEmbeddingAdapter, OpenAICompatibleEmbeddingAdapter)
|
|
14
|
+
|
|
15
|
+
__all__ = ["ChatResponse", "EmbeddingAdapter", "EmbeddingConfig", "EmbeddingError", "GeminiEmbeddingAdapter",
|
|
16
|
+
"MemoryExtractor", "OpenAICompatibleEmbeddingAdapter",
|
|
17
|
+
"MemoryLedger", "ModelConfig", "ModelSpec", "MongoMemoryLedger",
|
|
18
|
+
"MongoSyncedMemoryStore", "Memty", "NexorAI", "PortableMemoryStore", "RouteDecision", "RouteResult",
|
|
19
|
+
"SmartRouter", "evaluate_cases", "export_nxvault", "import_nxvault", "load_client"]
|
|
20
|
+
|
memoryport/api.py
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""Loopback-only HTTP API for the local memory vault.
|
|
2
|
+
|
|
3
|
+
This API is intended for a local UI or trusted desktop bridge. External agents
|
|
4
|
+
should use the approval-only MCP interface instead of receiving write access.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hmac
|
|
10
|
+
import os
|
|
11
|
+
import contextvars
|
|
12
|
+
import hashlib
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
import re
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from .factory import open_ledger
|
|
18
|
+
from .ledger import MemoryLedger
|
|
19
|
+
from .extractor import MemoryExtractor
|
|
20
|
+
|
|
21
|
+
try:
|
|
22
|
+
from fastapi import Depends, FastAPI, Header, HTTPException
|
|
23
|
+
from fastapi.responses import Response
|
|
24
|
+
except ImportError as error:
|
|
25
|
+
raise RuntimeError("HTTP API support requires: pip install 'memty[server]'") from error
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
_request_user: contextvars.ContextVar[str] = contextvars.ContextVar("memty_vault_user", default="local")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _ledger() -> MemoryLedger:
|
|
32
|
+
return open_ledger(_request_user.get())
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
async def _authorized(authorization: str | None = Header(default=None),
|
|
36
|
+
x_memty_user: str | None = Header(default=None),
|
|
37
|
+
x_nexor_user: str | None = Header(default=None)) -> None:
|
|
38
|
+
secret = os.getenv("MEMTY_VAULT_API_TOKEN") or os.getenv("NEXOR_VAULT_API_TOKEN", "")
|
|
39
|
+
if len(secret) < 32:
|
|
40
|
+
raise HTTPException(503, "Configure a random MEMTY_VAULT_API_TOKEN of at least 32 characters.")
|
|
41
|
+
supplied = authorization.removeprefix("Bearer ") if authorization else ""
|
|
42
|
+
if not hmac.compare_digest(supplied.encode(), secret.encode()):
|
|
43
|
+
raise HTTPException(401, "Invalid local vault token.")
|
|
44
|
+
multi_user = os.getenv("MEMTY_VAULT_MULTIUSER", os.getenv("NEXOR_VAULT_MULTIUSER", "false")).casefold() == "true"
|
|
45
|
+
user_id = x_memty_user or x_nexor_user
|
|
46
|
+
if multi_user and not user_id:
|
|
47
|
+
raise HTTPException(400, "X-Memty-User is required in multi-user mode.")
|
|
48
|
+
if user_id:
|
|
49
|
+
if not re.fullmatch(r"[A-Za-z0-9_-]{1,200}", user_id):
|
|
50
|
+
raise HTTPException(400, "Invalid vault user identifier.")
|
|
51
|
+
if not multi_user:
|
|
52
|
+
raise HTTPException(403, "Multi-user mode is disabled.")
|
|
53
|
+
_request_user.set(user_id)
|
|
54
|
+
else:
|
|
55
|
+
_request_user.set(os.getenv("MEMTY_VAULT_USER", os.getenv("NEXOR_VAULT_USER", "local")))
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
app = FastAPI(title="Memty Memory Vault", version="1.0.0", dependencies=[Depends(_authorized)])
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@app.get("/v1/health")
|
|
62
|
+
def health() -> dict[str, str]:
|
|
63
|
+
return {"status": "ok", "format": MemoryLedger.FORMAT}
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@app.get("/v1/memories")
|
|
67
|
+
def list_memories(history: bool = False, status: str | None = None) -> list[dict[str, Any]]:
|
|
68
|
+
records = _ledger().list(include_history=history)
|
|
69
|
+
newer = {record.get("supersedes_id"): record["id"] for record in records if record.get("supersedes_id")}
|
|
70
|
+
for record in records:
|
|
71
|
+
if record["status"] == "SUPERSEDED":
|
|
72
|
+
record["superseded_by_id"] = newer.get(record["id"])
|
|
73
|
+
if status:
|
|
74
|
+
records = [record for record in records if record["status"] == status.upper()]
|
|
75
|
+
return records
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@app.post("/v1/memories")
|
|
79
|
+
def create_memory(payload: dict[str, Any]) -> dict[str, Any]:
|
|
80
|
+
try:
|
|
81
|
+
return _ledger().record(
|
|
82
|
+
payload.get("namespace", "personal.general"), payload["slot_key"], payload["value"],
|
|
83
|
+
confidence=1.0, verification_state="USER_CONFIRMED",
|
|
84
|
+
sensitivity_tier=payload.get("sensitivity_tier", "PERSONAL"),
|
|
85
|
+
allowed_providers=payload.get("allowed_providers"),
|
|
86
|
+
blocked_providers=payload.get("blocked_providers"),
|
|
87
|
+
ephemeral_redaction=bool(payload.get("ephemeral_redaction", False)),
|
|
88
|
+
source_conversation_id=payload.get("source_conversation_id"),
|
|
89
|
+
source_message=payload.get("source_message"),
|
|
90
|
+
)
|
|
91
|
+
except (KeyError, TypeError, ValueError) as error:
|
|
92
|
+
raise HTTPException(400, str(error)) from error
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@app.post("/v1/extract")
|
|
96
|
+
def extract_memories(payload: dict[str, Any]) -> dict[str, Any]:
|
|
97
|
+
if not isinstance(payload.get("user_message"), str) or not isinstance(payload.get("memories"), list):
|
|
98
|
+
raise HTTPException(400, "user_message and a memories list are required.")
|
|
99
|
+
extractor = MemoryExtractor(_ledger(), lambda _prompt: {"memories": payload["memories"]})
|
|
100
|
+
try:
|
|
101
|
+
return extractor.ingest_turn(payload["user_message"], str(payload.get("assistant_response", "")),
|
|
102
|
+
conversation_id=payload.get("conversation_id"), model_id=payload.get("model_id"))
|
|
103
|
+
except (TypeError, ValueError) as error:
|
|
104
|
+
raise HTTPException(400, str(error)) from error
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@app.delete("/v1/memories/{memory_id}")
|
|
108
|
+
def delete_memory(memory_id: str) -> dict[str, Any]:
|
|
109
|
+
result = _ledger().delete(memory_id)
|
|
110
|
+
if result is None:
|
|
111
|
+
raise HTTPException(404, "Memory not found.")
|
|
112
|
+
return result
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@app.patch("/v1/memories/{memory_id}")
|
|
116
|
+
def edit_memory(memory_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|
117
|
+
result = _ledger().edit(memory_id, value=payload.get("value"), pinned=payload.get("pinned"))
|
|
118
|
+
if result is None:
|
|
119
|
+
raise HTTPException(404, "Active memory not found.")
|
|
120
|
+
return result
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
@app.post("/v1/context")
|
|
124
|
+
def compile_context(payload: dict[str, Any]) -> dict[str, Any]:
|
|
125
|
+
try:
|
|
126
|
+
return _ledger().context(payload.get("query", ""), provider=payload["provider"],
|
|
127
|
+
token_budget=int(payload.get("token_budget", 900)),
|
|
128
|
+
as_of=payload.get("as_of"), namespace=payload.get("namespace"))
|
|
129
|
+
except (KeyError, TypeError, ValueError) as error:
|
|
130
|
+
raise HTTPException(400, str(error)) from error
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@app.get("/v1/memories/{namespace}/{slot_key}/as-of")
|
|
134
|
+
def memory_as_of(namespace: str, slot_key: str, at: str, known_at: str | None = None) -> dict[str, Any] | None:
|
|
135
|
+
try:
|
|
136
|
+
return _ledger().as_of(namespace, slot_key, at, known_at=known_at)
|
|
137
|
+
except ValueError as error:
|
|
138
|
+
raise HTTPException(400, str(error)) from error
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@app.get("/v1/proposals")
|
|
142
|
+
def list_proposals() -> list[dict[str, Any]]:
|
|
143
|
+
return _ledger().proposals()
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
@app.post("/v1/proposals")
|
|
147
|
+
def propose_memory(payload: dict[str, Any]) -> dict[str, str]:
|
|
148
|
+
try:
|
|
149
|
+
policy = payload.get("policy") or {}
|
|
150
|
+
proposal_id = _ledger().propose(payload["namespace"], payload["slot_key"], payload["value"], **policy)
|
|
151
|
+
return {"proposal_id": proposal_id, "status": "PENDING_CONFIRMATION"}
|
|
152
|
+
except (KeyError, TypeError, ValueError) as error:
|
|
153
|
+
raise HTTPException(400, str(error)) from error
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
@app.post("/v1/proposals/{proposal_id}/approve")
|
|
157
|
+
def approve_proposal(proposal_id: str) -> dict[str, Any]:
|
|
158
|
+
result = _ledger().resolve_proposal(proposal_id, approve=True)
|
|
159
|
+
if result is None:
|
|
160
|
+
raise HTTPException(404, "Proposal not found or already resolved.")
|
|
161
|
+
return result
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@app.post("/v1/proposals/{proposal_id}/reject")
|
|
165
|
+
def reject_proposal(proposal_id: str) -> dict[str, Any]:
|
|
166
|
+
result = _ledger().resolve_proposal(proposal_id, approve=False)
|
|
167
|
+
if result is None:
|
|
168
|
+
raise HTTPException(404, "Proposal not found or already resolved.")
|
|
169
|
+
return result
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
@app.post("/v1/vault/export")
|
|
173
|
+
def export_vault(payload: dict[str, Any]) -> Response:
|
|
174
|
+
from .vault import encode_nxvault
|
|
175
|
+
try:
|
|
176
|
+
bundle = encode_nxvault(_ledger(), passphrase=payload["passphrase"])
|
|
177
|
+
except (KeyError, TypeError, ValueError, RuntimeError) as error:
|
|
178
|
+
raise HTTPException(400, str(error)) from error
|
|
179
|
+
return Response(bundle, media_type="application/vnd.nexor.nxvault",
|
|
180
|
+
headers={"Content-Disposition": "attachment; filename=memory.nxvault"})
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def main() -> None:
|
|
184
|
+
try:
|
|
185
|
+
import uvicorn
|
|
186
|
+
except ImportError as error:
|
|
187
|
+
raise RuntimeError("HTTP API support requires: pip install 'memty[server]'") from error
|
|
188
|
+
host = os.getenv("MEMTY_VAULT_API_HOST", os.getenv("NEXOR_VAULT_API_HOST", "127.0.0.1"))
|
|
189
|
+
allow_remote = os.getenv("MEMTY_VAULT_ALLOW_REMOTE", os.getenv("NEXOR_VAULT_ALLOW_REMOTE", "false"))
|
|
190
|
+
if host not in {"127.0.0.1", "localhost", "::1"} and allow_remote.casefold() != "true":
|
|
191
|
+
raise RuntimeError("Non-loopback binding requires MEMTY_VAULT_ALLOW_REMOTE=true and a strong API token.")
|
|
192
|
+
uvicorn.run("memty.api:app", host=host, port=int(os.getenv("MEMTY_VAULT_API_PORT", os.getenv("NEXOR_VAULT_API_PORT", "8765"))),
|
|
193
|
+
log_level="warning")
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
if __name__ == "__main__":
|
|
197
|
+
main()
|
|
198
|
+
|
memoryport/cli.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""Manage a local Memty temporal vault without a web account or model API."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from getpass import getpass
|
|
10
|
+
|
|
11
|
+
from .factory import open_ledger
|
|
12
|
+
from .factory import default_vault_path
|
|
13
|
+
from .vault import export_nxvault, import_nxvault
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def main() -> None:
|
|
17
|
+
parser = argparse.ArgumentParser(prog="memty vault")
|
|
18
|
+
parser.add_argument("--db", default=str(default_vault_path()))
|
|
19
|
+
parser.add_argument("--user", default=os.getenv("MEMTY_MEMORY_USER", os.getenv("NEXORAI_MEMORY_USER", os.getenv("MEMTY_VAULT_USER", os.getenv("NEXOR_VAULT_USER", "local")))))
|
|
20
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
21
|
+
commands.add_parser("list", help="list active and historical memories")
|
|
22
|
+
add = commands.add_parser("add", help="save a user-confirmed fact")
|
|
23
|
+
add.add_argument("namespace")
|
|
24
|
+
add.add_argument("slot_key")
|
|
25
|
+
add.add_argument("value")
|
|
26
|
+
add.add_argument("--sensitivity", default="PERSONAL")
|
|
27
|
+
add.add_argument("--allow-provider", action="append", default=[])
|
|
28
|
+
add.add_argument("--block-provider", action="append", default=[])
|
|
29
|
+
add.add_argument("--redact", action="store_true")
|
|
30
|
+
recall = commands.add_parser("context", help="compile provider-filtered context")
|
|
31
|
+
recall.add_argument("provider")
|
|
32
|
+
recall.add_argument("query")
|
|
33
|
+
recall.add_argument("--tokens", type=int, default=900)
|
|
34
|
+
remove = commands.add_parser("delete", help="forget a memory by id")
|
|
35
|
+
remove.add_argument("memory_id")
|
|
36
|
+
approve = commands.add_parser("approve", help="approve a pending proposal")
|
|
37
|
+
approve.add_argument("proposal_id")
|
|
38
|
+
reject = commands.add_parser("reject", help="reject a pending proposal")
|
|
39
|
+
reject.add_argument("proposal_id")
|
|
40
|
+
commands.add_parser("proposals", help="list pending proposals")
|
|
41
|
+
export = commands.add_parser("export", help="create a portable .nxvault file")
|
|
42
|
+
export.add_argument("path")
|
|
43
|
+
export.add_argument("--plaintext", action="store_true", help="allow an unencrypted export")
|
|
44
|
+
imp = commands.add_parser("import", help="import a .nxvault file")
|
|
45
|
+
imp.add_argument("path")
|
|
46
|
+
imp.add_argument("--replace", action="store_true", help="replace this vault's ledger")
|
|
47
|
+
evaluate = commands.add_parser("evaluate", help="run recall regression cases from a JSON file")
|
|
48
|
+
evaluate.add_argument("cases")
|
|
49
|
+
args = parser.parse_args()
|
|
50
|
+
ledger = open_ledger(args.user, path=args.db)
|
|
51
|
+
|
|
52
|
+
if args.command == "list":
|
|
53
|
+
print(json.dumps(ledger.list(include_history=True), ensure_ascii=False, indent=2))
|
|
54
|
+
elif args.command == "add":
|
|
55
|
+
result = ledger.record(args.namespace, args.slot_key, args.value,
|
|
56
|
+
verification_state="USER_CONFIRMED", sensitivity_tier=args.sensitivity,
|
|
57
|
+
allowed_providers=args.allow_provider, blocked_providers=args.block_provider,
|
|
58
|
+
ephemeral_redaction=args.redact)
|
|
59
|
+
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
60
|
+
elif args.command == "context":
|
|
61
|
+
print(json.dumps(ledger.context(args.query, provider=args.provider, token_budget=args.tokens),
|
|
62
|
+
ensure_ascii=False, indent=2))
|
|
63
|
+
elif args.command == "delete":
|
|
64
|
+
result = ledger.delete(args.memory_id)
|
|
65
|
+
if result is None:
|
|
66
|
+
raise SystemExit("Memory not found or already deleted.")
|
|
67
|
+
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
68
|
+
elif args.command == "proposals":
|
|
69
|
+
print(json.dumps(ledger.proposals(), ensure_ascii=False, indent=2))
|
|
70
|
+
elif args.command in {"approve", "reject"}:
|
|
71
|
+
result = ledger.resolve_proposal(args.proposal_id, approve=args.command == "approve")
|
|
72
|
+
if result is None:
|
|
73
|
+
raise SystemExit("Proposal not found or already resolved.")
|
|
74
|
+
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
75
|
+
elif args.command == "export":
|
|
76
|
+
passphrase = None if args.plaintext else getpass("Vault passphrase (min 10 characters): ")
|
|
77
|
+
path = export_nxvault(ledger, args.path, passphrase=passphrase, allow_plaintext=args.plaintext)
|
|
78
|
+
print(path)
|
|
79
|
+
elif args.command == "import":
|
|
80
|
+
passphrase = getpass("Vault passphrase (leave empty for plaintext): ") or None
|
|
81
|
+
count = import_nxvault(ledger, args.path, passphrase=passphrase, merge=not args.replace)
|
|
82
|
+
print(f"Imported {count} ledger events.")
|
|
83
|
+
elif args.command == "evaluate":
|
|
84
|
+
from .evaluation import evaluate_cases
|
|
85
|
+
with open(args.cases, "r", encoding="utf-8") as handle:
|
|
86
|
+
cases = json.load(handle)
|
|
87
|
+
if not isinstance(cases, list):
|
|
88
|
+
raise SystemExit("Evaluation file must contain a JSON list of cases.")
|
|
89
|
+
print(json.dumps(evaluate_cases(ledger, cases), ensure_ascii=False, indent=2))
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
if __name__ == "__main__":
|
|
93
|
+
main()
|
|
94
|
+
|
memoryport/embeddings.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Small standard-library embedding clients for portable semantic memory."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from typing import Any, Protocol
|
|
12
|
+
from urllib.error import HTTPError, URLError
|
|
13
|
+
from urllib.parse import urlparse
|
|
14
|
+
from urllib.request import Request, urlopen
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class EmbeddingError(RuntimeError):
|
|
18
|
+
"""An embedding provider could not return a usable vector."""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class EmbeddingConfig:
|
|
23
|
+
provider: str = "openai-compatible"
|
|
24
|
+
model: str = "nomic-embed-text"
|
|
25
|
+
base_url: str = ""
|
|
26
|
+
api_key_env: str = ""
|
|
27
|
+
timeout_seconds: float = 20.0
|
|
28
|
+
|
|
29
|
+
def __post_init__(self) -> None:
|
|
30
|
+
if self.provider not in {"openai-compatible", "openrouter", "ollama", "lm-studio", "gemini"}:
|
|
31
|
+
raise ValueError("unsupported embedding provider")
|
|
32
|
+
if not re.fullmatch(r"[A-Za-z0-9._-]{1,128}", self.model) or not math.isfinite(self.timeout_seconds) or self.timeout_seconds <= 0:
|
|
33
|
+
raise ValueError("embedding model and timeout must be valid")
|
|
34
|
+
if self.api_key_env and not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", self.api_key_env):
|
|
35
|
+
raise ValueError("embedding api_key_env must be a valid environment variable name")
|
|
36
|
+
|
|
37
|
+
@classmethod
|
|
38
|
+
def from_dict(cls, value: dict[str, Any]) -> "EmbeddingConfig":
|
|
39
|
+
accepted = {item.name for item in cls.__dataclass_fields__.values()}
|
|
40
|
+
return cls(**{key: item for key, item in value.items() if key in accepted})
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def cache_key(self) -> str:
|
|
44
|
+
identity = f"{self.provider}\0{self.model}\0{self.base_url.rstrip('/')}"
|
|
45
|
+
return f"{self.provider}:{self.model}:{hashlib.sha256(identity.encode()).hexdigest()[:16]}"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class EmbeddingAdapter(Protocol):
|
|
49
|
+
def embed(self, text: str, config: EmbeddingConfig) -> list[float]: ...
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _request(url: str, payload: dict[str, Any], headers: dict[str, str], timeout: float) -> dict[str, Any]:
|
|
53
|
+
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
|
54
|
+
request = Request(url, data=body, headers={"Content-Type": "application/json", **headers}, method="POST")
|
|
55
|
+
try:
|
|
56
|
+
with urlopen(request, timeout=timeout) as response:
|
|
57
|
+
result = json.loads(response.read().decode("utf-8"))
|
|
58
|
+
except (HTTPError, URLError, TimeoutError, OSError, UnicodeDecodeError, json.JSONDecodeError):
|
|
59
|
+
raise EmbeddingError("Could not get an embedding from the configured provider.") from None
|
|
60
|
+
if not isinstance(result, dict):
|
|
61
|
+
raise EmbeddingError("Embedding provider returned an invalid response.")
|
|
62
|
+
return result
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _base(config: EmbeddingConfig) -> str:
|
|
66
|
+
defaults = {
|
|
67
|
+
"openai-compatible": "https://api.openai.com/v1",
|
|
68
|
+
"openrouter": "https://openrouter.ai/api/v1",
|
|
69
|
+
"ollama": "http://localhost:11434/v1",
|
|
70
|
+
"lm-studio": "http://localhost:1234/v1",
|
|
71
|
+
"gemini": "https://generativelanguage.googleapis.com",
|
|
72
|
+
}
|
|
73
|
+
base = (config.base_url or defaults.get(config.provider, "")).rstrip("/")
|
|
74
|
+
parsed = urlparse(base)
|
|
75
|
+
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
|
76
|
+
raise EmbeddingError("Embedding base_url must be an HTTP or HTTPS URL.")
|
|
77
|
+
if parsed.username or parsed.password or parsed.query or parsed.fragment:
|
|
78
|
+
raise EmbeddingError("Use an environment variable for credentials; keep base_url free of query parameters.")
|
|
79
|
+
if parsed.scheme == "http" and parsed.hostname.lower() not in {"localhost", "127.0.0.1", "::1"}:
|
|
80
|
+
raise EmbeddingError("Use HTTPS for remote embedding providers; HTTP is allowed only on loopback.")
|
|
81
|
+
return base
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _key(config: EmbeddingConfig) -> str:
|
|
85
|
+
key = os.getenv(config.api_key_env, "").strip() if config.api_key_env else ""
|
|
86
|
+
required = config.provider in {"gemini"}
|
|
87
|
+
if required and not key:
|
|
88
|
+
raise EmbeddingError(f"Set {config.api_key_env or 'api_key_env'} for the embedding provider.")
|
|
89
|
+
return key
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class OpenAICompatibleEmbeddingAdapter:
|
|
93
|
+
"""OpenAI-compatible embeddings, including local Ollama and LM Studio."""
|
|
94
|
+
|
|
95
|
+
def embed(self, text: str, config: EmbeddingConfig) -> list[float]:
|
|
96
|
+
key = _key(config)
|
|
97
|
+
headers = {"Authorization": f"Bearer {key}"} if key else {}
|
|
98
|
+
response = _request(_base(config) + "/embeddings", {"model": config.model, "input": text},
|
|
99
|
+
headers, config.timeout_seconds)
|
|
100
|
+
rows = response.get("data") or []
|
|
101
|
+
vector = rows[0].get("embedding") if rows and isinstance(rows[0], dict) else None
|
|
102
|
+
return _validate_vector(vector)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class GeminiEmbeddingAdapter:
|
|
106
|
+
"""Gemini embedContent endpoint with a separately configured embedding model."""
|
|
107
|
+
|
|
108
|
+
def embed(self, text: str, config: EmbeddingConfig) -> list[float]:
|
|
109
|
+
key = _key(config)
|
|
110
|
+
base = _base(config) if config.base_url else "https://generativelanguage.googleapis.com"
|
|
111
|
+
base = base.removesuffix("/v1beta").removesuffix("/v1")
|
|
112
|
+
endpoint = f"{base}/v1beta/models/{config.model}:embedContent"
|
|
113
|
+
response = _request(endpoint, {"content": {"parts": [{"text": text}]}},
|
|
114
|
+
{"x-goog-api-key": key}, config.timeout_seconds)
|
|
115
|
+
vector = (response.get("embedding") or {}).get("values")
|
|
116
|
+
return _validate_vector(vector)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _validate_vector(value: Any) -> list[float]:
|
|
120
|
+
if not isinstance(value, list) or not value or len(value) > 32768:
|
|
121
|
+
raise EmbeddingError("Embedding provider returned no usable vector.")
|
|
122
|
+
try:
|
|
123
|
+
vector = [float(item) for item in value]
|
|
124
|
+
except (TypeError, ValueError):
|
|
125
|
+
raise EmbeddingError("Embedding provider returned an invalid vector.") from None
|
|
126
|
+
if not all(math.isfinite(item) for item in vector):
|
|
127
|
+
raise EmbeddingError("Embedding provider returned non-finite vector values.")
|
|
128
|
+
return vector
|
|
129
|
+
|
|
130
|
+
|
memoryport/evaluation.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Small deterministic evaluation harness for memory-state regressions."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from .ledger import MemoryLedger
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def evaluate_cases(ledger: MemoryLedger, cases: list[dict[str, Any]]) -> dict[str, Any]:
|
|
11
|
+
"""Measure stale-value bleed, tombstone stickiness, and context density.
|
|
12
|
+
|
|
13
|
+
Each case has ``query``, ``provider`` and optional ``token_budget`` and
|
|
14
|
+
``stale_values``. Set ``deleted_slot`` and ``deleted_value`` to probe the
|
|
15
|
+
explicit user-delete guard for a previously removed slot.
|
|
16
|
+
"""
|
|
17
|
+
stale_denominator = stale_hits = sticky_denominator = sticky_passes = 0
|
|
18
|
+
fact_count = token_count = 0
|
|
19
|
+
details = []
|
|
20
|
+
for index, case in enumerate(cases):
|
|
21
|
+
compiled = ledger.context(str(case.get("query", "")),
|
|
22
|
+
provider=str(case.get("provider", "local")),
|
|
23
|
+
token_budget=int(case.get("token_budget", 900)),
|
|
24
|
+
as_of=case.get("as_of"))
|
|
25
|
+
text = compiled["text"].casefold()
|
|
26
|
+
stale = [str(value) for value in case.get("stale_values", []) if str(value)]
|
|
27
|
+
stale_denominator += bool(stale)
|
|
28
|
+
leaked = [value for value in stale if value.casefold() in text]
|
|
29
|
+
stale_hits += bool(leaked)
|
|
30
|
+
fact_count += len(compiled["citations"])
|
|
31
|
+
token_count += compiled["approx_tokens"]
|
|
32
|
+
|
|
33
|
+
sticky_result = None
|
|
34
|
+
if case.get("deleted_slot") and case.get("deleted_value"):
|
|
35
|
+
sticky_denominator += 1
|
|
36
|
+
state = ledger.slot_state(str(case["namespace"]), str(case["slot_key"]))
|
|
37
|
+
sticky_result = bool(state and state["status"] == "DELETED"
|
|
38
|
+
and str(case["deleted_value"]).casefold() not in text)
|
|
39
|
+
sticky_passes += sticky_result
|
|
40
|
+
details.append({"case": index, "stale_values_found": leaked,
|
|
41
|
+
"context_facts": len(compiled["citations"]),
|
|
42
|
+
"approx_tokens": compiled["approx_tokens"],
|
|
43
|
+
"deleted_slot_blocked": sticky_result})
|
|
44
|
+
|
|
45
|
+
return {
|
|
46
|
+
"case_count": len(cases),
|
|
47
|
+
"stale_fact_bleed_rate": stale_hits / stale_denominator if stale_denominator else None,
|
|
48
|
+
"stale_fact_cases": stale_denominator,
|
|
49
|
+
"correction_stickiness_score": sticky_passes / sticky_denominator if sticky_denominator else None,
|
|
50
|
+
"deleted_slot_cases": sticky_denominator,
|
|
51
|
+
"token_density_facts_per_100_tokens": 100 * fact_count / token_count if token_count else 0.0,
|
|
52
|
+
"total_injected_facts": fact_count,
|
|
53
|
+
"approx_tokens": token_count,
|
|
54
|
+
"details": details,
|
|
55
|
+
"note": "Approximate token counts use 4 characters per token; compare only under the same tokenizer convention.",
|
|
56
|
+
}
|
|
57
|
+
|
memoryport/extractor.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""Model-agnostic durable-fact extraction into canonical ledger slots."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
from typing import Any, Callable
|
|
8
|
+
|
|
9
|
+
from .ledger import MemoryLedger
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
_SECRET_LIKE = re.compile(
|
|
13
|
+
r"(?i)(?:\b(?:api[_ -]?key|password|secret|access[_ -]?token|private[_ -]?key)\b\s*(?:[:=]|is\s+)\s*\S+|"
|
|
14
|
+
r"\b(?:sk-[A-Za-z0-9_-]{16,}|gh[pousr]_[A-Za-z0-9]{20,}|AKIA[0-9A-Z]{16})\b)"
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
_INSTRUCTIONS = """Extract at most five durable facts that the user explicitly states in their message.
|
|
18
|
+
Return only JSON in this schema:
|
|
19
|
+
{"memories":[{"namespace":"work.project","slot_key":"primary_database","value":"MongoDB Atlas","confidence":0.9,"source_quote":"exact substring from the user message","sensitivity_tier":"PERSONAL","allowed_providers":[],"blocked_providers":[],"ephemeral_redaction":false,"valid_from":null}]}
|
|
20
|
+
Use stable canonical namespace and slot keys. Return the same slot for a correction to an earlier value. Ignore questions, guesses, one-off tasks, secrets, credentials, and sensitive personal data. Do not derive facts from the assistant response. Keep values concise and factual. If nothing qualifies, return {"memories":[]}.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class MemoryExtractor:
|
|
25
|
+
"""Run a caller-supplied model function and validate its proposed memories.
|
|
26
|
+
|
|
27
|
+
``generate_json`` receives a prompt and returns a JSON string or parsed
|
|
28
|
+
object. The vault itself never contacts a model provider.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
def __init__(self, ledger: MemoryLedger, generate_json: Callable[[str], Any], *,
|
|
32
|
+
minimum_confidence: float = 0.8, max_facts: int = 5):
|
|
33
|
+
if not 0 <= minimum_confidence <= 1:
|
|
34
|
+
raise ValueError("minimum_confidence must be between 0 and 1")
|
|
35
|
+
if not 1 <= max_facts <= 20:
|
|
36
|
+
raise ValueError("max_facts must be between 1 and 20")
|
|
37
|
+
self.ledger = ledger
|
|
38
|
+
self.generate_json = generate_json
|
|
39
|
+
self.minimum_confidence = minimum_confidence
|
|
40
|
+
self.max_facts = max_facts
|
|
41
|
+
|
|
42
|
+
def ingest_turn(self, user_message: str, assistant_response: str = "", *,
|
|
43
|
+
conversation_id: str | None = None, model_id: str | None = None) -> dict[str, Any]:
|
|
44
|
+
user_message = str(user_message).strip()
|
|
45
|
+
if not user_message:
|
|
46
|
+
return {"saved": [], "skipped": 0}
|
|
47
|
+
prompt = (_INSTRUCTIONS + "\nUSER MESSAGE:\n" + user_message[:12000]
|
|
48
|
+
+ "\nASSISTANT RESPONSE (context only; do not extract facts from it):\n"
|
|
49
|
+
+ str(assistant_response)[:4000])
|
|
50
|
+
raw = self.generate_json(prompt)
|
|
51
|
+
parsed = json.loads(raw) if isinstance(raw, str) else raw
|
|
52
|
+
if not isinstance(parsed, dict) or not isinstance(parsed.get("memories", []), list):
|
|
53
|
+
raise ValueError("extractor must return a JSON object with a memories list")
|
|
54
|
+
|
|
55
|
+
saved = []
|
|
56
|
+
skipped = 0
|
|
57
|
+
user_folded = user_message.casefold()
|
|
58
|
+
for candidate in parsed.get("memories", [])[:self.max_facts]:
|
|
59
|
+
if not isinstance(candidate, dict):
|
|
60
|
+
skipped += 1
|
|
61
|
+
continue
|
|
62
|
+
quote = candidate.get("source_quote")
|
|
63
|
+
value = candidate.get("value")
|
|
64
|
+
try:
|
|
65
|
+
confidence = float(candidate.get("confidence", 0))
|
|
66
|
+
except (TypeError, ValueError):
|
|
67
|
+
skipped += 1
|
|
68
|
+
continue
|
|
69
|
+
if (not isinstance(quote, str) or not quote.strip() or quote.casefold() not in user_folded
|
|
70
|
+
or not isinstance(value, str) or not value.strip() or confidence < self.minimum_confidence
|
|
71
|
+
or _SECRET_LIKE.search(value) or _SECRET_LIKE.search(quote)):
|
|
72
|
+
skipped += 1
|
|
73
|
+
continue
|
|
74
|
+
tier = str(candidate.get("sensitivity_tier", "PERSONAL")).upper()
|
|
75
|
+
if tier in {"SECRET", "CONFIDENTIAL"}:
|
|
76
|
+
skipped += 1
|
|
77
|
+
continue
|
|
78
|
+
try:
|
|
79
|
+
record = self.ledger.record(
|
|
80
|
+
str(candidate.get("namespace", "personal.general")), str(candidate.get("slot_key", "fact")),
|
|
81
|
+
value, confidence=min(1.0, confidence), verification_state="AUTO_EXTRACTED",
|
|
82
|
+
sensitivity_tier=tier,
|
|
83
|
+
allowed_providers=candidate.get("allowed_providers") or [],
|
|
84
|
+
blocked_providers=candidate.get("blocked_providers") or [],
|
|
85
|
+
ephemeral_redaction=bool(candidate.get("ephemeral_redaction", False)),
|
|
86
|
+
valid_from=candidate.get("valid_from"), source_conversation_id=conversation_id,
|
|
87
|
+
source_message=quote.strip(), extracted_by_model=model_id,
|
|
88
|
+
)
|
|
89
|
+
except ValueError:
|
|
90
|
+
skipped += 1
|
|
91
|
+
continue
|
|
92
|
+
saved.append(record)
|
|
93
|
+
return {"saved": saved, "skipped": skipped}
|
|
94
|
+
|
memoryport/factory.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Create and cache the configured local or MongoDB ledger backend."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from functools import lru_cache
|
|
6
|
+
import os
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from .ledger import MemoryLedger
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def default_vault_path() -> Path:
|
|
13
|
+
"""Use Memty's data folder, reusing older vaults when they already exist."""
|
|
14
|
+
configured = (os.getenv("MEMTY_MEMORY_PATH") or os.getenv("MEMTY_VAULT_PATH")
|
|
15
|
+
or os.getenv("NEXORAI_MEMORY_PATH") or os.getenv("NEXOR_VAULT_PATH"))
|
|
16
|
+
if configured:
|
|
17
|
+
return Path(configured).expanduser()
|
|
18
|
+
home = Path.home()
|
|
19
|
+
current = home / ".memty" / "memory.sqlite3"
|
|
20
|
+
previous = home / ".nexorai" / "memory.sqlite3"
|
|
21
|
+
legacy = home / ".nexor" / "memory.sqlite3"
|
|
22
|
+
if current.exists():
|
|
23
|
+
return current
|
|
24
|
+
if previous.exists():
|
|
25
|
+
return previous
|
|
26
|
+
if legacy.exists():
|
|
27
|
+
return legacy
|
|
28
|
+
return current
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@lru_cache(maxsize=512)
|
|
32
|
+
def _sqlite(path: str, user_id: str) -> MemoryLedger:
|
|
33
|
+
return MemoryLedger(path, user_id=user_id)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@lru_cache(maxsize=512)
|
|
37
|
+
def _mongo(uri: str, database: str, collection: str, user_id: str):
|
|
38
|
+
from .mongo_ledger import MongoMemoryLedger
|
|
39
|
+
return MongoMemoryLedger(uri, user_id, database=database, collection=collection)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def open_ledger(user_id: str = "local", *, path: str | Path | None = None):
|
|
43
|
+
"""Open the selected backend, keeping one connection/store per user."""
|
|
44
|
+
backend = os.getenv("MEMTY_VAULT_BACKEND", os.getenv("NEXOR_VAULT_BACKEND", "sqlite")).casefold()
|
|
45
|
+
if backend == "mongodb":
|
|
46
|
+
uri = os.getenv("MEMTY_VAULT_MONGO_URI") or os.getenv("NEXOR_VAULT_MONGO_URI") or os.getenv("MONGO_URI")
|
|
47
|
+
if not uri:
|
|
48
|
+
raise RuntimeError("Set MEMTY_VAULT_MONGO_URI to use the MongoDB ledger backend")
|
|
49
|
+
return _mongo(uri, os.getenv("MEMORY_DATABASE", "Memory"),
|
|
50
|
+
os.getenv("MEMTY_VAULT_MONGO_COLLECTION", os.getenv("NEXOR_VAULT_MONGO_COLLECTION", "memory_vault_ledger")), str(user_id))
|
|
51
|
+
if backend != "sqlite":
|
|
52
|
+
raise ValueError("MEMTY_VAULT_BACKEND must be sqlite or mongodb")
|
|
53
|
+
if path is None:
|
|
54
|
+
if os.getenv("MEMTY_VAULT_MULTIUSER", os.getenv("NEXOR_VAULT_MULTIUSER", "false")).casefold() == "true":
|
|
55
|
+
root = Path(os.getenv("MEMTY_VAULT_ROOT", Path.home() / ".memty" / "users"))
|
|
56
|
+
import hashlib
|
|
57
|
+
path = root / (hashlib.sha256(str(user_id).encode()).hexdigest() + ".sqlite3")
|
|
58
|
+
else:
|
|
59
|
+
path = default_vault_path()
|
|
60
|
+
return _sqlite(str(path), str(user_id))
|
|
61
|
+
|