loop-memory 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- loop_memory/__init__.py +62 -0
- loop_memory/backends/__init__.py +13 -0
- loop_memory/backends/embedding.py +82 -0
- loop_memory/backends/sentence_embedder.py +30 -0
- loop_memory/backends/vector_store.py +139 -0
- loop_memory/cli/__init__.py +0 -0
- loop_memory/cli/_common.py +68 -0
- loop_memory/cli/commands/__init__.py +13 -0
- loop_memory/cli/commands/cognitive.py +205 -0
- loop_memory/cli/commands/diag.py +346 -0
- loop_memory/cli/commands/graph.py +21 -0
- loop_memory/cli/commands/hooks.py +212 -0
- loop_memory/cli/commands/read.py +362 -0
- loop_memory/cli/commands/serve.py +147 -0
- loop_memory/cli/commands/write.py +138 -0
- loop_memory/cli/main.py +115 -0
- loop_memory/engine/__init__.py +0 -0
- loop_memory/engine/loop.py +247 -0
- loop_memory/engine/reflect.py +89 -0
- loop_memory/examples/__init__.py +0 -0
- loop_memory/examples/demo.py +39 -0
- loop_memory/export/__init__.py +39 -0
- loop_memory/export/memory_md.py +629 -0
- loop_memory/graph/__init__.py +0 -0
- loop_memory/graph/build.py +259 -0
- loop_memory/graph/extract.py +197 -0
- loop_memory/ingest/__init__.py +0 -0
- loop_memory/ingest/loader.py +782 -0
- loop_memory/ingest/pipeline.py +458 -0
- loop_memory/jobs/__init__.py +0 -0
- loop_memory/jobs/cognitive.py +353 -0
- loop_memory/jobs/compact.py +371 -0
- loop_memory/jobs/consolidate.py +95 -0
- loop_memory/jobs/contradiction.py +281 -0
- loop_memory/jobs/evolution.py +2021 -0
- loop_memory/jobs/graph.py +395 -0
- loop_memory/jobs/llm_compact_pass.py +24 -0
- loop_memory/jobs/llm_consolidate.py +980 -0
- loop_memory/jobs/scheduler.py +495 -0
- loop_memory/llm/__init__.py +0 -0
- loop_memory/llm/base.py +80 -0
- loop_memory/llm/openai_adapter.py +31 -0
- loop_memory/llm/providers.py +517 -0
- loop_memory/mcp/__init__.py +804 -0
- loop_memory/memory/__init__.py +0 -0
- loop_memory/memory/types.py +199 -0
- loop_memory/privacy/__init__.py +22 -0
- loop_memory/privacy/private.py +46 -0
- loop_memory/privacy/redact.py +188 -0
- loop_memory/py.typed +0 -0
- loop_memory/sdk.py +875 -0
- loop_memory/sdk_extensions.py +384 -0
- loop_memory/security/__init__.py +20 -0
- loop_memory/security/secrets.py +464 -0
- loop_memory/serve/__init__.py +0 -0
- loop_memory/serve/app.py +506 -0
- loop_memory/serve/handlers.py +316 -0
- loop_memory/serve/routes/_shared.py +59 -0
- loop_memory/serve/routes/admin.py +970 -0
- loop_memory/serve/routes/cognitive.py +64 -0
- loop_memory/serve/routes/export.py +65 -0
- loop_memory/serve/routes/graph.py +101 -0
- loop_memory/serve/routes/insights.py +702 -0
- loop_memory/serve/routes/memories.py +435 -0
- loop_memory/serve/routes/sessions.py +75 -0
- loop_memory/serve/routes/system.py +493 -0
- loop_memory/serve/routes/wiki.py +812 -0
- loop_memory/serve/static/__init__.py +0 -0
- loop_memory/serve/static/index.html +15 -0
- loop_memory/serve/watcher.py +451 -0
- loop_memory/storage/__init__.py +5 -0
- loop_memory/storage/retrieval.py +365 -0
- loop_memory/storage/sqlite_store.py +3627 -0
- loop_memory/wiki/__init__.py +41 -0
- loop_memory/wiki/backfill.py +143 -0
- loop_memory/wiki/classifier.py +238 -0
- loop_memory/wiki/prompts.py +295 -0
- loop_memory/wiki/scope.py +227 -0
- loop_memory-0.4.0.dist-info/METADATA +627 -0
- loop_memory-0.4.0.dist-info/RECORD +84 -0
- loop_memory-0.4.0.dist-info/WHEEL +5 -0
- loop_memory-0.4.0.dist-info/entry_points.txt +2 -0
- loop_memory-0.4.0.dist-info/licenses/LICENSE +21 -0
- loop_memory-0.4.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
"""Read-only commands: chat, stats, recall, ask, export, inject."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import datetime as _dt
|
|
6
|
+
import io
|
|
7
|
+
import json
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from .._common import DEFAULT_DB, default_db_path, die, make_engine
|
|
12
|
+
from ...privacy import redact_text, strip_private_spans
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def run_chat(_args) -> int:
|
|
16
|
+
engine = make_engine()
|
|
17
|
+
print(f"[loop-memory] ready. {engine}")
|
|
18
|
+
print("Type your message, ':stats' for diagnostics, ':recall <q>' to search, ':quit' to exit.\n")
|
|
19
|
+
while True:
|
|
20
|
+
try:
|
|
21
|
+
line = input("you> ").strip()
|
|
22
|
+
except (EOFError, KeyboardInterrupt):
|
|
23
|
+
print()
|
|
24
|
+
return 0
|
|
25
|
+
if not line:
|
|
26
|
+
continue
|
|
27
|
+
if line in {":quit", ":q", ":exit"}:
|
|
28
|
+
return 0
|
|
29
|
+
if line == ":stats":
|
|
30
|
+
print(json.dumps(
|
|
31
|
+
{k: len(getattr(engine.short, "_items", [])) if k == "short" else 0 for k in ["short"]},
|
|
32
|
+
indent=2,
|
|
33
|
+
))
|
|
34
|
+
continue
|
|
35
|
+
if line.startswith(":recall "):
|
|
36
|
+
q = line[len(":recall "):].strip()
|
|
37
|
+
for i, m in enumerate(engine.recall(q), 1):
|
|
38
|
+
print(f" {i}. ({m.kind}) {m.text}")
|
|
39
|
+
continue
|
|
40
|
+
result = engine.turn(line)
|
|
41
|
+
print(f"bot> {result.reply}")
|
|
42
|
+
if result.diagnostics.get("stored"):
|
|
43
|
+
print(f" [stored {result.diagnostics['stored']} new memories]")
|
|
44
|
+
return 0
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def run_stats(_args) -> int:
|
|
48
|
+
from ...storage.sqlite_store import MemoryStore
|
|
49
|
+
print(json.dumps(MemoryStore(DEFAULT_DB).stats(), indent=2))
|
|
50
|
+
return 0
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def run_recall(args) -> int:
|
|
54
|
+
from ...storage.sqlite_store import MemoryStore
|
|
55
|
+
if not args:
|
|
56
|
+
return die("usage: loop-memory recall <query>")
|
|
57
|
+
store = MemoryStore(default_db_path())
|
|
58
|
+
query = " ".join(args)
|
|
59
|
+
r = store.recall(query, limit=10)
|
|
60
|
+
has = False
|
|
61
|
+
if r["wiki"]:
|
|
62
|
+
has = True
|
|
63
|
+
print(f"## Distilled knowledge ({len(r['wiki'])} match{'es' if len(r['wiki'])!=1 else ''})")
|
|
64
|
+
for w in r["wiki"]:
|
|
65
|
+
tag_s = " [" + ", ".join(w.get("tags") or []) + "]" if w.get("tags") else ""
|
|
66
|
+
print(f"- **{w['title']}** (`{w['slug']}`) — imp {w['importance']:.2f}{tag_s}")
|
|
67
|
+
if w.get("summary"):
|
|
68
|
+
print(f" > {w['summary'][:240]}")
|
|
69
|
+
print()
|
|
70
|
+
if r["memories"]:
|
|
71
|
+
has = True
|
|
72
|
+
print(f"## Raw memories ({len(r['memories'])} match{'es' if len(r['memories'])!=1 else ''})")
|
|
73
|
+
for m in r["memories"]:
|
|
74
|
+
tag_s = " [" + ", ".join(m.get("tags") or []) + "]" if m.get("tags") else ""
|
|
75
|
+
print(f"- [{m['kind']}] (imp={m['importance']:.2f}){tag_s}")
|
|
76
|
+
print(f" {m['text'][:240]}")
|
|
77
|
+
print()
|
|
78
|
+
if r["entities"]:
|
|
79
|
+
has = True
|
|
80
|
+
print(f"## Entities ({len(r['entities'])})")
|
|
81
|
+
for e in r["entities"]:
|
|
82
|
+
print(f"- {e['name']} _({e['entity_kind']}, w={e['weight']:.2f})_")
|
|
83
|
+
print()
|
|
84
|
+
if not has:
|
|
85
|
+
print(f"_(nothing matched {query!r})_")
|
|
86
|
+
return 0
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def run_export(args) -> int:
|
|
90
|
+
"""Export distilled wiki pages as one markdown file.
|
|
91
|
+
|
|
92
|
+
Usage: loop-memory export [--out PATH] [--q QUERY]
|
|
93
|
+
"""
|
|
94
|
+
from ...storage.sqlite_store import MemoryStore
|
|
95
|
+
out_path = None
|
|
96
|
+
query = None
|
|
97
|
+
i = 0
|
|
98
|
+
while i < len(args):
|
|
99
|
+
a = args[i]
|
|
100
|
+
if a == "--out" and i + 1 < len(args):
|
|
101
|
+
out_path = args[i + 1]; i += 2
|
|
102
|
+
elif a == "--q" and i + 1 < len(args):
|
|
103
|
+
query = args[i + 1]; i += 2
|
|
104
|
+
else:
|
|
105
|
+
i += 1
|
|
106
|
+
if out_path is None:
|
|
107
|
+
stamp = _dt.date.today().isoformat()
|
|
108
|
+
out_path = str(Path.home() / f"loop-memory-export-{stamp}.md")
|
|
109
|
+
out_path = str(Path(out_path).expanduser())
|
|
110
|
+
store = MemoryStore(default_db_path())
|
|
111
|
+
pages = store.list_wiki_pages(limit=500, query=query)
|
|
112
|
+
lines = ["# Loop Memory — Distilled Knowledge", ""]
|
|
113
|
+
lines.append(f"_Exported {len(pages)} wiki pages._")
|
|
114
|
+
lines.append("")
|
|
115
|
+
for p in pages:
|
|
116
|
+
title = (p.get("title") or "untitled").strip()
|
|
117
|
+
# Defensive redaction: bodies should already be redacted at
|
|
118
|
+
# write time, but a page distilled before the redaction hook
|
|
119
|
+
# shipped could still contain a leaked secret. Run the page
|
|
120
|
+
# through the same pipeline one more time so the exported
|
|
121
|
+
# markdown is safe to paste into any public doc.
|
|
122
|
+
body = redact_text(strip_private_spans((p.get("body") or "").strip()))
|
|
123
|
+
summary = redact_text(strip_private_spans((p.get("summary") or "").strip()))
|
|
124
|
+
lines.append(f"## {title}")
|
|
125
|
+
lines.append("")
|
|
126
|
+
if summary and summary != title:
|
|
127
|
+
lines.append(f"> {summary}")
|
|
128
|
+
lines.append("")
|
|
129
|
+
lines.append(body)
|
|
130
|
+
lines.append("")
|
|
131
|
+
Path(out_path).write_text("\n".join(lines), encoding="utf-8")
|
|
132
|
+
print(f"✅ wrote {len(pages)} pages → {out_path}")
|
|
133
|
+
print(" paste this file into any LLM client as context to apply your distilled knowledge.")
|
|
134
|
+
return 0
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def run_digest(args) -> int:
|
|
138
|
+
"""Build a tight, byte-budgeted markdown digest of the long-term
|
|
139
|
+
memory store. Designed to be injected as ``AGENTS.md`` at session
|
|
140
|
+
start so the assistant carries a compact summary of the user's
|
|
141
|
+
distilled knowledge — without paying the cost of every historical
|
|
142
|
+
turn being replayed into context.
|
|
143
|
+
|
|
144
|
+
The output is plain markdown, ordered by importance, capped at
|
|
145
|
+
``--max-chars`` (default 12000 ≈ 3000 tokens). This is small
|
|
146
|
+
enough to live permanently in the assistant's system prompt,
|
|
147
|
+
dramatically reducing how much session history needs to be
|
|
148
|
+
carried per turn.
|
|
149
|
+
|
|
150
|
+
Usage: loop-memory digest [--out PATH] [--max-chars 12000] [--q QUERY]
|
|
151
|
+
"""
|
|
152
|
+
from ...storage.sqlite_store import MemoryStore
|
|
153
|
+
out_path = None
|
|
154
|
+
max_chars = 12000
|
|
155
|
+
query = None
|
|
156
|
+
i = 0
|
|
157
|
+
while i < len(args):
|
|
158
|
+
a = args[i]
|
|
159
|
+
if a == "--out" and i + 1 < len(args):
|
|
160
|
+
out_path = args[i + 1]; i += 2
|
|
161
|
+
elif a == "--max-chars" and i + 1 < len(args):
|
|
162
|
+
max_chars = max(500, int(args[i + 1])); i += 2
|
|
163
|
+
elif a == "--q" and i + 1 < len(args):
|
|
164
|
+
query = args[i + 1]; i += 2
|
|
165
|
+
else:
|
|
166
|
+
i += 1
|
|
167
|
+
if out_path is None:
|
|
168
|
+
out_path = str(Path.home() / ".loop_memory" / "AGENTS.md")
|
|
169
|
+
out_path = str(Path(out_path).expanduser())
|
|
170
|
+
Path(out_path).parent.mkdir(parents=True, exist_ok=True)
|
|
171
|
+
store = MemoryStore(default_db_path())
|
|
172
|
+
pages = store.list_wiki_pages(limit=200, query=query)
|
|
173
|
+
# Order by importance desc, then updated_at desc — surface the
|
|
174
|
+
# most useful and freshest knowledge first.
|
|
175
|
+
pages.sort(key=lambda p: (float(p.get("importance") or 0), float(p.get("updated_at") or 0)), reverse=True)
|
|
176
|
+
|
|
177
|
+
# Score the page's density so we keep self-contained pages and
|
|
178
|
+
# drop thin ones when the budget runs out.
|
|
179
|
+
def density(p):
|
|
180
|
+
body = (p.get("body") or "").strip()
|
|
181
|
+
facts = p.get("key_facts") or []
|
|
182
|
+
return len(body) + 80 * len(facts)
|
|
183
|
+
|
|
184
|
+
kept: list = []
|
|
185
|
+
total = 0
|
|
186
|
+
overhead = 280 # header + footer
|
|
187
|
+
budget = max_chars - overhead
|
|
188
|
+
for p in pages:
|
|
189
|
+
body = redact_text(strip_private_spans((p.get("body") or "").strip()))
|
|
190
|
+
summary = redact_text(strip_private_spans((p.get("summary") or "").strip()))
|
|
191
|
+
title = (p.get("title") or "untitled").strip()
|
|
192
|
+
facts = p.get("key_facts") or []
|
|
193
|
+
# Build the candidate block
|
|
194
|
+
block_lines = [f"## {title}", ""]
|
|
195
|
+
if summary and summary != title:
|
|
196
|
+
block_lines += [f"> {summary}", ""]
|
|
197
|
+
if facts:
|
|
198
|
+
block_lines.append("**Key facts:**")
|
|
199
|
+
for f in facts[:8]:
|
|
200
|
+
block_lines.append(f"- {f}")
|
|
201
|
+
block_lines.append("")
|
|
202
|
+
if body:
|
|
203
|
+
# Cap each page body at 800 chars to keep digest compact.
|
|
204
|
+
if len(body) > 800:
|
|
205
|
+
body = body[:800].rstrip() + "…"
|
|
206
|
+
block_lines += [body, ""]
|
|
207
|
+
block = "\n".join(block_lines)
|
|
208
|
+
if total + len(block) > budget:
|
|
209
|
+
break
|
|
210
|
+
kept.append((p, block))
|
|
211
|
+
total += len(block)
|
|
212
|
+
|
|
213
|
+
lines = ["# Distilled knowledge — auto-injected memory digest", ""]
|
|
214
|
+
lines.append(f"_Compiled from {len(kept)} of {len(pages)} wiki pages, capped at {max_chars:,} chars (~{max_chars//4:,} tokens)._")
|
|
215
|
+
lines.append(f"_Generated { _dt.datetime.now().isoformat(timespec='seconds') }._")
|
|
216
|
+
lines.append("")
|
|
217
|
+
lines.append("Read me at the start of every task. Update by running `loop-memory digest` again, "
|
|
218
|
+
"or via the web UI's Settings → 'Recompile digest'.")
|
|
219
|
+
lines.append("")
|
|
220
|
+
for _, block in kept:
|
|
221
|
+
lines.append(block)
|
|
222
|
+
Path(out_path).write_text("\n".join(lines), encoding="utf-8")
|
|
223
|
+
print(f"✅ digest → {out_path} ({total:,} chars, {len(kept)} pages)")
|
|
224
|
+
return 0
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def run_ask(args) -> int:
|
|
228
|
+
"""Print a copy-pasteable context block for a query — no server needed."""
|
|
229
|
+
from ...storage.sqlite_store import MemoryStore
|
|
230
|
+
if not args:
|
|
231
|
+
return die("usage: loop-memory ask <query>")
|
|
232
|
+
q = " ".join(args).strip()
|
|
233
|
+
store = MemoryStore(default_db_path())
|
|
234
|
+
r = store.recall(q, limit=8)
|
|
235
|
+
n_wiki = len(r["wiki"])
|
|
236
|
+
n_mem = len(r["memories"])
|
|
237
|
+
print(f"# Distilled knowledge — {q}\n")
|
|
238
|
+
print(f"_matched {n_wiki} wiki pages + {n_mem} memories (unified recall)_\n")
|
|
239
|
+
if not (n_wiki or n_mem):
|
|
240
|
+
print(f"_(no memories or wiki pages matched {q!r})_")
|
|
241
|
+
print()
|
|
242
|
+
print("If this is a brand-new question, the distillation pipeline "
|
|
243
|
+
"needs to run at least once: open the web UI → AI Consolidate, "
|
|
244
|
+
"or run `loop-memory consolidate-now`.")
|
|
245
|
+
return 0
|
|
246
|
+
for w in r["wiki"][:5]:
|
|
247
|
+
print(f"## {w.get('title')}")
|
|
248
|
+
print()
|
|
249
|
+
if w.get("summary"):
|
|
250
|
+
print(f"> {w['summary']}")
|
|
251
|
+
print()
|
|
252
|
+
body = (w.get("body") or "").strip()
|
|
253
|
+
if len(body) > 800:
|
|
254
|
+
body = body[:800] + "…"
|
|
255
|
+
print(body)
|
|
256
|
+
print()
|
|
257
|
+
for m in r["memories"][:4]:
|
|
258
|
+
print(f"## Memory ({m['kind']})")
|
|
259
|
+
print()
|
|
260
|
+
text = (m.get("text") or "").strip()
|
|
261
|
+
if len(text) > 600:
|
|
262
|
+
text = text[:600] + "…"
|
|
263
|
+
print(text)
|
|
264
|
+
print()
|
|
265
|
+
return 0
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def run_inject(args) -> int:
|
|
269
|
+
"""Print a context block of distilled wiki + relevant memories.
|
|
270
|
+
|
|
271
|
+
Designed for SessionStart hooks so every new conversation starts
|
|
272
|
+
with the user's curated knowledge already in context.
|
|
273
|
+
|
|
274
|
+
With no arguments, surfaces the user's highest-importance wiki
|
|
275
|
+
pages + preference facts. With a query argument (passed by the
|
|
276
|
+
hook from the user's first message), it returns the most relevant
|
|
277
|
+
memories for that query.
|
|
278
|
+
"""
|
|
279
|
+
from ...storage.sqlite_store import MemoryStore
|
|
280
|
+
store = MemoryStore(default_db_path())
|
|
281
|
+
query = " ".join(args).strip()
|
|
282
|
+
out = io.StringIO()
|
|
283
|
+
out.write("# Long-term memory context\n")
|
|
284
|
+
if query:
|
|
285
|
+
r = store.recall(query, limit=10)
|
|
286
|
+
wiki = r["wiki"][:6]
|
|
287
|
+
mem = r["memories"][:6]
|
|
288
|
+
out.write(
|
|
289
|
+
f"_(generated by loop-memory for query {query!r}: "
|
|
290
|
+
f"{len(wiki)} wiki pages + {len(mem)} memories)_\n\n"
|
|
291
|
+
)
|
|
292
|
+
if wiki:
|
|
293
|
+
out.write("## Distilled knowledge (wiki, ranked for this query)\n\n")
|
|
294
|
+
for w in wiki:
|
|
295
|
+
title = w.get("title", w.get("slug", "?"))
|
|
296
|
+
slug = w.get("slug", "")
|
|
297
|
+
tags = w.get("tags") or []
|
|
298
|
+
tag_s = f" — [{', '.join(tags[:4])}]" if tags else ""
|
|
299
|
+
text = (w.get("summary") or w.get("body") or "").strip()
|
|
300
|
+
if len(text) > 380:
|
|
301
|
+
text = text[:379] + "…"
|
|
302
|
+
out.write(f"- **{title}** (`{slug}`){tag_s}\n")
|
|
303
|
+
if text:
|
|
304
|
+
out.write(f" {text}\n")
|
|
305
|
+
out.write("\n")
|
|
306
|
+
if mem:
|
|
307
|
+
out.write("## Raw memories (ranked)\n\n")
|
|
308
|
+
for m in mem:
|
|
309
|
+
t = (m.get("text") or "").strip()
|
|
310
|
+
if len(t) > 280:
|
|
311
|
+
t = t[:279] + "…"
|
|
312
|
+
tag_s = ""
|
|
313
|
+
if m.get("tags"):
|
|
314
|
+
tag_s = f" [{', '.join(m['tags'][:3])}]"
|
|
315
|
+
try:
|
|
316
|
+
from datetime import datetime
|
|
317
|
+
when = datetime.fromtimestamp(float(m["created_at"])).strftime("%Y-%m-%d")
|
|
318
|
+
except Exception:
|
|
319
|
+
when = "?"
|
|
320
|
+
out.write(
|
|
321
|
+
f"- [{m['kind']} · {when} · imp={m['importance']:.2f}]{tag_s} {t}\n"
|
|
322
|
+
)
|
|
323
|
+
else:
|
|
324
|
+
# No query: surface the user's top preferences / highest-importance
|
|
325
|
+
# wiki pages so a brand-new conversation starts with the user
|
|
326
|
+
# already in context.
|
|
327
|
+
with store._conn() as c:
|
|
328
|
+
pref = c.execute(
|
|
329
|
+
"SELECT text, importance FROM memories "
|
|
330
|
+
"WHERE kind='fact' AND importance >= 0.6 "
|
|
331
|
+
"ORDER BY importance DESC, created_at DESC LIMIT 3"
|
|
332
|
+
).fetchall()
|
|
333
|
+
pages = store.list_wiki_pages(limit=10, min_importance=0.4)
|
|
334
|
+
out.write(
|
|
335
|
+
f"_(generated by loop-memory: {len(pages)} wiki pages, "
|
|
336
|
+
f"{len(pref)} preference facts)_\n\n"
|
|
337
|
+
)
|
|
338
|
+
if pref:
|
|
339
|
+
out.write("## User preferences (use these to guide style)\n\n")
|
|
340
|
+
for r in pref:
|
|
341
|
+
t = (r["text"] or "").strip()
|
|
342
|
+
if len(t) > 360:
|
|
343
|
+
t = t[:359] + "…"
|
|
344
|
+
out.write(f"- {t}\n")
|
|
345
|
+
out.write("\n")
|
|
346
|
+
if pages:
|
|
347
|
+
out.write("## Distilled knowledge (wiki)\n\n")
|
|
348
|
+
for pg in pages[:8]:
|
|
349
|
+
title = pg.get("title", pg.get("slug", "?"))
|
|
350
|
+
slug = pg.get("slug", "")
|
|
351
|
+
tags = pg.get("tags") or []
|
|
352
|
+
tag_s = f" — [{', '.join(tags[:4])}]" if tags else ""
|
|
353
|
+
summary = (pg.get("summary") or "").strip()
|
|
354
|
+
body = (pg.get("body") or "").strip()
|
|
355
|
+
text = summary or body
|
|
356
|
+
if len(text) > 320:
|
|
357
|
+
text = text[:319] + "…"
|
|
358
|
+
out.write(f"- **{title}** (`{slug}`){tag_s}\n")
|
|
359
|
+
if text:
|
|
360
|
+
out.write(f" {text}\n")
|
|
361
|
+
sys.stdout.write(out.getvalue())
|
|
362
|
+
return 0
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Server / hook / mcp commands."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from .._common import DEFAULT_DB, die
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def run_serve(args) -> int:
|
|
13
|
+
port = 7767
|
|
14
|
+
if "--port" in args:
|
|
15
|
+
i = args.index("--port")
|
|
16
|
+
port = int(args[i + 1])
|
|
17
|
+
host = "127.0.0.1"
|
|
18
|
+
if "--host" in args:
|
|
19
|
+
i = args.index("--host")
|
|
20
|
+
host = args[i + 1]
|
|
21
|
+
# Audit O9: warn loudly when binding outside loopback. By default
|
|
22
|
+
# the server only listens on 127.0.0.1; non-loopback binds expose
|
|
23
|
+
# the auth-protected but still-trusting endpoints (ingest, secrets,
|
|
24
|
+
# wiki import, etc.) to anyone on the network. The user must opt
|
|
25
|
+
# in by typing --host explicitly AND acknowledge the warning.
|
|
26
|
+
if host not in ("127.0.0.1", "localhost", "::1"):
|
|
27
|
+
print(
|
|
28
|
+
f"\n!! WARNING: loop-memory is binding to {host!r}, which is\n"
|
|
29
|
+
" reachable from other hosts on your network.\n"
|
|
30
|
+
" Anyone who can reach this port can:\n"
|
|
31
|
+
" * read every memory stored in this DB,\n"
|
|
32
|
+
" * write / delete memories,\n"
|
|
33
|
+
" * trigger ingest and consolidation,\n"
|
|
34
|
+
" * read / write wiki pages,\n"
|
|
35
|
+
" * configure LLM provider + API keys via /api/admin/*,\n"
|
|
36
|
+
" unless you also created an auth token (POST\n"
|
|
37
|
+
" /api/admin/auth/token). A token *only* protects the\n"
|
|
38
|
+
" browser; non-browser clients (curl, MCP) can still\n"
|
|
39
|
+
" mutate state unless they too send a Bearer header.\n"
|
|
40
|
+
" Press Ctrl-C now unless you know what you are doing.\n",
|
|
41
|
+
file=sys.stderr,
|
|
42
|
+
)
|
|
43
|
+
from ...serve.app import serve as _serve
|
|
44
|
+
print(f"[loop-memory] serving UI on http://{host}:{port}")
|
|
45
|
+
print(f"[loop-memory] db = {DEFAULT_DB}")
|
|
46
|
+
_serve(DEFAULT_DB, host=host, port=port)
|
|
47
|
+
return 0
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def run_hook(args) -> int:
|
|
51
|
+
"""Install a watcher that ingests new transcripts on change.
|
|
52
|
+
|
|
53
|
+
Accepts one or more ``--watch <path>`` flags. Multiple watch paths
|
|
54
|
+
are useful for openclaw, which has both ``agents/main/sessions``
|
|
55
|
+
(clawx transcripts) and ``workspace/memory`` (daily markdown logs).
|
|
56
|
+
"""
|
|
57
|
+
from ...backends.embedding import HashingEmbedder
|
|
58
|
+
from ...ingest.loader import get_loader
|
|
59
|
+
from ...ingest.pipeline import MemoryPipeline
|
|
60
|
+
from ...serve.watcher import run_watcher
|
|
61
|
+
from ...storage.sqlite_store import MemoryStore
|
|
62
|
+
if "--source" not in args or "--watch" not in args:
|
|
63
|
+
return die("usage: loop-memory hook --source <codex|claude|hermes> --watch <path> [--watch <path2> ...] [--once] [--idle SECONDS]")
|
|
64
|
+
s_idx = args.index("--source")
|
|
65
|
+
source = args[s_idx + 1]
|
|
66
|
+
# --once: process every eligible file once, then exit. Used by the
|
|
67
|
+
# server-side force-ingest endpoint so a manual button doesn't have
|
|
68
|
+
# to fork a permanent watcher.
|
|
69
|
+
once = "--once" in args
|
|
70
|
+
# --idle SECONDS: override the watcher's default 60s idle window.
|
|
71
|
+
# The server uses this to tighten the wait when the user clicks
|
|
72
|
+
# "Force ingest".
|
|
73
|
+
idle_seconds = 60.0
|
|
74
|
+
if "--idle" in args:
|
|
75
|
+
i = args.index("--idle")
|
|
76
|
+
try:
|
|
77
|
+
idle_seconds = float(args[i + 1])
|
|
78
|
+
except (ValueError, IndexError):
|
|
79
|
+
return die("--idle requires a numeric argument")
|
|
80
|
+
# Collect every --watch <path> pair (in order).
|
|
81
|
+
watches: list[Path] = []
|
|
82
|
+
i = 0
|
|
83
|
+
while i < len(args):
|
|
84
|
+
if args[i] == "--watch" and i + 1 < len(args):
|
|
85
|
+
watches.append(Path(args[i + 1]).expanduser())
|
|
86
|
+
i += 2
|
|
87
|
+
else:
|
|
88
|
+
i += 1
|
|
89
|
+
if not watches:
|
|
90
|
+
return die("--watch requires a path argument")
|
|
91
|
+
store = MemoryStore(DEFAULT_DB)
|
|
92
|
+
pipeline = MemoryPipeline(store, embedder=HashingEmbedder(dim=128))
|
|
93
|
+
loader = get_loader(source)
|
|
94
|
+
if once:
|
|
95
|
+
# --once mode: do exactly one ingest pass per watch dir, then
|
|
96
|
+
# return. Used by the server's force-ingest endpoint so a UI
|
|
97
|
+
# button can run a single batch without leaving a watcher
|
|
98
|
+
# process behind. We use idle_seconds=0 to ingest any file
|
|
99
|
+
# that has at least one new byte since the last successful
|
|
100
|
+
# ingest; the caller can override --idle to be stricter.
|
|
101
|
+
from ...serve.watcher import run_once
|
|
102
|
+
results = []
|
|
103
|
+
for w in watches:
|
|
104
|
+
r = run_once(loader, w, pipeline, idle_seconds=idle_seconds)
|
|
105
|
+
results.append({"watch_dir": str(w), **r})
|
|
106
|
+
# Persist results in a JSON line so callers (HTTP endpoint)
|
|
107
|
+
# can parse stdout deterministically.
|
|
108
|
+
import json as _json
|
|
109
|
+
print("LOOP_MEMORY_ONCE_RESULT " + _json.dumps(results))
|
|
110
|
+
return 0
|
|
111
|
+
if len(watches) == 1:
|
|
112
|
+
# Pass ``store=store`` so the watcher can hot-reload
|
|
113
|
+
# ingest frequency / poll cadence from Settings → 采集频率
|
|
114
|
+
# without needing a launchd restart.
|
|
115
|
+
run_watcher(loader, watches[0], pipeline, idle_seconds=idle_seconds, store=store)
|
|
116
|
+
return 0
|
|
117
|
+
# Multiple watches: spawn one thread per path so each watcher
|
|
118
|
+
# has its own poll loop and ledger (no cross-talk between
|
|
119
|
+
# directories).
|
|
120
|
+
import threading
|
|
121
|
+
threads = []
|
|
122
|
+
for w in watches:
|
|
123
|
+
t = threading.Thread(
|
|
124
|
+
target=run_watcher,
|
|
125
|
+
args=(loader, w, pipeline, 2.0, idle_seconds),
|
|
126
|
+
kwargs={"store": store},
|
|
127
|
+
daemon=True,
|
|
128
|
+
name=f"loop-memory-watcher-{w.name}",
|
|
129
|
+
)
|
|
130
|
+
t.start()
|
|
131
|
+
threads.append(t)
|
|
132
|
+
# Block forever (or until SIGINT) so the launchd plist keeps the
|
|
133
|
+
# process alive. All work happens on the spawned threads.
|
|
134
|
+
import signal as _sig
|
|
135
|
+
stop = threading.Event()
|
|
136
|
+
def _bye(*_): stop.set()
|
|
137
|
+
_sig.signal(_sig.SIGTERM, _bye)
|
|
138
|
+
_sig.signal(_sig.SIGINT, _bye)
|
|
139
|
+
stop.wait()
|
|
140
|
+
return 0
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def run_mcp(_args) -> int:
|
|
144
|
+
"""Run the MCP server on stdio (newline-delimited JSON-RPC 2.0)."""
|
|
145
|
+
from ...mcp import serve_stdio
|
|
146
|
+
serve_stdio()
|
|
147
|
+
return 0
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""Mutation commands: ingest, flush, consolidate, rescore, consolidate-now."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import urllib.error
|
|
8
|
+
import urllib.request
|
|
9
|
+
|
|
10
|
+
from .._common import DEFAULT_DB, die, parse_int_flag
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def run_ingest(args) -> int:
|
|
14
|
+
"""loop-memory ingest <codex|claude|hermes> [path] [--max-facts N] [--limit N]"""
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from ...backends.embedding import HashingEmbedder
|
|
18
|
+
from ...ingest.loader import default_paths, get_loader
|
|
19
|
+
from ...ingest.pipeline import MemoryPipeline
|
|
20
|
+
from ...storage.sqlite_store import MemoryStore
|
|
21
|
+
|
|
22
|
+
args = list(args)
|
|
23
|
+
max_facts, args = parse_int_flag(args, "--max-facts", 3)
|
|
24
|
+
limit, args = parse_int_flag(args, "--limit", 0)
|
|
25
|
+
if not args:
|
|
26
|
+
return die("usage: loop-memory ingest <codex|claude|hermes> [path] [--max-facts N] [--limit N]")
|
|
27
|
+
source = args[0]
|
|
28
|
+
root = Path(args[1]).expanduser() if len(args) > 1 else None
|
|
29
|
+
|
|
30
|
+
store = MemoryStore(DEFAULT_DB)
|
|
31
|
+
loader = get_loader(source)
|
|
32
|
+
base = root or default_paths()[source]
|
|
33
|
+
files = list(loader.discover(base))
|
|
34
|
+
if not files:
|
|
35
|
+
print(f"no transcripts under {base}", file=__import__("sys").stderr)
|
|
36
|
+
return 1
|
|
37
|
+
if limit:
|
|
38
|
+
files = files[:limit]
|
|
39
|
+
pipeline = MemoryPipeline(store, embedder=HashingEmbedder(dim=128), max_facts=max_facts)
|
|
40
|
+
ingested = 0
|
|
41
|
+
total_rows = 0
|
|
42
|
+
all_memory_ids: list = []
|
|
43
|
+
run_id = store.start_pipeline_run("ingest")
|
|
44
|
+
try:
|
|
45
|
+
for fp in files:
|
|
46
|
+
session = loader.load_one(fp)
|
|
47
|
+
if session is None:
|
|
48
|
+
continue
|
|
49
|
+
result = pipeline.run(session)
|
|
50
|
+
tag = "summary" if result.summary_items else "no-row"
|
|
51
|
+
print(f" + {fp.name}: {session.message_count:>4} turns -> {tag} {len(result.summary_items)} rows ({result.facts_count} facts)")
|
|
52
|
+
ingested += 1
|
|
53
|
+
total_rows += len(result.summary_items)
|
|
54
|
+
try:
|
|
55
|
+
for m in result.summary_items:
|
|
56
|
+
if getattr(m, "id", None):
|
|
57
|
+
all_memory_ids.append(m.id)
|
|
58
|
+
except Exception:
|
|
59
|
+
pass
|
|
60
|
+
finally:
|
|
61
|
+
store.finish_pipeline_run(
|
|
62
|
+
run_id,
|
|
63
|
+
in_count=len(files),
|
|
64
|
+
out_count=total_rows,
|
|
65
|
+
note=f"ingested {ingested} {source} sessions -> {total_rows} rows",
|
|
66
|
+
stats={"evidence_ids": all_memory_ids[-200:], "source": source, "files": [f.name for f in files[:50]]},
|
|
67
|
+
)
|
|
68
|
+
print(f"ingested {ingested} sessions -> {total_rows} memory rows (avg {total_rows/max(1,ingested):.1f} per session)")
|
|
69
|
+
return 0
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def run_flush(_args) -> int:
|
|
73
|
+
"""Force-reingest the latest transcript of each source."""
|
|
74
|
+
from ...backends.embedding import HashingEmbedder
|
|
75
|
+
from ...ingest.loader import default_paths, get_loader
|
|
76
|
+
from ...ingest.pipeline import MemoryPipeline
|
|
77
|
+
from ...storage.sqlite_store import MemoryStore
|
|
78
|
+
store = MemoryStore(DEFAULT_DB)
|
|
79
|
+
pipeline = MemoryPipeline(store, embedder=HashingEmbedder(dim=128))
|
|
80
|
+
n = 0
|
|
81
|
+
for src in ("codex", "claude", "hermes"):
|
|
82
|
+
loader = get_loader(src)
|
|
83
|
+
root = default_paths()[src]
|
|
84
|
+
files = list(loader.discover(root))
|
|
85
|
+
if not files:
|
|
86
|
+
continue
|
|
87
|
+
latest = max(files, key=lambda p: p.stat().st_mtime)
|
|
88
|
+
session = loader.load_one(latest)
|
|
89
|
+
if session is None:
|
|
90
|
+
continue
|
|
91
|
+
result = pipeline.run(session)
|
|
92
|
+
print(f" flushed {src}: {latest.name} -> {len(result.summary_items)} rows")
|
|
93
|
+
n += 1
|
|
94
|
+
return 0
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def run_consolidate(_args) -> int:
|
|
98
|
+
from ...backends.embedding import HashingEmbedder
|
|
99
|
+
from ...jobs.consolidate import Consolidator
|
|
100
|
+
from ...storage.sqlite_store import MemoryStore
|
|
101
|
+
store = MemoryStore(DEFAULT_DB)
|
|
102
|
+
report = Consolidator(store, embedder=HashingEmbedder(dim=128)).run()
|
|
103
|
+
print(json.dumps(report.__dict__, indent=2))
|
|
104
|
+
return 0
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def run_rescore(args) -> int:
|
|
108
|
+
from ...storage.sqlite_store import MemoryStore
|
|
109
|
+
half_life = 30.0
|
|
110
|
+
if args and args[0] == "--half-life" and len(args) >= 2:
|
|
111
|
+
half_life = float(args[1])
|
|
112
|
+
store = MemoryStore(DEFAULT_DB)
|
|
113
|
+
n = store.rescore_all(half_life)
|
|
114
|
+
print(f"rescored {n} memories with half_life_days={half_life}")
|
|
115
|
+
return 0
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def run_consolidate_now(_args) -> int:
|
|
119
|
+
"""Ask the running server to trigger a consolidation pass right now."""
|
|
120
|
+
import json as _json
|
|
121
|
+
port = os.environ.get("LOOP_MEMORY_PORT", "7767")
|
|
122
|
+
url = f"http://127.0.0.1:{port}/api/admin/consolidate-now"
|
|
123
|
+
try:
|
|
124
|
+
req = urllib.request.Request(url, method="POST")
|
|
125
|
+
with urllib.request.urlopen(req, timeout=8) as r:
|
|
126
|
+
data = _json.loads(r.read().decode())
|
|
127
|
+
except urllib.error.URLError as e:
|
|
128
|
+
print(f"could not reach loop-memory server on :{port}: {e.reason}", file=__import__("sys").stderr)
|
|
129
|
+
print(f"hint: start the server with `loop-memory serve --port {port}`.", file=__import__("sys").stderr)
|
|
130
|
+
return 1
|
|
131
|
+
except Exception as e:
|
|
132
|
+
print(f"request failed: {e}", file=__import__("sys").stderr)
|
|
133
|
+
return 1
|
|
134
|
+
if data.get("queued"):
|
|
135
|
+
print("✅ consolidation run queued — check the dashboard for live progress.")
|
|
136
|
+
else:
|
|
137
|
+
print("✅ consolidation done:", _json.dumps(data.get("result"), indent=2))
|
|
138
|
+
return 0
|