deeptutor-memory 1.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deeptutor/agents/loop/native_memory.py +24 -0
- deeptutor/api/routers/memory.py +785 -0
- deeptutor/services/memory/__init__.py +42 -0
- deeptutor/services/memory/consolidator/__init__.py +62 -0
- deeptutor/services/memory/consolidator/chunker.py +131 -0
- deeptutor/services/memory/consolidator/guards.py +104 -0
- deeptutor/services/memory/consolidator/line_doc.py +486 -0
- deeptutor/services/memory/consolidator/meta.py +194 -0
- deeptutor/services/memory/consolidator/modes/__init__.py +31 -0
- deeptutor/services/memory/consolidator/modes/_runtime.py +319 -0
- deeptutor/services/memory/consolidator/modes/_shims.py +129 -0
- deeptutor/services/memory/consolidator/modes/audit.py +481 -0
- deeptutor/services/memory/consolidator/modes/dedup.py +228 -0
- deeptutor/services/memory/consolidator/modes/merge.py +227 -0
- deeptutor/services/memory/consolidator/modes/update.py +706 -0
- deeptutor/services/memory/consolidator/parse.py +135 -0
- deeptutor/services/memory/consolidator/prompts/en/_meta.yaml +36 -0
- deeptutor/services/memory/consolidator/prompts/en/audit_l2.yaml +42 -0
- deeptutor/services/memory/consolidator/prompts/en/audit_l3.yaml +38 -0
- deeptutor/services/memory/consolidator/prompts/en/dedup.yaml +37 -0
- deeptutor/services/memory/consolidator/prompts/en/update_l2.yaml +45 -0
- deeptutor/services/memory/consolidator/prompts/en/update_l3.yaml +51 -0
- deeptutor/services/memory/consolidator/prompts/zh/_meta.yaml +35 -0
- deeptutor/services/memory/consolidator/prompts/zh/audit_l2.yaml +40 -0
- deeptutor/services/memory/consolidator/prompts/zh/audit_l3.yaml +34 -0
- deeptutor/services/memory/consolidator/prompts/zh/dedup.yaml +34 -0
- deeptutor/services/memory/consolidator/prompts/zh/update_l2.yaml +40 -0
- deeptutor/services/memory/consolidator/prompts/zh/update_l3.yaml +43 -0
- deeptutor/services/memory/consolidator/references.py +388 -0
- deeptutor/services/memory/consolidator/runs.py +405 -0
- deeptutor/services/memory/document.py +235 -0
- deeptutor/services/memory/ids.py +80 -0
- deeptutor/services/memory/ops.py +149 -0
- deeptutor/services/memory/paths.py +104 -0
- deeptutor/services/memory/prompts/en.yaml +156 -0
- deeptutor/services/memory/prompts/zh.yaml +144 -0
- deeptutor/services/memory/recall.py +237 -0
- deeptutor/services/memory/settings.py +198 -0
- deeptutor/services/memory/snapshot/__init__.py +127 -0
- deeptutor/services/memory/snapshot/adapters.py +690 -0
- deeptutor/services/memory/snapshot/diff.py +67 -0
- deeptutor/services/memory/snapshot/entity.py +92 -0
- deeptutor/services/memory/snapshot/store.py +113 -0
- deeptutor/services/memory/store.py +453 -0
- deeptutor/services/memory/trace.py +153 -0
- deeptutor_cli/memory.py +99 -0
- deeptutor_memory-1.7.0.dist-info/METADATA +6 -0
- deeptutor_memory-1.7.0.dist-info/RECORD +50 -0
- deeptutor_memory-1.7.0.dist-info/WHEEL +5 -0
- deeptutor_memory-1.7.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,785 @@
|
|
|
1
|
+
"""Memory v3 API — workbench backend.
|
|
2
|
+
|
|
3
|
+
Three layers, three modes (update / audit / dedup). Long-running work
|
|
4
|
+
is owned by the :mod:`runs` manager so a refresh / nav-away does not
|
|
5
|
+
kill it; clients re-attach by polling ``/runs/{id}/events?since=N``.
|
|
6
|
+
|
|
7
|
+
- ``GET /overview`` → all 11 docs' state + L1 backlog
|
|
8
|
+
- ``GET /doc/{layer}/{key}`` → raw MD
|
|
9
|
+
- ``GET /doc/{layer}/{key}/lines`` → line-numbered view
|
|
10
|
+
- ``PUT /doc/{layer}/{key}`` → user-edited save
|
|
11
|
+
- ``DELETE /doc/{layer}/{key}/entry/{id}`` → drop one entry
|
|
12
|
+
- ``POST /runs/start`` → start update/audit/dedup; returns run_id
|
|
13
|
+
- ``GET /runs/{id}`` → run state
|
|
14
|
+
- ``GET /runs/{id}/events?since=N`` → SSE-replay events from cursor N
|
|
15
|
+
- ``POST /runs/{id}/cancel`` → cooperative cancellation
|
|
16
|
+
- ``POST /runs/{id}/undo`` → restore latest run write
|
|
17
|
+
- ``GET /runs?layer=L2&key=chat`` → active+recent runs for one doc
|
|
18
|
+
- ``GET /settings`` → memory: settings subtree
|
|
19
|
+
- ``PUT /settings`` → save memory: settings subtree
|
|
20
|
+
- ``GET /trace/{surface}`` → paginated L1 events
|
|
21
|
+
- ``DELETE /trace/{surface}/day/{date}`` → drop one day of trace
|
|
22
|
+
- ``DELETE /trace/{surface}`` → drop all trace for a surface
|
|
23
|
+
- ``GET /backup`` → list v1-migration backup dirs (if any)
|
|
24
|
+
|
|
25
|
+
The legacy per-mode endpoints (``POST /doc/{layer}/{key}/update`` etc.)
|
|
26
|
+
are kept for the moment as thin wrappers that start a run and stream
|
|
27
|
+
its events — older clients keep working.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
from dataclasses import asdict
|
|
33
|
+
from datetime import date as date_cls
|
|
34
|
+
import json
|
|
35
|
+
import logging
|
|
36
|
+
import re
|
|
37
|
+
from typing import Literal
|
|
38
|
+
|
|
39
|
+
from fastapi import APIRouter, HTTPException
|
|
40
|
+
from fastapi.responses import StreamingResponse
|
|
41
|
+
from pydantic import BaseModel
|
|
42
|
+
|
|
43
|
+
from deeptutor.services.memory import (
|
|
44
|
+
L3_SLOTS,
|
|
45
|
+
SURFACES,
|
|
46
|
+
Surface,
|
|
47
|
+
get_memory_store,
|
|
48
|
+
paths,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
_ENTRY_ID_RE = re.compile(r"^m_[0-9A-HJKMNP-TV-Z]{26}$")
|
|
52
|
+
|
|
53
|
+
logger = logging.getLogger(__name__)
|
|
54
|
+
router = APIRouter()
|
|
55
|
+
|
|
56
|
+
Layer = Literal["L2", "L3"]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
# ── Helpers ──────────────────────────────────────────────────────────────
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _validate_doc_key(layer: Layer, key: str) -> None:
|
|
63
|
+
if layer == "L2" and key not in SURFACES:
|
|
64
|
+
raise HTTPException(status_code=404, detail=f"unknown surface {key!r}")
|
|
65
|
+
if layer == "L3" and key not in L3_SLOTS:
|
|
66
|
+
raise HTTPException(status_code=404, detail=f"unknown L3 slot {key!r}")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _validate_layer(layer: str) -> Layer:
|
|
70
|
+
if layer not in {"L2", "L3"}:
|
|
71
|
+
raise HTTPException(status_code=400, detail="layer must be L2 or L3")
|
|
72
|
+
return layer # type: ignore[return-value]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _validate_surface(surface: str) -> Surface:
|
|
76
|
+
if surface not in SURFACES:
|
|
77
|
+
raise HTTPException(status_code=404, detail=f"unknown surface {surface!r}")
|
|
78
|
+
return surface # type: ignore[return-value]
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
# ── Overview / list ──────────────────────────────────────────────────────
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@router.get("/overview")
|
|
85
|
+
async def get_overview():
|
|
86
|
+
store = get_memory_store()
|
|
87
|
+
rows = [asdict(r) for r in store.overview()]
|
|
88
|
+
backup_dir = paths.backup_root()
|
|
89
|
+
backups: list[str] = []
|
|
90
|
+
if backup_dir.exists():
|
|
91
|
+
backups = sorted(p.name for p in backup_dir.iterdir() if p.is_dir())
|
|
92
|
+
return {"docs": rows, "backups": backups}
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@router.get("/resolve_entry/{entry_id}")
|
|
96
|
+
async def resolve_entry(entry_id: str):
|
|
97
|
+
"""Find which L2 doc owns this entry id.
|
|
98
|
+
|
|
99
|
+
L3 docs cite L2 entries by their ``m_<ULID>`` entry id; the workbench
|
|
100
|
+
UI uses this resolver to turn an L3 footnote click into a navigation
|
|
101
|
+
to the right L2 surface + scroll-to anchor.
|
|
102
|
+
|
|
103
|
+
Scans the seven L2 mds in order; first hit wins. 404 if no L2 doc
|
|
104
|
+
contains the id (e.g. the entry was deleted or the id is stale).
|
|
105
|
+
"""
|
|
106
|
+
if not _ENTRY_ID_RE.match(entry_id):
|
|
107
|
+
raise HTTPException(status_code=400, detail="not a valid entry id")
|
|
108
|
+
from deeptutor.services.memory.document import parse
|
|
109
|
+
|
|
110
|
+
for surface in SURFACES:
|
|
111
|
+
path = paths.l2_file(surface)
|
|
112
|
+
if not path.exists():
|
|
113
|
+
continue
|
|
114
|
+
try:
|
|
115
|
+
doc = parse(path.read_text(encoding="utf-8"))
|
|
116
|
+
except Exception: # noqa: BLE001 — malformed L2 should not 500 the resolver
|
|
117
|
+
continue
|
|
118
|
+
for entry in doc.all_entries():
|
|
119
|
+
if entry.id == entry_id:
|
|
120
|
+
return {"layer": "L2", "key": surface, "entry_id": entry_id}
|
|
121
|
+
raise HTTPException(status_code=404, detail="entry not found in any L2 doc")
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@router.get("/backup")
|
|
125
|
+
async def list_backups():
|
|
126
|
+
backup_dir = paths.backup_root()
|
|
127
|
+
if not backup_dir.exists():
|
|
128
|
+
return {"backups": []}
|
|
129
|
+
out: list[dict] = []
|
|
130
|
+
for entry in sorted(backup_dir.iterdir()):
|
|
131
|
+
if entry.is_dir():
|
|
132
|
+
files = sorted(p.name for p in entry.iterdir())
|
|
133
|
+
out.append({"name": entry.name, "files": files})
|
|
134
|
+
return {"backups": out}
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
# ── Doc read / write / delete ────────────────────────────────────────────
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
@router.get("/doc/{layer}/{key}")
|
|
141
|
+
async def get_doc(layer: str, key: str):
|
|
142
|
+
lyr = _validate_layer(layer)
|
|
143
|
+
_validate_doc_key(lyr, key)
|
|
144
|
+
return {"layer": lyr, "key": key, "content": get_memory_store().read_raw(lyr, key)}
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class DocWriteRequest(BaseModel):
|
|
148
|
+
content: str
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
@router.put("/doc/{layer}/{key}")
|
|
152
|
+
async def put_doc(layer: str, key: str, payload: DocWriteRequest):
|
|
153
|
+
lyr = _validate_layer(layer)
|
|
154
|
+
_validate_doc_key(lyr, key)
|
|
155
|
+
await get_memory_store().overwrite_doc(lyr, key, payload.content)
|
|
156
|
+
return {"layer": lyr, "key": key, "saved": True}
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
@router.delete("/doc/{layer}/{key}/entry/{entry_id}")
|
|
160
|
+
async def delete_entry(layer: str, key: str, entry_id: str):
|
|
161
|
+
lyr = _validate_layer(layer)
|
|
162
|
+
_validate_doc_key(lyr, key)
|
|
163
|
+
ok = await get_memory_store().delete_entry(lyr, key, entry_id)
|
|
164
|
+
if not ok:
|
|
165
|
+
raise HTTPException(status_code=404, detail="entry not found")
|
|
166
|
+
return {"layer": lyr, "key": key, "deleted": entry_id}
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
@router.post("/doc/{layer}/{key}/reset")
|
|
170
|
+
async def reset_doc(layer: str, key: str):
|
|
171
|
+
"""Wipe the doc + its meta sidecar so the next update starts fresh.
|
|
172
|
+
|
|
173
|
+
Destructive — the caller has confirmed. After this returns the .md
|
|
174
|
+
file is gone *and* the ``seen_entity_refs`` set is cleared, so a
|
|
175
|
+
subsequent ``run_update`` re-ingests every L1 entity instead of
|
|
176
|
+
treating them as already-seen.
|
|
177
|
+
|
|
178
|
+
Refuses while a consolidator run is active on this doc; the caller
|
|
179
|
+
can cancel first.
|
|
180
|
+
"""
|
|
181
|
+
lyr = _validate_layer(layer)
|
|
182
|
+
_validate_doc_key(lyr, key)
|
|
183
|
+
|
|
184
|
+
from deeptutor.services.memory.consolidator.runs import get_run_manager
|
|
185
|
+
|
|
186
|
+
if get_run_manager().active_for(lyr, key) is not None:
|
|
187
|
+
raise HTTPException(
|
|
188
|
+
status_code=409,
|
|
189
|
+
detail="cancel the active run before resetting this doc",
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
from deeptutor.services.memory.consolidator import meta as meta_mod
|
|
193
|
+
|
|
194
|
+
doc_path = paths.l2_file(key) if lyr == "L2" else paths.l3_file(key) # type: ignore[arg-type]
|
|
195
|
+
meta_path = (
|
|
196
|
+
meta_mod.l2_meta_path(key) # type: ignore[arg-type]
|
|
197
|
+
if lyr == "L2"
|
|
198
|
+
else meta_mod.l3_meta_path(key) # type: ignore[arg-type]
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
removed_doc = False
|
|
202
|
+
removed_meta = False
|
|
203
|
+
try:
|
|
204
|
+
if doc_path.exists():
|
|
205
|
+
doc_path.unlink()
|
|
206
|
+
removed_doc = True
|
|
207
|
+
if meta_path.exists():
|
|
208
|
+
meta_path.unlink()
|
|
209
|
+
removed_meta = True
|
|
210
|
+
except OSError as exc:
|
|
211
|
+
raise HTTPException(status_code=500, detail=f"reset failed: {exc}") from exc
|
|
212
|
+
|
|
213
|
+
return {
|
|
214
|
+
"layer": lyr,
|
|
215
|
+
"key": key,
|
|
216
|
+
"reset": True,
|
|
217
|
+
"removed_doc": removed_doc,
|
|
218
|
+
"removed_meta": removed_meta,
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
# ── Doc update (SSE-streamed consolidator) ───────────────────────────────
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
class LLMSelectionPayload(BaseModel):
|
|
226
|
+
profile_id: str
|
|
227
|
+
model_id: str
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
class RunStartRequest(BaseModel):
|
|
231
|
+
layer: str
|
|
232
|
+
key: str
|
|
233
|
+
mode: Literal["update", "audit", "dedup", "merge"]
|
|
234
|
+
language: str = "en"
|
|
235
|
+
budget: int | None = None
|
|
236
|
+
iterations: int | None = None
|
|
237
|
+
llm_selection: LLMSelectionPayload | None = None
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _runner_for(req: RunStartRequest):
|
|
241
|
+
"""Return an ``async on_event → None`` runner for the requested mode."""
|
|
242
|
+
from deeptutor.services.memory.consolidator import (
|
|
243
|
+
run_audit,
|
|
244
|
+
run_dedup,
|
|
245
|
+
run_merge,
|
|
246
|
+
run_update,
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
selection = (
|
|
250
|
+
{"profile_id": req.llm_selection.profile_id, "model_id": req.llm_selection.model_id}
|
|
251
|
+
if req.llm_selection
|
|
252
|
+
else None
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
if req.mode == "update":
|
|
256
|
+
|
|
257
|
+
async def go(on_event):
|
|
258
|
+
await run_update(
|
|
259
|
+
req.layer,
|
|
260
|
+
req.key,
|
|
261
|
+
language=req.language,
|
|
262
|
+
budget=req.budget,
|
|
263
|
+
llm_selection=selection,
|
|
264
|
+
on_event=on_event,
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
return go
|
|
268
|
+
if req.mode == "audit":
|
|
269
|
+
|
|
270
|
+
async def go(on_event):
|
|
271
|
+
await run_audit(
|
|
272
|
+
req.layer,
|
|
273
|
+
req.key,
|
|
274
|
+
language=req.language,
|
|
275
|
+
budget=req.budget,
|
|
276
|
+
llm_selection=selection,
|
|
277
|
+
on_event=on_event,
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
return go
|
|
281
|
+
if req.mode == "dedup":
|
|
282
|
+
|
|
283
|
+
async def go(on_event):
|
|
284
|
+
await run_dedup(
|
|
285
|
+
req.layer,
|
|
286
|
+
req.key,
|
|
287
|
+
language=req.language,
|
|
288
|
+
iterations=req.iterations,
|
|
289
|
+
llm_selection=selection,
|
|
290
|
+
on_event=on_event,
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
return go
|
|
294
|
+
if req.mode == "merge":
|
|
295
|
+
|
|
296
|
+
async def go(on_event):
|
|
297
|
+
await run_merge(
|
|
298
|
+
req.layer,
|
|
299
|
+
req.key,
|
|
300
|
+
language=req.language,
|
|
301
|
+
on_event=on_event,
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
return go
|
|
305
|
+
raise HTTPException(status_code=400, detail=f"unknown mode {req.mode!r}")
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
@router.post("/runs/start")
|
|
309
|
+
async def start_run(req: RunStartRequest):
|
|
310
|
+
"""Start one consolidator mode and return a run handle.
|
|
311
|
+
|
|
312
|
+
The run survives client disconnects; reconnect via
|
|
313
|
+
``GET /runs/{id}/events?since=N``.
|
|
314
|
+
"""
|
|
315
|
+
lyr = _validate_layer(req.layer)
|
|
316
|
+
_validate_doc_key(lyr, req.key)
|
|
317
|
+
if lyr == "L3" and req.key == "preferences" and req.mode not in ("dedup", "merge"):
|
|
318
|
+
raise HTTPException(
|
|
319
|
+
status_code=405,
|
|
320
|
+
detail="preferences is written by the write_memory tool, not consolidated",
|
|
321
|
+
)
|
|
322
|
+
from deeptutor.services.memory.consolidator.runs import (
|
|
323
|
+
RunBusyError,
|
|
324
|
+
get_run_manager,
|
|
325
|
+
)
|
|
326
|
+
|
|
327
|
+
manager = get_run_manager()
|
|
328
|
+
runner = _runner_for(req)
|
|
329
|
+
selection = (
|
|
330
|
+
{"profile_id": req.llm_selection.profile_id, "model_id": req.llm_selection.model_id}
|
|
331
|
+
if req.llm_selection
|
|
332
|
+
else None
|
|
333
|
+
)
|
|
334
|
+
try:
|
|
335
|
+
run = await manager.start(
|
|
336
|
+
layer=lyr,
|
|
337
|
+
key=req.key,
|
|
338
|
+
mode=req.mode,
|
|
339
|
+
runner=runner,
|
|
340
|
+
params={
|
|
341
|
+
"budget": req.budget,
|
|
342
|
+
"iterations": req.iterations,
|
|
343
|
+
"language": req.language,
|
|
344
|
+
"llm_selection": selection,
|
|
345
|
+
},
|
|
346
|
+
language=req.language,
|
|
347
|
+
)
|
|
348
|
+
except RunBusyError as exc:
|
|
349
|
+
raise HTTPException(status_code=409, detail=str(exc))
|
|
350
|
+
return run.to_dict()
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
@router.get("/runs/{run_id}")
|
|
354
|
+
async def get_run(run_id: str):
|
|
355
|
+
from deeptutor.services.memory.consolidator.runs import get_run_manager
|
|
356
|
+
|
|
357
|
+
run = get_run_manager().get(run_id)
|
|
358
|
+
if run is None:
|
|
359
|
+
raise HTTPException(status_code=404, detail="unknown run_id")
|
|
360
|
+
return run.to_dict()
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
@router.post("/runs/{run_id}/cancel")
|
|
364
|
+
async def cancel_run(run_id: str):
|
|
365
|
+
from deeptutor.services.memory.consolidator.runs import get_run_manager
|
|
366
|
+
|
|
367
|
+
ok = await get_run_manager().cancel(run_id)
|
|
368
|
+
if not ok:
|
|
369
|
+
raise HTTPException(status_code=409, detail="not active")
|
|
370
|
+
return {"run_id": run_id, "cancelled": True}
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
@router.post("/runs/{run_id}/undo")
|
|
374
|
+
async def undo_run_edit(run_id: str):
|
|
375
|
+
from deeptutor.services.memory.consolidator.runs import (
|
|
376
|
+
RunBusyError,
|
|
377
|
+
get_run_manager,
|
|
378
|
+
)
|
|
379
|
+
|
|
380
|
+
manager = get_run_manager()
|
|
381
|
+
try:
|
|
382
|
+
event = await manager.undo_last(run_id)
|
|
383
|
+
except KeyError:
|
|
384
|
+
raise HTTPException(status_code=404, detail="unknown run_id")
|
|
385
|
+
except RunBusyError as exc:
|
|
386
|
+
raise HTTPException(status_code=409, detail=str(exc))
|
|
387
|
+
if event is None:
|
|
388
|
+
raise HTTPException(status_code=409, detail="nothing to undo")
|
|
389
|
+
run = manager.get(run_id)
|
|
390
|
+
return {
|
|
391
|
+
"run_id": run_id,
|
|
392
|
+
"undone": True,
|
|
393
|
+
"undo_count": len(run.undo_stack) if run else 0,
|
|
394
|
+
"event": {"seq": event.seq, "ts": event.ts, **event.payload},
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
@router.get("/runs")
|
|
399
|
+
async def list_runs(layer: str | None = None, key: str | None = None):
|
|
400
|
+
from deeptutor.services.memory.consolidator.runs import get_run_manager
|
|
401
|
+
|
|
402
|
+
lyr = _validate_layer(layer) if layer is not None else None
|
|
403
|
+
if lyr and key is not None:
|
|
404
|
+
_validate_doc_key(lyr, key)
|
|
405
|
+
runs = get_run_manager().list_for(layer=lyr, key=key)
|
|
406
|
+
return {"runs": [r.to_dict() for r in runs]}
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
@router.get("/runs/{run_id}/events")
|
|
410
|
+
async def stream_run_events(run_id: str, since: int = 0):
|
|
411
|
+
"""SSE-replay events from the next sequence cursor until the run ends.
|
|
412
|
+
|
|
413
|
+
Reconnecting after a refresh: pass ``last_seen_seq + 1``. The manager
|
|
414
|
+
replays the buffered tail, then blocks on new events until the run reaches
|
|
415
|
+
a terminal state.
|
|
416
|
+
"""
|
|
417
|
+
from deeptutor.services.memory.consolidator.runs import get_run_manager
|
|
418
|
+
|
|
419
|
+
manager = get_run_manager()
|
|
420
|
+
run = manager.get(run_id)
|
|
421
|
+
if run is None:
|
|
422
|
+
raise HTTPException(status_code=404, detail="unknown run_id")
|
|
423
|
+
|
|
424
|
+
async def producer():
|
|
425
|
+
cursor = max(0, since)
|
|
426
|
+
# Initial backfill (if any) is delivered as a batch up front.
|
|
427
|
+
while True:
|
|
428
|
+
events = await manager.wait_for_events(run, since=cursor)
|
|
429
|
+
for ev in events:
|
|
430
|
+
yield (
|
|
431
|
+
"data: "
|
|
432
|
+
+ json.dumps(
|
|
433
|
+
{"seq": ev.seq, "ts": ev.ts, **ev.payload},
|
|
434
|
+
ensure_ascii=False,
|
|
435
|
+
)
|
|
436
|
+
+ "\n\n"
|
|
437
|
+
)
|
|
438
|
+
cursor = max(cursor, ev.seq + 1)
|
|
439
|
+
if not run.active:
|
|
440
|
+
# Drain any final events that arrived between wait return and now.
|
|
441
|
+
final = [event for event in run.events if event.seq >= cursor]
|
|
442
|
+
for ev in final:
|
|
443
|
+
yield (
|
|
444
|
+
"data: "
|
|
445
|
+
+ json.dumps(
|
|
446
|
+
{"seq": ev.seq, "ts": ev.ts, **ev.payload},
|
|
447
|
+
ensure_ascii=False,
|
|
448
|
+
)
|
|
449
|
+
+ "\n\n"
|
|
450
|
+
)
|
|
451
|
+
break
|
|
452
|
+
|
|
453
|
+
return StreamingResponse(producer(), media_type="text/event-stream")
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
# ── Legacy per-mode endpoints (kept as thin wrappers over /runs/start) ──
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def _legacy_run_stream(req: RunStartRequest) -> StreamingResponse:
|
|
460
|
+
"""Old contract: POST /doc/{layer}/{key}/<mode> streams events inline."""
|
|
461
|
+
from deeptutor.services.memory.consolidator.runs import (
|
|
462
|
+
RunBusyError,
|
|
463
|
+
get_run_manager,
|
|
464
|
+
)
|
|
465
|
+
|
|
466
|
+
async def producer():
|
|
467
|
+
manager = get_run_manager()
|
|
468
|
+
runner = _runner_for(req)
|
|
469
|
+
selection = (
|
|
470
|
+
{"profile_id": req.llm_selection.profile_id, "model_id": req.llm_selection.model_id}
|
|
471
|
+
if req.llm_selection
|
|
472
|
+
else None
|
|
473
|
+
)
|
|
474
|
+
try:
|
|
475
|
+
run = await manager.start(
|
|
476
|
+
layer=req.layer,
|
|
477
|
+
key=req.key,
|
|
478
|
+
mode=req.mode,
|
|
479
|
+
runner=runner,
|
|
480
|
+
params={
|
|
481
|
+
"budget": req.budget,
|
|
482
|
+
"iterations": req.iterations,
|
|
483
|
+
"language": req.language,
|
|
484
|
+
"llm_selection": selection,
|
|
485
|
+
},
|
|
486
|
+
language=req.language,
|
|
487
|
+
)
|
|
488
|
+
except RunBusyError as exc:
|
|
489
|
+
yield (
|
|
490
|
+
"data: "
|
|
491
|
+
+ json.dumps({"stage": "error", "message": str(exc)}, ensure_ascii=False)
|
|
492
|
+
+ "\n\n"
|
|
493
|
+
)
|
|
494
|
+
return
|
|
495
|
+
cursor = 0
|
|
496
|
+
while True:
|
|
497
|
+
events = await manager.wait_for_events(run, since=cursor)
|
|
498
|
+
for ev in events:
|
|
499
|
+
yield (
|
|
500
|
+
"data: "
|
|
501
|
+
+ json.dumps({**ev.payload, "seq": ev.seq}, ensure_ascii=False)
|
|
502
|
+
+ "\n\n"
|
|
503
|
+
)
|
|
504
|
+
cursor = ev.seq + 1
|
|
505
|
+
if not run.active:
|
|
506
|
+
break
|
|
507
|
+
|
|
508
|
+
return StreamingResponse(producer(), media_type="text/event-stream")
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
class UpdateRequest(BaseModel):
|
|
512
|
+
language: str = "en"
|
|
513
|
+
budget: int | None = None
|
|
514
|
+
llm_selection: LLMSelectionPayload | None = None
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
class AuditRequest(BaseModel):
|
|
518
|
+
language: str = "en"
|
|
519
|
+
budget: int | None = None
|
|
520
|
+
llm_selection: LLMSelectionPayload | None = None
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
class DedupRequest(BaseModel):
|
|
524
|
+
language: str = "en"
|
|
525
|
+
iterations: int | None = None
|
|
526
|
+
llm_selection: LLMSelectionPayload | None = None
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
@router.post("/doc/{layer}/{key}/update")
|
|
530
|
+
async def update_doc(layer: str, key: str, payload: UpdateRequest | None = None):
|
|
531
|
+
lyr = _validate_layer(layer)
|
|
532
|
+
_validate_doc_key(lyr, key)
|
|
533
|
+
req = RunStartRequest(
|
|
534
|
+
layer=lyr,
|
|
535
|
+
key=key,
|
|
536
|
+
mode="update",
|
|
537
|
+
language=(payload.language if payload else "en") or "en",
|
|
538
|
+
budget=payload.budget if payload else None,
|
|
539
|
+
llm_selection=payload.llm_selection if payload else None,
|
|
540
|
+
)
|
|
541
|
+
return _legacy_run_stream(req)
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
@router.post("/doc/{layer}/{key}/audit")
|
|
545
|
+
async def audit_doc(layer: str, key: str, payload: AuditRequest | None = None):
|
|
546
|
+
lyr = _validate_layer(layer)
|
|
547
|
+
_validate_doc_key(lyr, key)
|
|
548
|
+
req = RunStartRequest(
|
|
549
|
+
layer=lyr,
|
|
550
|
+
key=key,
|
|
551
|
+
mode="audit",
|
|
552
|
+
language=(payload.language if payload else "en") or "en",
|
|
553
|
+
budget=payload.budget if payload else None,
|
|
554
|
+
llm_selection=payload.llm_selection if payload else None,
|
|
555
|
+
)
|
|
556
|
+
return _legacy_run_stream(req)
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
@router.post("/doc/{layer}/{key}/dedup")
|
|
560
|
+
async def dedup_doc(layer: str, key: str, payload: DedupRequest | None = None):
|
|
561
|
+
lyr = _validate_layer(layer)
|
|
562
|
+
_validate_doc_key(lyr, key)
|
|
563
|
+
req = RunStartRequest(
|
|
564
|
+
layer=lyr,
|
|
565
|
+
key=key,
|
|
566
|
+
mode="dedup",
|
|
567
|
+
language=(payload.language if payload else "en") or "en",
|
|
568
|
+
iterations=payload.iterations if payload else None,
|
|
569
|
+
llm_selection=payload.llm_selection if payload else None,
|
|
570
|
+
)
|
|
571
|
+
return _legacy_run_stream(req)
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
@router.get("/doc/{layer}/{key}/lines")
|
|
575
|
+
async def get_doc_lines(layer: str, key: str):
|
|
576
|
+
"""Return the line-numbered, footnote-stripped view of a doc.
|
|
577
|
+
|
|
578
|
+
Used by the workbench's "show line numbers" toggle so the same line
|
|
579
|
+
indices the audit/dedup LLMs see are visible to the user. Footnote
|
|
580
|
+
block is omitted because edit ops never reference it directly.
|
|
581
|
+
"""
|
|
582
|
+
lyr = _validate_layer(layer)
|
|
583
|
+
_validate_doc_key(lyr, key)
|
|
584
|
+
from deeptutor.services.memory import paths
|
|
585
|
+
from deeptutor.services.memory.consolidator.line_doc import render_view
|
|
586
|
+
from deeptutor.services.memory.document import Document, parse
|
|
587
|
+
|
|
588
|
+
path = paths.l2_file(key) if lyr == "L2" else paths.l3_file(key) # type: ignore[arg-type]
|
|
589
|
+
doc = (
|
|
590
|
+
parse(path.read_text(encoding="utf-8"))
|
|
591
|
+
if path.exists()
|
|
592
|
+
else Document(title=_default_title(lyr, key))
|
|
593
|
+
)
|
|
594
|
+
view = render_view(doc)
|
|
595
|
+
return {
|
|
596
|
+
"layer": lyr,
|
|
597
|
+
"key": key,
|
|
598
|
+
"lines": [
|
|
599
|
+
{
|
|
600
|
+
"number": line.number,
|
|
601
|
+
"kind": line.kind,
|
|
602
|
+
"text": line.text,
|
|
603
|
+
"entry_id": line.entry_id,
|
|
604
|
+
"section": line.section,
|
|
605
|
+
}
|
|
606
|
+
for line in view.lines
|
|
607
|
+
],
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
# ── Settings ────────────────────────────────────────────────────────────
|
|
612
|
+
|
|
613
|
+
|
|
614
|
+
@router.get("/settings")
|
|
615
|
+
async def get_memory_settings_endpoint():
|
|
616
|
+
"""Return the current ``memory:`` subtree (defaults merged in)."""
|
|
617
|
+
from deeptutor.services.memory.settings import memory_settings_dict
|
|
618
|
+
|
|
619
|
+
return memory_settings_dict()
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
@router.put("/settings")
|
|
623
|
+
async def put_memory_settings(payload: dict):
|
|
624
|
+
"""Merge the payload into the ``memory:`` subtree and persist."""
|
|
625
|
+
from deeptutor.services.memory.settings import (
|
|
626
|
+
memory_settings_dict,
|
|
627
|
+
save_memory_settings,
|
|
628
|
+
)
|
|
629
|
+
|
|
630
|
+
save_memory_settings(payload)
|
|
631
|
+
return memory_settings_dict()
|
|
632
|
+
|
|
633
|
+
|
|
634
|
+
def _default_title(layer: str, key: str) -> str:
|
|
635
|
+
if layer == "L2":
|
|
636
|
+
return f"{key} memory"
|
|
637
|
+
return {
|
|
638
|
+
"recent": "Recent summary",
|
|
639
|
+
"profile": "User profile",
|
|
640
|
+
"scope": "Knowledge scope",
|
|
641
|
+
"preferences": "Preferences",
|
|
642
|
+
}.get(key, f"{key} memory")
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
class ApplyOpsRequest(BaseModel):
|
|
646
|
+
ops: list[dict]
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
@router.post("/doc/{layer}/{key}/apply")
|
|
650
|
+
async def apply_doc_ops(layer: str, key: str, payload: ApplyOpsRequest):
|
|
651
|
+
"""Commit a list of previously-previewed ops to a doc atomically."""
|
|
652
|
+
lyr = _validate_layer(layer)
|
|
653
|
+
_validate_doc_key(lyr, key)
|
|
654
|
+
if lyr == "L3" and key == "preferences":
|
|
655
|
+
raise HTTPException(
|
|
656
|
+
status_code=405,
|
|
657
|
+
detail="preferences is written by the write_memory tool, not consolidated",
|
|
658
|
+
)
|
|
659
|
+
if not payload.ops:
|
|
660
|
+
return {"accepted": True, "reason": "no ops to apply", "results": []}
|
|
661
|
+
|
|
662
|
+
report = await get_memory_store().apply_ops_payload(lyr, key, payload.ops)
|
|
663
|
+
return {
|
|
664
|
+
"accepted": report.accepted,
|
|
665
|
+
"reason": report.reason,
|
|
666
|
+
"results": [
|
|
667
|
+
{
|
|
668
|
+
"status": r.status,
|
|
669
|
+
"entry_id": r.entry_id,
|
|
670
|
+
"detail": r.detail,
|
|
671
|
+
}
|
|
672
|
+
for r in report.results
|
|
673
|
+
],
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
# ── Trace browser ────────────────────────────────────────────────────────
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
@router.get("/trace/{surface}")
|
|
681
|
+
async def get_trace(surface: str, limit: int = 200, offset: int = 0):
|
|
682
|
+
surf = _validate_surface(surface)
|
|
683
|
+
from deeptutor.services.memory.trace import iter_since
|
|
684
|
+
|
|
685
|
+
events = []
|
|
686
|
+
for i, event in enumerate(iter_since(surf)):
|
|
687
|
+
if i < offset:
|
|
688
|
+
continue
|
|
689
|
+
if len(events) >= max(1, min(limit, 1000)):
|
|
690
|
+
break
|
|
691
|
+
events.append(asdict(event))
|
|
692
|
+
return {"surface": surf, "events": events, "offset": offset, "limit": limit}
|
|
693
|
+
|
|
694
|
+
|
|
695
|
+
@router.delete("/trace/{surface}")
|
|
696
|
+
async def clear_trace(surface: str):
|
|
697
|
+
surf = _validate_surface(surface)
|
|
698
|
+
removed = 0
|
|
699
|
+
for path in paths.trace_dir(surf).glob("*.jsonl"):
|
|
700
|
+
try:
|
|
701
|
+
path.unlink()
|
|
702
|
+
removed += 1
|
|
703
|
+
except OSError:
|
|
704
|
+
continue
|
|
705
|
+
return {"surface": surf, "removed_files": removed}
|
|
706
|
+
|
|
707
|
+
|
|
708
|
+
@router.delete("/trace/{surface}/day/{day}")
|
|
709
|
+
async def clear_trace_day(surface: str, day: str):
|
|
710
|
+
surf = _validate_surface(surface)
|
|
711
|
+
try:
|
|
712
|
+
parsed = date_cls.fromisoformat(day)
|
|
713
|
+
except ValueError:
|
|
714
|
+
raise HTTPException(status_code=400, detail="day must be YYYY-MM-DD")
|
|
715
|
+
path = paths.trace_file(surf, parsed)
|
|
716
|
+
if not path.exists():
|
|
717
|
+
raise HTTPException(status_code=404, detail="no trace for that day")
|
|
718
|
+
try:
|
|
719
|
+
path.unlink()
|
|
720
|
+
except OSError as exc:
|
|
721
|
+
raise HTTPException(status_code=500, detail=str(exc))
|
|
722
|
+
return {"surface": surf, "day": day, "deleted": True}
|
|
723
|
+
|
|
724
|
+
|
|
725
|
+
# ── Snapshot (L1 workspace mirror) ───────────────────────────────────────
|
|
726
|
+
|
|
727
|
+
|
|
728
|
+
@router.get("/snapshot/{surface}")
|
|
729
|
+
async def get_snapshot(surface: str):
|
|
730
|
+
"""Return the current entity list for ``surface`` from workspace.
|
|
731
|
+
|
|
732
|
+
Snapshot is always derived live from workspace at call time. The response
|
|
733
|
+
also includes ``pending_changes`` — the diff vs the last persisted state.
|
|
734
|
+
Refresh commits these pending changes into ``changes.jsonl``.
|
|
735
|
+
"""
|
|
736
|
+
surf = _validate_surface(surface)
|
|
737
|
+
from deeptutor.services.memory import snapshot as snap
|
|
738
|
+
|
|
739
|
+
entities = snap.read_snapshot(surf)
|
|
740
|
+
pending = snap.pending_changes(surf, entities)
|
|
741
|
+
state = snap.current_state(surf)
|
|
742
|
+
return {
|
|
743
|
+
"surface": surf,
|
|
744
|
+
"entities": [e.to_dict() for e in entities],
|
|
745
|
+
"last_refresh": state.get("last_refresh"),
|
|
746
|
+
"pending_changes": [c.to_dict() for c in pending],
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
|
|
750
|
+
@router.post("/snapshot/{surface}/refresh")
|
|
751
|
+
async def refresh_snapshot(surface: str):
|
|
752
|
+
"""Reconcile persisted state with current workspace; record diffs."""
|
|
753
|
+
surf = _validate_surface(surface)
|
|
754
|
+
from deeptutor.services.memory import snapshot as snap
|
|
755
|
+
|
|
756
|
+
changes = snap.refresh_snapshot(surf)
|
|
757
|
+
state = snap.current_state(surf)
|
|
758
|
+
return {
|
|
759
|
+
"surface": surf,
|
|
760
|
+
"changes": [c.to_dict() for c in changes],
|
|
761
|
+
"last_refresh": state.get("last_refresh"),
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
|
|
765
|
+
@router.get("/snapshot/{surface}/changes")
|
|
766
|
+
async def get_changes(surface: str, limit: int = 200, offset: int = 0):
|
|
767
|
+
surf = _validate_surface(surface)
|
|
768
|
+
from deeptutor.services.memory import snapshot as snap
|
|
769
|
+
|
|
770
|
+
entries = snap.read_changes(surf, limit=limit, offset=offset)
|
|
771
|
+
return {
|
|
772
|
+
"surface": surf,
|
|
773
|
+
"changes": [c.to_dict() for c in entries],
|
|
774
|
+
"limit": limit,
|
|
775
|
+
"offset": offset,
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
|
|
779
|
+
@router.delete("/snapshot/{surface}/changes")
|
|
780
|
+
async def clear_snapshot_changes(surface: str):
|
|
781
|
+
surf = _validate_surface(surface)
|
|
782
|
+
from deeptutor.services.memory import snapshot as snap
|
|
783
|
+
|
|
784
|
+
snap.clear_changes(surf)
|
|
785
|
+
return {"surface": surf, "cleared": True}
|