mapwright-agent 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
mapwright/track.py ADDED
@@ -0,0 +1,715 @@
1
+ """The project store: file versions, steps, annotations and their verification.
2
+
3
+ One directory per project under ~/.mapwright/projects/<key>/:
4
+
5
+ journal.jsonl append-only log: prompts, tool calls, steps, todos, claims, notes, links
6
+ steps/NNNN.json.gz each step: the file index (path -> content hash), the model, what changed
7
+ blobs/ab/<sha>.gz every version of every tracked text file, stored once by hash
8
+ cache.json.gz per-file analysis results, so a new process starts warm
9
+ meta.json root, last change time
10
+
11
+ Several processes share it (the daemon, the MCP server an agent runs, hook calls, the CLI), so
12
+ writes take a file lock. Analysis never trusts the agent: what changed is computed from the
13
+ files, and every claim the agent makes is checked against the code and marked verified or not.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import contextlib
19
+ import difflib
20
+ import gzip
21
+ import hashlib
22
+ import json
23
+ import os
24
+ import re
25
+ import threading
26
+ import time
27
+ from collections import Counter
28
+ from pathlib import Path
29
+
30
+ from . import analyze as A
31
+ from . import concepts as C
32
+ from . import config, infra, noise, titles
33
+ from . import ignore as ignore_mod
34
+ from .redact import redact
35
+
36
+ try:
37
+ import fcntl
38
+ except ImportError: # Windows: in-process locking only
39
+ fcntl = None
40
+
41
+ SCHEMA = 5
42
+ _KEY = re.compile(r"^[A-Za-z0-9_.-]{1,120}$")
43
+ _LOCKS: dict[str, threading.RLock] = {}
44
+ COALESCE_SECONDS = 2.0 # auto steps closer together than this are merged (the later turn end picks up the rest)
45
+ SESSION_STALE_SECONDS = 600 # a session with no event for this long is not "in a turn" any more
46
+ MAX_DEFER_SECONDS = 300 # how long a finished turn waits for other sessions still working
47
+ JOURNAL_CAP_BYTES = 20 * 1024 * 1024 # past this, hook events (prompt/tool/turn_end) stop being journaled
48
+ MAX_TOOL_EVENTS_PER_STEP = 500
49
+
50
+
51
+ def project_key(root: str) -> str:
52
+ slug = re.sub(r"[^A-Za-z0-9_.-]+", "-", os.path.basename(root.rstrip("/")) or "project").strip("-.")[:40] or "project"
53
+ return f"{slug}-{hashlib.sha1(root.encode()).hexdigest()[:8]}"
54
+
55
+
56
+ def _gz_write(path: Path, data: bytes) -> None:
57
+ path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
58
+ tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp")
59
+ with gzip.open(tmp, "wb", compresslevel=5) as fh:
60
+ fh.write(data)
61
+ os.chmod(tmp, 0o600)
62
+ os.replace(tmp, path)
63
+
64
+
65
+ def _gz_read(path: Path) -> bytes | None:
66
+ try:
67
+ with gzip.open(path, "rb") as fh:
68
+ return fh.read()
69
+ except (OSError, EOFError):
70
+ return None
71
+
72
+
73
+ _TITLE_CHECKS = [ # words in a step title -> what must have changed for the title to be plausible
74
+ (r"\btests?\b|\bspecs?\b", "test", "a test file"),
75
+ (r"\bmigrations?\b", "migration", "a migration"),
76
+ (r"\b(endpoint|route|api)s?\b", "route", "an HTTP route"),
77
+ (r"\b(schema|table|column|model)s?\b", "data", "a table or model"),
78
+ (r"\b(docs?|readme|documentation)\b", "docs", "a documentation file"),
79
+ (r"\b(docker|compose|terraform|deploy|infra|ci)\b", "infra", "an infrastructure file"),
80
+ ]
81
+
82
+
83
+ class Project:
84
+ def __init__(self, root: str):
85
+ self.root = os.path.realpath(A.find_root(os.path.abspath(root)))
86
+ self.key = project_key(self.root)
87
+ self.dir = config.home() / "projects" / self.key
88
+ self.dir.mkdir(mode=0o700, parents=True, exist_ok=True)
89
+ self.name = os.path.basename(self.root) or self.root
90
+ self.cache: dict[str, dict] = {}
91
+ self.index: dict[str, str] = {}
92
+ self.model: dict | None = None
93
+ self.concepts: dict = {}
94
+ self.infra: dict = {"nodes": [], "edges": []}
95
+ self.truncated = False
96
+ self.truncated_info: dict | None = None
97
+ self._journal_cache: tuple[int, list[dict]] = (-1, [])
98
+ self._lock = _LOCKS.setdefault(self.key, threading.RLock())
99
+ self._held = 0
100
+ self._step_cache: dict[int, dict] = {}
101
+ self._rechecked: dict[int, str] = {}
102
+ self.version = 0
103
+ meta = config.read_json(self.dir / "meta.json", {})
104
+ if not meta:
105
+ config.write_json(self.dir / "meta.json", {"schema": SCHEMA, "root": self.root, "key": self.key, "created": time.time()})
106
+ self._load_cache()
107
+ config.register(self.key, self.root)
108
+
109
+ # ------------------------------------------------------------------ storage helpers
110
+ @contextlib.contextmanager
111
+ def locked(self):
112
+ """Exclusive across threads (RLock) and processes (flock). Re-entrant: only the outermost
113
+ level takes the file lock, since a second flock on a new handle would wait for itself."""
114
+ with self._lock:
115
+ fh = open(self.dir / "lock", "a") if fcntl and self._held == 0 else None
116
+ self._held += 1
117
+ try:
118
+ if fh:
119
+ fcntl.flock(fh, fcntl.LOCK_EX)
120
+ yield
121
+ finally:
122
+ self._held -= 1
123
+ if fh:
124
+ fcntl.flock(fh, fcntl.LOCK_UN)
125
+ fh.close()
126
+
127
+ def _blob_path(self, sha: str) -> Path:
128
+ return self.dir / "blobs" / sha[:2] / f"{sha}.gz"
129
+
130
+ def put_blob(self, sha: str, text: str) -> None:
131
+ p = self._blob_path(sha)
132
+ if not p.exists():
133
+ _gz_write(p, text.encode("utf-8", "replace"))
134
+
135
+ def blob(self, sha: str | None) -> str | None:
136
+ if not sha or sha.startswith("secret:") or not re.match(r"^[0-9a-f]{16}$", sha):
137
+ return None
138
+ raw = _gz_read(self._blob_path(sha))
139
+ return raw.decode("utf-8", "replace") if raw is not None else None
140
+
141
+ def _load_cache(self) -> None:
142
+ raw = _gz_read(self.dir / "cache.json.gz")
143
+ try:
144
+ data = json.loads(raw) if raw else {}
145
+ except ValueError:
146
+ data = {}
147
+ if data.get("schema") == SCHEMA:
148
+ self.cache = data.get("files", {})
149
+
150
+ def _save_cache(self) -> None:
151
+ _gz_write(self.dir / "cache.json.gz", json.dumps({"schema": SCHEMA, "files": self.cache}, separators=(",", ":")).encode())
152
+
153
+ def journal(self) -> list[dict]:
154
+ p = self.dir / "journal.jsonl"
155
+ try:
156
+ size = p.stat().st_size
157
+ except OSError:
158
+ return []
159
+ if size == self._journal_cache[0]:
160
+ return self._journal_cache[1]
161
+ out = []
162
+ with open(p, encoding="utf-8") as fh:
163
+ for line in fh:
164
+ try:
165
+ ev = json.loads(line)
166
+ except ValueError:
167
+ continue
168
+ if isinstance(ev, dict) and "t" in ev:
169
+ out.append(ev)
170
+ self._journal_cache = (size, out)
171
+ return out
172
+
173
+ def append(self, ev: dict) -> dict:
174
+ with self.locked():
175
+ evs = self.journal()
176
+ ev = {"id": (evs[-1]["id"] + 1) if evs else 1, "ts": time.time(), **ev}
177
+ fd = os.open(self.dir / "journal.jsonl", os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600)
178
+ with os.fdopen(fd, "a", encoding="utf-8") as fh:
179
+ fh.write(json.dumps(ev, separators=(",", ":")) + "\n")
180
+ self.version += 1
181
+ return ev
182
+
183
+ # ------------------------------------------------------------------ scanning & analysis
184
+ def refresh(self) -> list[str]:
185
+ """Re-read whatever changed on disk. Returns the paths whose content changed."""
186
+ with self.locked():
187
+ st = config.settings()
188
+ matcher = ignore_mod.Matcher(self.root, st.get("ignore") or [], st.get("respect_gitignore", True))
189
+ info: dict = {}
190
+ stats, self.truncated = A.scan(self.root, matcher, st.get("max_files"), info)
191
+ self.truncated_info = info if self.truncated else None
192
+ changed = []
193
+ new_cache = {}
194
+ for rel, sig in stats.items():
195
+ old = self.cache.get(rel)
196
+ if old and tuple(old["stat"]) == tuple(sig):
197
+ new_cache[rel] = old
198
+ continue
199
+ entry = self._analyze(rel, sig)
200
+ if not old or old["sha"] != entry["sha"]:
201
+ changed.append(rel)
202
+ new_cache[rel] = entry
203
+ removed = [r for r in self.cache if r not in stats]
204
+ changed += removed
205
+ if changed or self.model is None:
206
+ self.cache = new_cache
207
+ self.index = {r: e["sha"] for r, e in self.cache.items()}
208
+ self._build()
209
+ if changed:
210
+ self._save_cache()
211
+ meta = config.read_json(self.dir / "meta.json", {})
212
+ meta["last_change"] = time.time()
213
+ config.write_json(self.dir / "meta.json", meta)
214
+ self.version += 1
215
+ if not self.steps():
216
+ self._snapshot("baseline (before tracking started)", "", "baseline")
217
+ return changed
218
+
219
+ def _analyze(self, rel: str, sig) -> dict:
220
+ if A._secretish(rel):
221
+ return {"stat": list(sig), "sha": f"secret:{sig[0]}:{sig[1]}", "part": None, "concepts": {}, "secret": True}
222
+ text = A.read_text(self.root, rel)
223
+ if text is None:
224
+ return {"stat": list(sig), "sha": f"secret:{sig[0]}:{sig[1]}", "part": None, "concepts": {}, "binary": True}
225
+ sha = A._sha(text)
226
+ self.put_blob(sha, text)
227
+ part = A.analyze_file(rel, text) if os.path.splitext(rel)[1].lower() in A.CODE_EXT else None
228
+ concepts = C.detect_file(rel, text, part) if part else {}
229
+ return {"stat": list(sig), "sha": sha, "part": part, "concepts": concepts, "lines": text.count("\n") + 1}
230
+
231
+ def _build(self) -> None:
232
+ parts = {r: e["part"] for r, e in self.cache.items() if e.get("part")}
233
+ self.model = A.link(parts)
234
+ agg: dict[str, list[dict]] = {}
235
+ for rel, e in sorted(self.cache.items()):
236
+ for cid, evs in (e.get("concepts") or {}).items():
237
+ for ev in evs:
238
+ agg.setdefault(cid, []).append({"file": rel, **ev})
239
+ self.concepts = {cid: evs[:12] for cid, evs in agg.items()}
240
+ self.infra = infra.detect(self.root, self.index, self.text, self.model["modules"])
241
+ self.model["concepts"] = {cid: [{"file": e["file"], "line": e["line"], "symbol": e["symbol"]} for e in evs[:6]]
242
+ for cid, evs in self.concepts.items()}
243
+ self.model["infra"] = self.infra
244
+
245
+ def ensure(self) -> "Project":
246
+ if self.model is None:
247
+ self.refresh()
248
+ return self
249
+
250
+ def text(self, rel: str, step: int | None = None) -> str | None:
251
+ """Text of a tracked file, now or at a step. Secret-looking files are never returned."""
252
+ if A._secretish(rel):
253
+ return None
254
+ idx = self.index if step is None else self.step_files(step)
255
+ return self.blob(idx.get(rel))
256
+
257
+ # ------------------------------------------------------------------ steps
258
+ def steps(self) -> list[dict]:
259
+ return [e for e in self.journal() if e["t"] == "step"]
260
+
261
+ def load_step(self, n: int) -> dict | None:
262
+ if n in self._step_cache:
263
+ return self._step_cache[n]
264
+ raw = _gz_read(self.dir / "steps" / f"{n:04d}.json.gz")
265
+ if raw is None:
266
+ return None
267
+ data = json.loads(raw)
268
+ if len(self._step_cache) > 6:
269
+ self._step_cache.pop(next(iter(self._step_cache)))
270
+ self._step_cache[n] = data
271
+ return data
272
+
273
+ def step_files(self, n: int) -> dict:
274
+ s = self.load_step(n)
275
+ return s["files"] if s else {}
276
+
277
+ def changes_between(self, a: dict, b: dict, counts: bool = True) -> list[dict]:
278
+ out = []
279
+ for rel in sorted(set(a) | set(b)):
280
+ if a.get(rel) == b.get(rel):
281
+ continue
282
+ kind = "added" if rel not in a else "deleted" if rel not in b else "modified"
283
+ ch = {"path": rel, "kind": kind}
284
+ if counts:
285
+ ta, tb = self.blob(a.get(rel)), self.blob(b.get(rel))
286
+ if ta is None and tb is None:
287
+ ch["note"] = "secret-looking or binary: content not read"
288
+ else:
289
+ sm = difflib.SequenceMatcher(None, (ta or "").splitlines(), (tb or "").splitlines(), autojunk=False)
290
+ add = rem = 0
291
+ for op, i1, i2, j1, j2 in sm.get_opcodes():
292
+ if op in ("replace", "delete"):
293
+ rem += i2 - i1
294
+ if op in ("replace", "insert"):
295
+ add += j2 - j1
296
+ ch["added"], ch["removed"] = add, rem
297
+ out.append(ch)
298
+ return out
299
+
300
+ def pending(self, counts: bool = True) -> list[dict]:
301
+ steps = self.steps()
302
+ last = self.step_files(steps[-1]["n"]) if steps else {}
303
+ return self.changes_between(last, self.index, counts)
304
+
305
+ def _title_check(self, title: str, changes: list[dict], marks: dict) -> str:
306
+ paths = [c["path"] for c in changes]
307
+ low = title.lower()
308
+ for pat, what, words in _TITLE_CHECKS:
309
+ if not re.search(pat, low):
310
+ continue
311
+ ok = {
312
+ "test": any(A.layer_of(p) == "test" for p in paths),
313
+ "migration": any("migration" in p.lower() or "alembic" in p.lower() for p in paths),
314
+ "route": any(r["file"] in paths for r in (self.model or {}).get("routes", [])),
315
+ "data": bool(marks.get("tables")) or any(A.layer_of(p) == "data" or p.endswith((".sql", ".prisma")) for p in paths),
316
+ "docs": any(p.lower().endswith((".md", ".rst", ".txt")) or p.lower().startswith("docs/") for p in paths),
317
+ "infra": any(re.search(r"docker|compose|\.tf$|\.github/|k8s|helm|deploy", p.lower()) for p in paths),
318
+ }[what]
319
+ if not ok:
320
+ return f"title mentions {words}, but no such file changed in this step"
321
+ return "ok"
322
+
323
+ def _model_at(self, files: dict) -> dict:
324
+ parts = {}
325
+ for rel, sha in files.items():
326
+ if os.path.splitext(rel)[1].lower() in A.CODE_EXT:
327
+ cached = self.cache.get(rel)
328
+ if cached and cached.get("sha") == sha and cached.get("part"):
329
+ parts[rel] = cached["part"]
330
+ else:
331
+ text = self.blob(sha)
332
+ if text is not None:
333
+ parts[rel] = A.analyze_file(rel, text)
334
+ return A.link({k: v for k, v in parts.items() if v})
335
+
336
+ def title_check(self, step: dict) -> str:
337
+ """A stored title warning, re-checked with the current analysis (which may have learned to see
338
+ what it missed then, e.g. tables created in SQL strings). Cached per process."""
339
+ stored = step.get("title_check", "ok")
340
+ if stored == "ok" or step["n"] == 0:
341
+ return stored
342
+ if step["n"] not in self._rechecked:
343
+ data, prev = self.load_step(step["n"]), self.load_step(step["n"] - 1)
344
+ if not data or not prev:
345
+ self._rechecked[step["n"]] = stored
346
+ else:
347
+ saved = self.model
348
+ try:
349
+ now_model = self._model_at(data["files"])
350
+ self.model = now_model # _title_check reads routes from self.model
351
+ marks = A.diff_models(self._model_at(prev["files"]), now_model)
352
+ self._rechecked[step["n"]] = self._title_check(step["title"], data["changes"], marks)
353
+ finally:
354
+ self.model = saved
355
+ return self._rechecked[step["n"]]
356
+
357
+ def _snapshot(self, title: str, intent: str, source: str, prompt: str | None = None) -> dict:
358
+ steps = self.steps()
359
+ n = (steps[-1]["n"] + 1) if steps else 0
360
+ prev = self.load_step(steps[-1]["n"]) if steps else None
361
+ all_changes = self.changes_between(prev["files"] if prev else {}, self.index) if prev else []
362
+ real, noisy = noise.split(all_changes)
363
+ changes = real + noisy # noisy ones stay tracked, flagged with `noise`
364
+ marks = A.diff_models(prev["model"] if prev else None, self.model)
365
+ delta = {k: dict(Counter(v.values())) for k, v in marks.items() if v}
366
+ check = self._title_check(title, real, marks) if source == "agent" else "ok"
367
+ summary = {"files": len(real), "added": sum(c.get("added", 0) for c in real),
368
+ "removed": sum(c.get("removed", 0) for c in real), "delta": delta,
369
+ "modules": len(self.model["modules"]), "functions": len(self.model["functions"]),
370
+ "classes": len(self.model["classes"]), "tables": len(self.model["tables"]), "routes": len(self.model["routes"])}
371
+ if noisy:
372
+ summary["noise"] = len(noisy)
373
+ evs = self.journal()
374
+ since = steps[-1]["id"] if steps else 0
375
+ tools = Counter(e.get("tool") for e in evs if e["id"] > since and e["t"] == "tool")
376
+ if prompt is None:
377
+ prompt = next((e.get("text") for e in reversed(evs) if e["id"] > since and e["t"] == "prompt"), "")
378
+ data = {"n": n, "title": title, "intent": intent, "source": source, "ts": time.time(), "files": dict(self.index),
379
+ "model": self.model, "changes": changes, "summary": summary, "marks": marks, "title_check": check,
380
+ "tools": dict(tools), "prompt": prompt}
381
+ _gz_write(self.dir / "steps" / f"{n:04d}.json.gz", json.dumps(data, separators=(",", ":")).encode())
382
+ self._step_cache[n] = data
383
+ keep = ("path", "kind", "added", "removed", "note")
384
+ logged = [{k: c[k] for k in keep if k in c} for c in real][:200]
385
+ if noisy:
386
+ logged.append({"path": noise.collapse_line(noisy), "kind": "noise", "noise": len(noisy)})
387
+ self.append({"t": "step", "n": n, "title": title[:120], "intent": intent[:300], "source": source,
388
+ "summary": summary, "title_check": check, "prompt": prompt[:200], "changes": logged})
389
+ return data
390
+
391
+ def step(self, title: str, intent: str = "", source: str = "agent") -> dict:
392
+ """Close a step: snapshot the files and the model as they are now. The agent supplies only the
393
+ title; the changed files and the design delta are computed, not taken from the agent."""
394
+ title = " ".join((title or "").split())[:120] or "untitled step"
395
+ with self.locked():
396
+ self.refresh()
397
+ data = self._snapshot(title, " ".join((intent or "").split())[:300], source)
398
+ return data
399
+
400
+ def _sessions(self, evs: list[dict], since: int) -> dict[str, dict]:
401
+ """Per session since the last step: id/ts of its last activity (prompt or tool) and last turn end."""
402
+ out: dict[str, dict] = {}
403
+ for e in evs:
404
+ if e["id"] <= since or e["t"] not in ("prompt", "tool", "turn_end"):
405
+ continue
406
+ st = out.setdefault(str(e.get("session") or ""), {"act": 0, "act_ts": 0.0, "end": 0, "end_ts": 0.0})
407
+ if e["t"] == "turn_end":
408
+ st["end"], st["end_ts"] = e["id"], e["ts"]
409
+ else:
410
+ st["act"], st["act_ts"] = e["id"], e["ts"]
411
+ return out
412
+
413
+ def _turn_prompt(self, evs: list[dict], since: int, session: str | None) -> str:
414
+ """The prompt that drove this turn: the first substantive one from the ending session (any session
415
+ if it has none), not simply the latest one, which is often 'ok' or a follow-up."""
416
+ prompts = [e for e in evs if e["id"] > since and e["t"] == "prompt" and e.get("text")]
417
+ mine = [e for e in prompts if session is None or str(e.get("session") or "") == session] or prompts
418
+ for e in mine:
419
+ if titles.prompt_title(e["text"]):
420
+ return e["text"]
421
+ return mine[0]["text"] if mine else ""
422
+
423
+ def maybe_auto_step(self, session: str | None = None, final: bool = False) -> dict | None:
424
+ """Close a step by itself when the settings say so and something meaningful changed.
425
+
426
+ Never creates an empty step, nor one for noise-only changes (lockfiles, build output, ...). In
427
+ `turn` mode a step closes only when a session's turn has ended and no other session is mid-turn
428
+ (for at most MAX_DEFER_SECONDS), and not within COALESCE_SECONDS of the previous step, so a burst
429
+ of turn ends (one per subagent) makes one step. `final` (session end) skips the waiting."""
430
+ mode = config.settings()["auto_step"]
431
+ if mode == "off":
432
+ return None
433
+ with self.locked():
434
+ self.refresh()
435
+ pend = self.pending(counts=False)
436
+ if not pend or not noise.split(pend)[0]:
437
+ return None
438
+ evs = self.journal()
439
+ steps = self.steps()
440
+ since = steps[-1]["id"] if steps else 0
441
+ now = time.time()
442
+ sess = self._sessions(evs, since)
443
+ ended = {k: v for k, v in sess.items() if v["end"] > v["act"]}
444
+ if not final and steps and steps[-1].get("source") == "auto" and now - steps[-1].get("ts", 0) < COALESCE_SECONDS:
445
+ return None
446
+ if mode == "turn" and not ended:
447
+ if sess:
448
+ return None # a turn is in progress
449
+ mode = "idle" # no hooks: fall back to quiet-period steps
450
+ if mode == "turn" and not final:
451
+ busy = [v for v in sess.values() if v["act"] > v["end"] and now - v["act_ts"] < SESSION_STALE_SECONDS]
452
+ if busy and now - max(v["end_ts"] for v in ended.values()) < MAX_DEFER_SECONDS:
453
+ return None # another session is still working
454
+ if mode == "idle" and not ended:
455
+ last = config.read_json(self.dir / "meta.json", {}).get("last_change", 0)
456
+ if time.time() - last < config.settings()["idle_seconds"]:
457
+ return None
458
+ if session is None and ended:
459
+ session = max(ended, key=lambda k: ended[k]["end"])
460
+ prompt = self._turn_prompt(evs, since, session)
461
+ real = noise.split(pend)[0]
462
+ new_syms = []
463
+ if self.model:
464
+ prev = self.load_step(steps[-1]["n"]) if steps else None
465
+ marks = A.diff_models(prev["model"] if prev else None, self.model)
466
+ new_syms = [k.split(":")[-1].split(".")[-1] for kind in ("functions", "classes") for k, v in (marks.get(kind) or {}).items() if v == "new"][:3]
467
+ title = titles.derive_title(prompt, real, new_syms)
468
+ return self._snapshot(title, prompt[:300], "auto", prompt=prompt)
469
+
470
+ # ------------------------------------------------------------------ annotations
471
+ def annotations(self) -> dict:
472
+ todos: dict[str, dict] = {}
473
+ claims, notes, links = [], {}, []
474
+ for e in self.journal():
475
+ t = e["t"]
476
+ if t == "todo":
477
+ todos[e["tid"]] = {"id": e["tid"], "text": e["text"], "status": "open", "added_step": e.get("step"),
478
+ "added": e["ts"], "evidence": None, "by": e.get("by", "agent")}
479
+ elif t == "todo_done" and e.get("tid") in todos:
480
+ todos[e["tid"]].update(status="done", evidence=e.get("evidence"), done_step=e.get("step"), done=e["ts"])
481
+ elif t == "todo_drop" and e.get("tid") in todos:
482
+ todos[e["tid"]]["status"] = "dropped"
483
+ elif t == "claim":
484
+ claims = [c for c in claims if not (c["concept"] == e["concept"] and c["symbol"] == e["symbol"])]
485
+ claims.append({"id": e["id"], "concept": e["concept"], "symbol": e["symbol"], "note": e.get("note", ""), "ts": e["ts"]})
486
+ elif t == "note":
487
+ notes[e["symbol"]] = {"symbol": e["symbol"], "role": e["role"], "ts": e["ts"]}
488
+ elif t == "link":
489
+ links = [x for x in links if not (x["from"] == e["from"] and x["to"] == e["to"])]
490
+ links.append({"id": e["id"], "from": e["from"], "to": e["to"], "label": e.get("label", ""), "ts": e["ts"]})
491
+ elif t == "retract":
492
+ claims = [c for c in claims if c["id"] != e.get("ref")]
493
+ links = [x for x in links if x["id"] != e.get("ref")]
494
+ return {"todos": list(todos.values()), "claims": claims, "notes": list(notes.values()), "links": links}
495
+
496
+ def _segment(self, kind: str, node: dict) -> str:
497
+ text = self.text(node["file"]) or ""
498
+ if kind in ("functions", "classes") and node.get("line"):
499
+ ls = text.splitlines()
500
+ return "\n".join(ls[(node.get("start") or node["line"]) - 1: node.get("end") or node["line"] + 60])
501
+ return text
502
+
503
+ def verify(self) -> dict:
504
+ """Check every annotation against the code as it is now."""
505
+ self.ensure()
506
+ ann = self.annotations()
507
+ problems = []
508
+ for c in ann["claims"]:
509
+ hit = A.resolve(self.model, c["symbol"])
510
+ if not hit:
511
+ c["status"], c["reason"] = "missing", f"no symbol named {c['symbol']}"
512
+ elif c["concept"] not in C.CATALOG:
513
+ c["status"], c["reason"] = "unverified", f"{c['concept']} is not in the catalog"
514
+ else:
515
+ kind, sid, node = hit
516
+ c["resolved"], c["file"], c["line"] = sid, node["file"], node.get("line")
517
+ part = (self.cache.get(node["file"]) or {}).get("part") or {}
518
+ strong = C.evidence_in(c["concept"], self._segment(kind, node))
519
+ weak = None if strong else C.evidence_in(c["concept"], "", part.get("raw_imports", []))
520
+ if strong:
521
+ c["status"], c["reason"] = "verified", strong
522
+ elif weak:
523
+ c["status"], c["reason"] = "weak", f"only {weak}; nothing in {sid.split(':')[-1]} itself"
524
+ else:
525
+ c["status"], c["reason"] = "unverified", f"no sign of {C.CATALOG[c['concept']][0].lower()} in {sid.split(':')[-1]}"
526
+ if c["status"] != "verified":
527
+ problems.append({"kind": "claim", "id": c["id"], "status": c["status"], "text": f"{c['concept']} → {c['symbol']}: {c['reason']}"})
528
+ for nt in ann["notes"]:
529
+ hit = A.resolve(self.model, nt["symbol"])
530
+ nt["status"] = "ok" if hit else "stale"
531
+ if hit:
532
+ nt["resolved"] = hit[1]
533
+ else:
534
+ problems.append({"kind": "note", "status": "stale", "text": f"note on {nt['symbol']}: the symbol no longer exists"})
535
+ for ln in ann["links"]:
536
+ a, b = A.resolve(self.model, ln["from"]), A.resolve(self.model, ln["to"])
537
+ ln["status"] = "ok" if a and b else "stale"
538
+ if a and b:
539
+ ln["from_id"], ln["to_id"] = a[1], b[1]
540
+ else:
541
+ problems.append({"kind": "link", "id": ln["id"], "status": "stale",
542
+ "text": f"link {ln['from']} → {ln['to']}: {'from' if not a else 'to'} side not found"})
543
+ for td in ann["todos"]:
544
+ if td["status"] != "done":
545
+ continue
546
+ ev = (td.get("evidence") or "").strip()
547
+ if not ev:
548
+ td["verified"], td["reason"] = False, "marked done without evidence"
549
+ else:
550
+ hit = A.resolve(self.model, ev)
551
+ path = hit[2]["file"] if hit else (ev if ev in self.index else None)
552
+ if not path:
553
+ td["verified"], td["reason"] = False, f"evidence {ev} not found"
554
+ else:
555
+ base = self.step_files(td["added_step"]) if td.get("added_step") is not None else {}
556
+ if base.get(path) == self.index.get(path) and path in base:
557
+ td["verified"], td["reason"] = False, f"{path} has not changed since the todo was added"
558
+ else:
559
+ td["verified"], td["reason"] = True, f"{path} changed after the todo was added"
560
+ if not td["verified"]:
561
+ problems.append({"kind": "todo", "id": td["id"], "status": "unverified", "text": f"todo “{td['text'][:60]}” done: {td['reason']}"})
562
+ for s in self.steps():
563
+ check = self.title_check(s)
564
+ if check != "ok":
565
+ problems.append({"kind": "step", "id": s["n"], "status": "mismatch", "text": f"step {s['n']} “{s['title'][:50]}”: {check}"})
566
+ for rel, e in self.cache.items():
567
+ if (e.get("part") or {}).get("error"):
568
+ problems.append({"kind": "parse", "status": "error", "text": f"{rel} does not parse (the agent may be mid-edit)"})
569
+ return {**ann, "problems": problems}
570
+
571
+ def todo_add(self, text: str, by: str = "agent") -> dict:
572
+ text = " ".join((text or "").split())[:200]
573
+ if not text:
574
+ raise ValueError("a todo needs text")
575
+ steps = self.steps()
576
+ n = len([e for e in self.journal() if e["t"] == "todo"]) + 1
577
+ return self.append({"t": "todo", "tid": f"T{n}", "text": text, "step": steps[-1]["n"] if steps else None, "by": by})
578
+
579
+ def todo_done(self, tid: str, evidence: str = "") -> dict:
580
+ if not any(t["id"] == tid for t in self.annotations()["todos"]):
581
+ raise ValueError(f"no todo {tid}")
582
+ self.refresh()
583
+ steps = self.steps()
584
+ self.append({"t": "todo_done", "tid": tid, "evidence": (evidence or "")[:200], "step": steps[-1]["n"] if steps else None})
585
+ return next(t for t in self.verify()["todos"] if t["id"] == tid)
586
+
587
+ def todo_drop(self, tid: str) -> None:
588
+ self.append({"t": "todo_drop", "tid": tid})
589
+
590
+ def claim(self, concept: str, symbol: str, note: str = "") -> dict:
591
+ if concept not in C.CATALOG:
592
+ raise ValueError(f"unknown concept {concept}; see `catalog`")
593
+ self.refresh()
594
+ ev = self.append({"t": "claim", "concept": concept, "symbol": symbol[:200], "note": " ".join(note.split())[:160]})
595
+ return next(c for c in self.verify()["claims"] if c["id"] == ev["id"])
596
+
597
+ def note(self, symbol: str, role: str) -> dict:
598
+ role = " ".join((role or "").split())[:140]
599
+ if not role:
600
+ raise ValueError("a note needs text")
601
+ self.refresh()
602
+ self.append({"t": "note", "symbol": symbol[:200], "role": role})
603
+ return next(n for n in self.verify()["notes"] if n["symbol"] == symbol[:200])
604
+
605
+ def link(self, a: str, b: str, label: str = "") -> dict:
606
+ self.refresh()
607
+ ev = self.append({"t": "link", "from": a[:200], "to": b[:200], "label": " ".join(label.split())[:60]})
608
+ return next(x for x in self.verify()["links"] if x["id"] == ev["id"])
609
+
610
+ def record(self, kind: str, **data) -> dict:
611
+ """Prompts, tool calls and turn ends reported by hooks or plugins. Text is redacted. Hook events
612
+ are dropped (returns {}) when the journal is past its size cap, when a tool event repeats the
613
+ previous one without adding information, or when too many tool events piled up since the last step."""
614
+ if kind == "prompt":
615
+ data["text"] = redact(" ".join(str(data.get("text", "")).split()))[:300]
616
+ if data.get("session") is not None:
617
+ data["session"] = str(data["session"])[:64]
618
+ if kind in ("prompt", "tool", "turn_end"):
619
+ with self.locked():
620
+ try:
621
+ if (self.dir / "journal.jsonl").stat().st_size > JOURNAL_CAP_BYTES:
622
+ return {}
623
+ except OSError:
624
+ pass
625
+ if kind == "tool":
626
+ evs = self.journal()
627
+ last = evs[-1] if evs else {}
628
+ if (last.get("t") == "tool" and last.get("tool") == data.get("tool") and not data.get("files")
629
+ and not last.get("files") and last.get("session") == data.get("session")):
630
+ return {}
631
+ steps = self.steps()
632
+ since = steps[-1]["id"] if steps else 0
633
+ if sum(1 for e in evs if e["id"] > since and e["t"] == "tool") >= MAX_TOOL_EVENTS_PER_STEP:
634
+ return {}
635
+ return self.append({"t": kind, **data})
636
+ return self.append({"t": kind, **data})
637
+
638
+ # ------------------------------------------------------------------ views
639
+ def files_view(self, base: dict | None) -> list[dict]:
640
+ rows = []
641
+ for rel, e in sorted(self.cache.items()):
642
+ part = e.get("part") or {}
643
+ status = None
644
+ if base is not None:
645
+ status = "A" if rel not in base else "M" if base[rel] != e["sha"] else None
646
+ rows.append({"path": rel, "lang": part.get("lang") or os.path.splitext(rel)[1].lstrip(".") or rel,
647
+ "layer": A.layer_of(rel, part) if part else None, "lines": e.get("lines"),
648
+ "functions": len(part.get("functions", [])), "classes": len(part.get("classes", [])),
649
+ "secret": bool(e.get("secret")), "status": status, "error": bool(part.get("error"))})
650
+ if base is not None:
651
+ rows += [{"path": rel, "lang": "", "layer": None, "lines": 0, "functions": 0, "classes": 0, "secret": False,
652
+ "status": "D", "error": False} for rel in sorted(set(base) - set(self.cache))]
653
+ return rows
654
+
655
+ def state(self) -> dict:
656
+ self.ensure()
657
+ steps = self.steps()
658
+ base = self.step_files(steps[-1]["n"]) if steps else None
659
+ v = self.verify()
660
+ fns = self.model["functions"]
661
+ hot = sorted(fns.items(), key=lambda kv: -kv[1]["complexity"])[:12]
662
+ evs = self.journal()
663
+ return {
664
+ "key": self.key, "name": self.name, "root": self.root, "version": self.version, "truncated": self.truncated,
665
+ "truncated_info": self.truncated_info,
666
+ "steps": [{**{k: s.get(k) for k in ("n", "title", "intent", "source", "ts", "summary", "changes", "prompt")}, "title_check": self.title_check(s)}
667
+ for s in steps],
668
+ "pending": self.pending(),
669
+ "files": self.files_view(base),
670
+ "counts": {"files": len(self.cache), "modules": len(self.model["modules"]), "classes": len(self.model["classes"]),
671
+ "functions": len(fns), "tables": len(self.model["tables"]), "routes": len(self.model["routes"]),
672
+ "concepts": len(self.concepts), "infra": len(self.infra["nodes"])},
673
+ "layers": dict(Counter(m["layer"] for m in self.model["modules"].values())),
674
+ "routes": self.model["routes"],
675
+ "concepts": [{"id": cid, "title": C.CATALOG[cid][0], "icon": C.CATALOG[cid][1], "group": C.CATALOG[cid][2],
676
+ "meaning": C.CATALOG[cid][3], "evidence": evs_[:8],
677
+ "claims": [c for c in v["claims"] if c["concept"] == cid]}
678
+ for cid, evs_ in sorted(self.concepts.items(), key=lambda kv: list(C.CATALOG).index(kv[0]))]
679
+ + [{"id": cid, "title": C.CATALOG[cid][0], "icon": C.CATALOG[cid][1], "group": C.CATALOG[cid][2],
680
+ "meaning": C.CATALOG[cid][3], "evidence": [], "claims": [c for c in v["claims"] if c["concept"] == cid]}
681
+ for cid in sorted({c["concept"] for c in v["claims"]} - set(self.concepts)) if cid in C.CATALOG],
682
+ "infra": self.infra,
683
+ "hotspots": [{"id": fid, "complexity": f["complexity"], "loc": f["loc"], "hints": f["hints"], "file": f["file"], "line": f["line"]}
684
+ for fid, f in hot],
685
+ "algorithms": sorted(({"id": fid, "hints": f["hints"], "file": f["file"], "line": f["line"]}
686
+ for fid, f in fns.items() if any(h not in ("async I/O",) for h in f["hints"])),
687
+ key=lambda x: (-len(x["hints"]), x["id"]))[:60],
688
+ "todos": v["todos"], "claims": v["claims"], "notes": v["notes"], "links": v["links"], "problems": v["problems"],
689
+ "activity": [{k: e.get(k) for k in ("id", "ts", "t", "text", "tool", "files", "n", "title", "tid", "concept", "symbol", "role", "agent")}
690
+ for e in evs[-80:]],
691
+ }
692
+
693
+
694
+ _PROJECTS: dict[tuple[str, str], Project] = {}
695
+ _PLOCK = threading.Lock()
696
+
697
+
698
+ def open_project(root: str) -> Project:
699
+ """One Project object per root (and per MAPWRIGHT_HOME) in this process."""
700
+ real = os.path.realpath(A.find_root(os.path.abspath(root)))
701
+ k = (str(config.home()), real)
702
+ with _PLOCK:
703
+ p = _PROJECTS.get(k)
704
+ if p is None:
705
+ p = _PROJECTS[k] = Project(real)
706
+ return p
707
+
708
+
709
+ def by_key(key: str) -> Project | None:
710
+ if not _KEY.match(key or ""):
711
+ return None
712
+ entry = config.projects().get(key)
713
+ if not entry or not os.path.isdir(entry["root"]):
714
+ return None
715
+ return open_project(entry["root"])