pi-sleep-agent 0.8.8 → 0.8.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/package.json +1 -1
- package/sleepkit/cli.py +93 -10
- package/sleepkit/replay.py +49 -3
- package/sleepkit/store.py +29 -0
package/README.md
CHANGED
|
@@ -29,7 +29,7 @@ Sleep adds a reviewable memory cycle to nine selectable coding-agent targets. It
|
|
|
29
29
|
- **Budgeted consolidation.** Sleep runs only after the record and time gates pass.
|
|
30
30
|
- **Review before permanence.** New memories stay provisional until you promote or refuse them.
|
|
31
31
|
- **Nightly dream cycle.** Mine repeated preferences, consolidate recent observations, and measure memory effects while you are away.
|
|
32
|
-
- **Measured replay.** Run the same task with memory off and on before deciding whether a memory helps.
|
|
32
|
+
- **Measured replay.** Run the same task with memory off and on before deciding whether a memory helps. Each completed pass leaves a one-line quality summary (`helps / hurts / ties`) in `status` and in the nightly report, so the question stays answerable between passes; a pass whose two arms ran the same prompt says it measured nothing instead of reporting zeros.
|
|
33
33
|
- **Repeated preference detection.** `mine` finds instructions you gave in multiple sessions without a model call.
|
|
34
34
|
- **Project isolation.** A project's provisional memory is injected only into that project.
|
|
35
35
|
- **Reviewable permanence.** Each approved memory in your index carries its review date and source, so you can tell when a saved instruction was pinned.
|
|
@@ -317,11 +317,13 @@ Run the two dream components separately when you want more control:
|
|
|
317
317
|
| `./sleep.py demote <id> --note "..."` | Un-approve a promoted memory: mark the approved record as retracted (kept on record, not deleted) and reopen the candidate as refused. Also accepts a permanent record's filename slug or the `id:` it carries, so a memory whose candidate record is gone is still reversible |
|
|
318
318
|
| `./sleep.py refuse <id> --note "..."` | Record that a candidate is incorrect |
|
|
319
319
|
| `./sleep.py replay --run --limit 3` | Compare selected tasks with memory off and on |
|
|
320
|
+
| `./sleep.py status -v` | ... and the last measured memory quality (`3 helps / 1 hurts / 4 ties from 8 samples`, or "no replay measurement has run yet") |
|
|
320
321
|
| `./sleep.py mine --dry-run` | Find preferences repeated across sessions |
|
|
321
322
|
| `./sleep.py nightly --dry-run` | Preview the unattended nightly workflow |
|
|
322
323
|
| `./sleep.py doctor` | Find duplicate or overlapping memory records, flag contradicting candidates (same subject, one side carries a correction marker) and print the `supersede <older> <newer>` command to run, flag provisional candidates not rediscovered for 4+ weeks, flag supersede pointers that name a record no longer in the store, and report records parked in quarantine awaiting a human (report only: it never supersedes, drops, merges or un-parks anything itself) |
|
|
323
324
|
| `./sleep.py doctor --stale <value>` | Show which approved records still hold a stale value (e.g. a changed URL) |
|
|
324
325
|
| `./sleep.py doctor --contradictions` | Ask a model to adjudicate shortlisted pairs of provisional records for conflicts (`--budget`, default $0.25); report-only, and each finding carries the `supersede <older> <newer>` command to run |
|
|
326
|
+
| `./sleep.py doctor --contradictions --tier t2` | Adjudicate pairs of **approved** permanent memory instead of candidates (`--tier both` does both); a T2 finding is labelled `contradicting-memory` and carries the `demote <older>` command, because approved memory is un-approved, never superseded |
|
|
325
327
|
|
|
326
328
|
The headless model call picks its transport by what exists on the machine, in this order:
|
|
327
329
|
the host agent's own `omo -p` (used when an `omo` binary is on PATH), then `claude
|
|
@@ -403,7 +405,7 @@ candidates/ provisional T1 records
|
|
|
403
405
|
reports/ replay reports
|
|
404
406
|
logs/ hook and nightly logs
|
|
405
407
|
.quarantine/ rejected unsafe input
|
|
406
|
-
state.json sleep and replay timestamps
|
|
408
|
+
state.json sleep and replay timestamps, and the last measured memory-quality summary
|
|
407
409
|
env optional scheduled-run environment
|
|
408
410
|
```
|
|
409
411
|
|
package/package.json
CHANGED
package/sleepkit/cli.py
CHANGED
|
@@ -66,6 +66,10 @@ def cmd_status(a) -> int:
|
|
|
66
66
|
except OSError:
|
|
67
67
|
permanent = []
|
|
68
68
|
_p("sleep memory: T0=disposable captures, T1=provisional awaiting review, T2=permanent (promote-only)")
|
|
69
|
+
try:
|
|
70
|
+
_p(replay_mod.quality_line())
|
|
71
|
+
except Exception:
|
|
72
|
+
pass
|
|
69
73
|
_p("T0 buffer %d records / %d files / %d bytes (captured while you worked)" % (bc["records"], bc["files"], bc["bytes"]))
|
|
70
74
|
_p("T1 candidates %d provisional, %d refused, %d total (distilled, shown as unverified context)" % (len(prov), len(ref), len(cands)))
|
|
71
75
|
_p(" quarantine %d parked (needs human eyes) (held out until you review)" % len(q))
|
|
@@ -207,7 +211,14 @@ def _doctor_shortlist() -> list:
|
|
|
207
211
|
return out
|
|
208
212
|
|
|
209
213
|
|
|
210
|
-
def
|
|
214
|
+
def _tier_of(cid: str, memory_ids: set) -> str:
|
|
215
|
+
"""Which tier a record belongs to, so a pair's own membership decides its wording."""
|
|
216
|
+
return "t2" if cid in memory_ids else "t1"
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _llm_contradictions(pairs: list, budget: float, model: str | None,
|
|
220
|
+
tier: str = "t1", records: dict | None = None,
|
|
221
|
+
memory_ids: set | None = None) -> list:
|
|
211
222
|
"""Ask the model to adjudicate shortlisted pairs. Report-only: returns findings,
|
|
212
223
|
changes nothing. A failed or unparseable call yields [] so doctor still reports the
|
|
213
224
|
rest of its scan instead of dying on a model problem."""
|
|
@@ -217,8 +228,15 @@ def _llm_contradictions(pairs: list, budget: float, model: str | None) -> list:
|
|
|
217
228
|
# under meta["_body"] (that key only exists on the list_candidates path). Reading the
|
|
218
229
|
# wrong one sent the model three pairs of empty bodies, and it answered, correctly,
|
|
219
230
|
# that there was nothing to compare - a silently useless call, not a crash.
|
|
220
|
-
|
|
231
|
+
# `records` carries (meta, body) for the ids the caller already parsed. A permanent
|
|
232
|
+
# T2 record is not in the candidate store, so `store.get_candidate` returns nothing
|
|
233
|
+
# for it and this function used to hand the model three pairs of EMPTY bodies - the
|
|
234
|
+
# model then answered, correctly, that no claim was present, and the whole T2 pass
|
|
235
|
+
# reported a clean store without reading a single memory.
|
|
236
|
+
seen = dict(records or {})
|
|
221
237
|
for cid in {c for pair in pairs for c in pair}:
|
|
238
|
+
if cid in seen:
|
|
239
|
+
continue
|
|
222
240
|
got = store.get_candidate(cid)
|
|
223
241
|
seen[cid] = (got[0], got[1]) if got else ({}, "")
|
|
224
242
|
lines = []
|
|
@@ -228,8 +246,16 @@ def _llm_contradictions(pairs: list, budget: float, model: str | None) -> list:
|
|
|
228
246
|
meta, body = seen.get(cid, ({}, ""))
|
|
229
247
|
lines.append(" id=%s ts=%s body=%s" % (cid, meta.get("ts", "?"),
|
|
230
248
|
(body or "").replace("\n", " ").strip()))
|
|
231
|
-
|
|
232
|
-
|
|
249
|
+
if tier in ("t2", "both"):
|
|
250
|
+
user = ("Memory records, grouped into candidate pairs. Some are unreviewed "
|
|
251
|
+
"provisional candidates; some are permanent memories a human already "
|
|
252
|
+
"approved.\nAdjudicate each pair per your instructions: two approved "
|
|
253
|
+
"memories that now assert opposite things mean the older one needs a "
|
|
254
|
+
"human verdict, not that the memory store is broken.\n\n"
|
|
255
|
+
+ "\n".join(lines))
|
|
256
|
+
else:
|
|
257
|
+
user = ("Unreviewed provisional memory records, grouped into candidate pairs.\n"
|
|
258
|
+
"Adjudicate each pair per your instructions.\n\n" + "\n".join(lines))
|
|
233
259
|
pf = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "prompts",
|
|
234
260
|
"contradiction-check.md")
|
|
235
261
|
# The subprocess seam: tests replace llm.run with a fake so the adjudication path is
|
|
@@ -264,6 +290,7 @@ def _llm_contradictions(pairs: list, budget: float, model: str | None) -> list:
|
|
|
264
290
|
if not isinstance(doc, dict):
|
|
265
291
|
return []
|
|
266
292
|
found = []
|
|
293
|
+
mem = memory_ids or set()
|
|
267
294
|
for item in doc.get("pairs") or []:
|
|
268
295
|
if not isinstance(item, dict) or not item.get("contradiction"):
|
|
269
296
|
continue
|
|
@@ -271,18 +298,63 @@ def _llm_contradictions(pairs: list, budget: float, model: str | None) -> list:
|
|
|
271
298
|
older, newer = pairs[int(item["pair"]) - 1]
|
|
272
299
|
except (KeyError, IndexError, ValueError, TypeError):
|
|
273
300
|
continue
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
301
|
+
# The label follows the pair, not the --tier flag: in `both` mode a T1 finding
|
|
302
|
+
# must not be reported as memory, nor a memory finding as a candidate.
|
|
303
|
+
tier_of_pair = "t2" if (older in mem and newer in mem) else "t1"
|
|
304
|
+
# T2 has no supersede verb: the only honest action on approved memory is a
|
|
305
|
+
# human demote (un-approve) of the record that lost, so the suggestion differs.
|
|
306
|
+
if tier_of_pair == "t2":
|
|
307
|
+
found.append({"issue": "contradicting-memory",
|
|
308
|
+
"pair": [older, newer], "older": older, "newer": newer,
|
|
309
|
+
"source": "model", "reason": str(item.get("reason", ""))[:200],
|
|
310
|
+
"suggest": "sleep.py demote %s" % older})
|
|
311
|
+
else:
|
|
312
|
+
found.append({"issue": "contradicting-candidates",
|
|
313
|
+
"pair": [older, newer], "older": older, "newer": newer,
|
|
314
|
+
"source": "model", "reason": str(item.get("reason", ""))[:200],
|
|
315
|
+
"suggest": "sleep.py supersede %s %s" % (older, newer)})
|
|
278
316
|
return found
|
|
279
317
|
|
|
280
318
|
|
|
319
|
+
def _doctor_memory_shortlist(memory_dir: str | None = None) -> list:
|
|
320
|
+
"""(id, body, ts) for every active permanent T2 record, the input to candidate_pairs."""
|
|
321
|
+
return store.permanent_records(memory_dir)
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _memory_records(memory_dir: str | None = None) -> dict:
|
|
325
|
+
"""id -> (meta, body) for active permanent records, the lookup the adjudicator needs."""
|
|
326
|
+
out = {}
|
|
327
|
+
for cid, body, ts in store.permanent_records(memory_dir):
|
|
328
|
+
out[cid] = ({"id": cid, "ts": ts}, body)
|
|
329
|
+
return out
|
|
330
|
+
|
|
331
|
+
|
|
281
332
|
def cmd_doctor(a) -> int:
|
|
282
333
|
issues = store.doctor(needle=a.stale)
|
|
283
334
|
if getattr(a, "contradictions", False):
|
|
284
|
-
|
|
285
|
-
|
|
335
|
+
tier = getattr(a, "tier", "t1") or "t1"
|
|
336
|
+
mdir = getattr(a, "cwd", None) or None
|
|
337
|
+
records: dict = {}
|
|
338
|
+
memory_ids: set = set()
|
|
339
|
+
if tier == "t2":
|
|
340
|
+
recs = _doctor_memory_shortlist(mdir)
|
|
341
|
+
records = _memory_records(mdir)
|
|
342
|
+
memory_ids = set(records)
|
|
343
|
+
elif tier == "both":
|
|
344
|
+
mem_recs = _doctor_memory_shortlist(mdir)
|
|
345
|
+
records = _memory_records(mdir)
|
|
346
|
+
memory_ids = set(records)
|
|
347
|
+
recs = _doctor_shortlist() + mem_recs
|
|
348
|
+
else:
|
|
349
|
+
recs = _doctor_shortlist()
|
|
350
|
+
# Keep only same-tier pairs: a T1-vs-T2 pair has no single honest action (there is
|
|
351
|
+
# no verb that both supersedes a candidate and un-approves a memory), so asking the
|
|
352
|
+
# model about it would produce a suggestion no command can carry out.
|
|
353
|
+
pairpairs = [p for p in store.candidate_pairs(recs)
|
|
354
|
+
if (_tier_of(p[0], memory_ids) == _tier_of(p[1], memory_ids))]
|
|
355
|
+
issues += _llm_contradictions(pairpairs, getattr(a, "budget", 0.25),
|
|
356
|
+
getattr(a, "model", None), tier=tier,
|
|
357
|
+
records=records, memory_ids=memory_ids)
|
|
286
358
|
if a.json:
|
|
287
359
|
_p(json.dumps(issues, indent=2, sort_keys=True, default=str))
|
|
288
360
|
elif not issues:
|
|
@@ -362,7 +434,14 @@ def cmd_nightly(a) -> int:
|
|
|
362
434
|
limit=a.replay_limit,
|
|
363
435
|
interval_hours=a.replay_interval,
|
|
364
436
|
dry_run=a.dry_run)
|
|
437
|
+
# The report is machine-read (launchd tails it, tests parse it), so stdout stays a
|
|
438
|
+
# single JSON document. The human-readable line rides inside it and is echoed as
|
|
439
|
+
# PLAIN TEXT only on a terminal, where a person is the one reading the night.
|
|
440
|
+
qline = rep["replay"].get("quality_line") or replay_mod.quality_line()
|
|
441
|
+
rep["replay"]["quality_line"] = qline
|
|
365
442
|
_p(json.dumps(rep, indent=2, sort_keys=True, default=str))
|
|
443
|
+
if sys.stdout.isatty():
|
|
444
|
+
_p(qline)
|
|
366
445
|
# "ran" only means the gate let the pass through. If the model call itself failed,
|
|
367
446
|
# the night produced nothing and must not report success to launchd.
|
|
368
447
|
slept = rep["sleep"].get("action") in ("skipped", "would-run") or (
|
|
@@ -540,6 +619,10 @@ def main(argv=None) -> int:
|
|
|
540
619
|
"conflicts (costs budget; report-only, supersedes nothing)")
|
|
541
620
|
dc.add_argument("--budget", type=float, default=0.25,
|
|
542
621
|
help="max USD for --contradictions (default 0.25)")
|
|
622
|
+
dc.add_argument("--tier", choices=["t1", "t2", "both"], default="t1",
|
|
623
|
+
help="which tier --contradictions adjudicates: unreviewed "
|
|
624
|
+
"candidates (t1, default), approved permanent memory (t2), "
|
|
625
|
+
"or both")
|
|
543
626
|
dc.set_defaults(fn=cmd_doctor)
|
|
544
627
|
|
|
545
628
|
rp = sub.add_parser("replay", help="layer 3: A/B measure provisional memory, report-only")
|
package/sleepkit/replay.py
CHANGED
|
@@ -283,6 +283,49 @@ def replay_due(interval_hours: float) -> tuple:
|
|
|
283
283
|
return False, "%.1f of %.1f h since the last pass" % (elapsed, interval_hours)
|
|
284
284
|
|
|
285
285
|
|
|
286
|
+
def record_quality(rep: dict) -> dict:
|
|
287
|
+
"""Persist the memory-quality summary of a completed replay pass.
|
|
288
|
+
|
|
289
|
+
The dated report file holds the detail, but a skipped night leaves no trace at all,
|
|
290
|
+
so "is my memory actually helping?" had no answer between runs. A small rolling
|
|
291
|
+
summary in state.json keeps the last real measurement readable from `status` and
|
|
292
|
+
the nightly report without parsing report files.
|
|
293
|
+
"""
|
|
294
|
+
tally = rep.get("verdict_tally") or {}
|
|
295
|
+
summary = {"ran": paths.now(), "tally": tally,
|
|
296
|
+
"helpful": tally.get("helps", 0),
|
|
297
|
+
"harmed": tally.get("hurts", 0),
|
|
298
|
+
"tie": tally.get("tie", 0),
|
|
299
|
+
"samples": rep.get("samples", 0),
|
|
300
|
+
"answered": rep.get("answered", 0),
|
|
301
|
+
"treatment_delivered": bool(rep.get("treatment_delivered"))}
|
|
302
|
+
paths.write_state({"replay_quality": summary})
|
|
303
|
+
return summary
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def quality_line(state: dict | None = None) -> str:
|
|
307
|
+
"""One line for the nightly report: what the last real measurement said.
|
|
308
|
+
|
|
309
|
+
A skip is not a quality signal, so the line states the last measurement that
|
|
310
|
+
actually happened and how long ago, instead of pretending the skip measured
|
|
311
|
+
anything. Never-run is reported as such rather than as a zero score.
|
|
312
|
+
"""
|
|
313
|
+
from . import consolidate # lazy: hours_since lives there
|
|
314
|
+
st = state if state is not None else paths.read_state()
|
|
315
|
+
q = st.get("replay_quality") or {}
|
|
316
|
+
if not q or not q.get("ran"):
|
|
317
|
+
return "memory quality: no replay measurement has run yet"
|
|
318
|
+
age = consolidate.hours_since(q.get("ran"))
|
|
319
|
+
when = "just now" if age is None or age < 1 else (
|
|
320
|
+
"%.0fh ago" % age if age < 48 else "%.0fd ago" % (age / 24.0))
|
|
321
|
+
if not q.get("treatment_delivered"):
|
|
322
|
+
return ("memory quality: last pass (%s) delivered no treatment, so it measured "
|
|
323
|
+
"nothing" % when)
|
|
324
|
+
return ("memory quality: %d helps / %d hurts / %d ties from %d samples (last pass %s)"
|
|
325
|
+
% (q.get("helpful", 0), q.get("harmed", 0), q.get("tie", 0),
|
|
326
|
+
q.get("samples", 0), when))
|
|
327
|
+
|
|
328
|
+
|
|
286
329
|
def mark_replayed() -> str:
|
|
287
330
|
"""Stamp last_replay so the next nightly pass waits out its interval."""
|
|
288
331
|
ts = paths.now()
|
|
@@ -302,9 +345,10 @@ def nightly(tasks_path: str = DEFAULT_TASKS, model: str | None = None, budget: f
|
|
|
302
345
|
"""
|
|
303
346
|
due, why = replay_due(interval_hours)
|
|
304
347
|
if dry_run:
|
|
305
|
-
return {"skipped": True, "reason": "dry run", "due": due, "detail": why
|
|
348
|
+
return {"skipped": True, "reason": "dry run", "due": due, "detail": why,
|
|
349
|
+
"quality_line": quality_line()}
|
|
306
350
|
if not due:
|
|
307
|
-
return {"skipped": True, "reason": why}
|
|
351
|
+
return {"skipped": True, "reason": why, "quality_line": quality_line()}
|
|
308
352
|
try:
|
|
309
353
|
rep = run_suite(tasks_path=tasks_path, limit=limit, model=model, budget=budget)
|
|
310
354
|
report = write_report(rep)
|
|
@@ -319,9 +363,11 @@ def nightly(tasks_path: str = DEFAULT_TASKS, model: str | None = None, budget: f
|
|
|
319
363
|
return {"failed": True, "attempted": paths.now(), "report": report,
|
|
320
364
|
"reason": "no model call answered: 0/%d samples exited 0; first answer "
|
|
321
365
|
"began %r" % (rep["samples"], (first.get("answer") or "")[:80])}
|
|
366
|
+
summary = record_quality(rep)
|
|
322
367
|
return {"ran": True, "report": report, "clock": mark_replayed(),
|
|
323
368
|
"treatment_delivered": rep.get("treatment_delivered"),
|
|
324
|
-
"verdict_tally": rep.get("verdict_tally")
|
|
369
|
+
"verdict_tally": rep.get("verdict_tally"),
|
|
370
|
+
"quality": summary, "quality_line": quality_line()}
|
|
325
371
|
|
|
326
372
|
|
|
327
373
|
def write_report(rep: dict) -> str:
|
package/sleepkit/store.py
CHANGED
|
@@ -650,6 +650,35 @@ def _first_ref(value) -> str:
|
|
|
650
650
|
return str(value or "").split("|")[0].strip()
|
|
651
651
|
|
|
652
652
|
|
|
653
|
+
def permanent_records(memory_dir: str | None = None) -> list:
|
|
654
|
+
"""(id, body, ts) for every ACTIVE permanent (T2) record, the T2 analogue of the
|
|
655
|
+
provisional shortlist input.
|
|
656
|
+
|
|
657
|
+
Retracted files are excluded: a memory a person already un-approved is a decision,
|
|
658
|
+
not an inconsistency to re-report. `id` prefers the frontmatter id and falls back to
|
|
659
|
+
the filename slug, so a hand-written file (no id in frontmatter) is still nameable
|
|
660
|
+
in a finding and actionable with `demote <slug>`.
|
|
661
|
+
"""
|
|
662
|
+
mdir = memory_dir or paths.memory_dir()
|
|
663
|
+
out = []
|
|
664
|
+
if not os.path.isdir(mdir):
|
|
665
|
+
return out
|
|
666
|
+
for name in sorted(os.listdir(mdir)):
|
|
667
|
+
if not name.endswith(".md") or name == "MEMORY.md":
|
|
668
|
+
continue
|
|
669
|
+
path = os.path.join(mdir, name)
|
|
670
|
+
try:
|
|
671
|
+
with open(path, encoding="utf-8") as fh:
|
|
672
|
+
meta, body = parse(fh.read())
|
|
673
|
+
except (OSError, ValueError):
|
|
674
|
+
continue
|
|
675
|
+
if meta.get("status") == "retracted":
|
|
676
|
+
continue
|
|
677
|
+
ref = str(meta.get("id") or name[:-3]).strip()
|
|
678
|
+
out.append((ref, body or "", str(meta.get("ts") or meta.get("promoted_at") or "")))
|
|
679
|
+
return out
|
|
680
|
+
|
|
681
|
+
|
|
653
682
|
def _archived_candidates() -> list:
|
|
654
683
|
"""Parsed frontmatter of archived candidates (promoted/superseded records live there)."""
|
|
655
684
|
out = []
|