riskrails 4.0.1 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +2 -2
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +22 -13
- package/bin/menu.js +116 -0
- package/bin/riskrails.js +119 -20
- package/hooks/AGENTS.md +0 -6
- package/hooks/CLAUDE.md +0 -6
- package/hooks/guard/dialog.py +14 -2
- package/hooks/guard/paths.py +48 -1406
- package/hooks/guard/paths_base.py +59 -0
- package/hooks/guard/paths_bash.py +176 -0
- package/hooks/guard/paths_classify.py +205 -0
- package/hooks/guard/paths_fs.py +137 -0
- package/hooks/guard/paths_human.py +105 -0
- package/hooks/guard/paths_interp.py +162 -0
- package/hooks/guard/paths_movers.py +393 -0
- package/hooks/guard/paths_shell.py +288 -0
- package/package.json +1 -1
- package/skills/AGENTS.md +0 -6
- package/skills/CLAUDE.md +0 -6
- package/skills/ai-init/scaffold-ai.sh +2 -1
- package/skills/ai-init/templates/.ai/policies/safety.md +4 -3
- package/skills/ai-init/templates/gitignore.snippet +2 -0
- package/skills/ai-task/memory_cleanup.py +213 -0
- package/skills/ai-task/memory_compact.py +25 -0
- package/skills/ai-task/memory_config.py +120 -0
- package/skills/ai-task/memory_export.py +246 -0
- package/skills/ai-task/memory_format.py +7 -3
- package/skills/ai-task/memory_hook.py +172 -0
- package/skills/ai-task/memory_io.py +122 -0
- package/skills/ai-task/memory_migrate.py +187 -0
- package/skills/ai-task/memory_packages.py +83 -0
- package/skills/ai-task/memory_risks.py +84 -0
- package/skills/ai-task/memory_status.py +76 -0
- package/skills/ai-task/memory_sync.py +20 -1481
- package/skills/ai-task/memory_verify.py +316 -0
- package/skills/ai-task/rr_state/cli.py +2 -1
- package/skills/ai-task/rr_state/close.py +2 -2
- package/skills/ai-task/rr_state/dialog_cmd.py +183 -6
- package/skills/ai-task/rr_state/labels.py +14 -2
- package/skills/ai-task/rr_state/memory_cmd.py +91 -68
- package/skills/project-update/history/blobs/8ec171f0fbdf38cc6996d496f01491fd1f21f04a435a118ad979b127a8be06f1 +85 -0
- package/skills/project-update/history/blobs/fa0cb370b024e1a30e1355912032657b08f609e47773a603c193b1e609b0f997 +8 -0
- package/skills/project-update/history/index.json +4 -2
- package/skills/project-update/rr_update/__init__.py +1 -0
- package/skills/project-update/rr_update/apply.py +188 -0
- package/skills/project-update/rr_update/blocks.py +38 -0
- package/skills/project-update/rr_update/build.py +96 -0
- package/skills/project-update/rr_update/cleanup.py +194 -0
- package/skills/project-update/rr_update/cli.py +102 -0
- package/skills/project-update/rr_update/files.py +106 -0
- package/skills/project-update/rr_update/gate.py +60 -0
- package/skills/project-update/rr_update/hints.py +128 -0
- package/skills/project-update/rr_update/history.py +117 -0
- package/skills/project-update/rr_update/kept.py +120 -0
- package/skills/project-update/rr_update/paths.py +108 -0
- package/skills/project-update/rr_update/plan.py +123 -0
- package/skills/project-update/rr_update/report.py +51 -0
- package/skills/project-update/rr_update/rules.py +178 -0
- package/skills/project-update/rr_update/shared.py +74 -0
- package/skills/project-update/rr_update/siblings.py +37 -0
- package/skills/project-update/rr_update/weights.py +171 -0
- package/skills/project-update/update.py +8 -1682
- package/skills/rr-setup/SKILL.md +79 -21
- package/skills/rr-setup/scaffold_ai.py +8 -2
- package/skills/rr-setup/setup.py +371 -4
|
@@ -7,1484 +7,21 @@ writers here: no .ai/memory/, no .ai/local/memory/, no hook (R15). state.py
|
|
|
7
7
|
is the only caller of the writers; the path guard refuses an agent's direct
|
|
8
8
|
write to either directory (R19). Nothing here reads a package to grant or
|
|
9
9
|
refuse anything.
|
|
10
|
+
The rest lives in the sibling modules beside it: memory_io, memory_config, memory_packages,
|
|
11
|
+
memory_verify, memory_hook, memory_risks, memory_cleanup, memory_migrate, memory_export and
|
|
12
|
+
memory_status; this module keeps the pre-push entry that the git hook runs by path.
|
|
10
13
|
"""
|
|
11
|
-
import collections
|
|
12
|
-
import contextlib
|
|
13
|
-
import json
|
|
14
14
|
import os
|
|
15
|
-
import random
|
|
16
|
-
import re
|
|
17
|
-
import shutil
|
|
18
|
-
import subprocess
|
|
19
15
|
import sys
|
|
20
|
-
import tarfile
|
|
21
|
-
import tempfile
|
|
22
|
-
import time
|
|
23
|
-
from datetime import datetime, timezone
|
|
24
|
-
|
|
25
|
-
try:
|
|
26
|
-
import fcntl
|
|
27
|
-
except ImportError: # not POSIX: last writer wins within one clone (docs/local-memory.md)
|
|
28
|
-
fcntl = None
|
|
29
16
|
|
|
30
17
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
31
|
-
import
|
|
32
|
-
import
|
|
33
|
-
from memory_compact import FsReader # noqa: E402,F401 pylint: disable=wrong-import-position,unused-import
|
|
34
|
-
|
|
35
|
-
POLICY_REL = os.path.join(".ai", "policies", "memory.json")
|
|
36
|
-
MEMORY_REL = os.path.join(".ai", "memory")
|
|
37
|
-
LOCAL_REL = os.path.join(".ai", "local", "memory")
|
|
38
|
-
SCHEMA = 1
|
|
39
|
-
DEFAULTS = {"schema": 1, "mode": "files", "period": "quarter", "package_max_bytes": 1048576, "cleanup": False,
|
|
40
|
-
"excerpt_max_bytes": 1024, "keep_rules": 1, "recall": {"budget_tokens": 1500, "max_items": 40}}
|
|
41
|
-
PENDING_MAX = 5
|
|
42
|
-
BATCH_CHARS = "0123456789abcdefghjkmnpqrstvwxyz"
|
|
43
|
-
HOOK_MARKER = "# riskrails-memory-pre-push 1"
|
|
44
|
-
VALIDATOR = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "hooks",
|
|
45
|
-
"memory-pre-push.py"))
|
|
46
|
-
SELF = os.path.abspath(__file__) # the fallback validator: this module, run as a script
|
|
47
|
-
FINDINGS_MAX = 40
|
|
48
|
-
ZERO_SHA = "0" * 40
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
class MemoryError_(Exception): # pylint: disable=invalid-name # not the builtin MemoryError
|
|
52
|
-
"""A refusal with its exit code (§I4: 1 findings, 2 usage, 3 aborted, 5 human, 6 scope)."""
|
|
53
|
-
|
|
54
|
-
def __init__(self, message, code=2):
|
|
55
|
-
super().__init__(message)
|
|
56
|
-
self.code = code
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
def now():
|
|
60
|
-
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
def crash(point):
|
|
64
|
-
"""Fault injection (A9): AI_MEMORY_CRASH_AT=B2..B6 ends the process here."""
|
|
65
|
-
if os.environ.get("AI_MEMORY_CRASH_AT") == point:
|
|
66
|
-
sys.stdout.flush()
|
|
67
|
-
sys.stderr.flush()
|
|
68
|
-
os._exit(70) # pylint: disable=protected-access # a crash, not an exit
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
# ------------------------------------------------------------------ configuration (§D11, §I6)
|
|
72
|
-
def author_slug(name):
|
|
73
|
-
slug = re.sub(r"[^a-z0-9-]+", "-", (name or "").casefold()).strip("-")[:32].strip("-")
|
|
74
|
-
return slug if slug and mf.AUTHOR_RE.match(slug) else "unknown"
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
class MemoryConfig:
|
|
78
|
-
"""The project policy (.ai/policies/memory.json) and the developer's own
|
|
79
|
-
choices (.ai/local/memory/config.json)."""
|
|
80
|
-
|
|
81
|
-
def __init__(self, root, policy, local, present):
|
|
82
|
-
self.root, self.policy, self.local, self.present = root, policy, local, present
|
|
83
|
-
|
|
84
|
-
def __getattr__(self, name):
|
|
85
|
-
if name in ("mode", "period", "package_max_bytes", "cleanup", "excerpt_max_bytes", "keep_rules"):
|
|
86
|
-
return self.policy.get(name, DEFAULTS[name])
|
|
87
|
-
raise AttributeError(name)
|
|
88
|
-
|
|
89
|
-
@property
|
|
90
|
-
def recall_budget(self):
|
|
91
|
-
return int((self.policy.get("recall") or {}).get("budget_tokens", 1500))
|
|
92
|
-
|
|
93
|
-
@property
|
|
94
|
-
def recall_max_items(self):
|
|
95
|
-
return int((self.policy.get("recall") or {}).get("max_items", 40))
|
|
96
|
-
|
|
97
|
-
@property
|
|
98
|
-
def author(self):
|
|
99
|
-
return self.local.get("author") or author_slug(os.environ.get("USER") or os.environ.get("USERNAME"))
|
|
100
|
-
|
|
101
|
-
@property
|
|
102
|
-
def index(self):
|
|
103
|
-
return self.local.get("index") or "none"
|
|
104
|
-
|
|
105
|
-
@property
|
|
106
|
-
def packages(self):
|
|
107
|
-
return self.mode == "packages"
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
def read_json(path):
|
|
111
|
-
with open(path, encoding="utf-8") as fh:
|
|
112
|
-
return json.load(fh)
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
def load_config(root):
|
|
116
|
-
path = os.path.join(root, POLICY_REL)
|
|
117
|
-
present = os.path.exists(path)
|
|
118
|
-
policy = dict(DEFAULTS)
|
|
119
|
-
if present:
|
|
120
|
-
try:
|
|
121
|
-
loaded = read_json(path)
|
|
122
|
-
except (OSError, ValueError) as exc:
|
|
123
|
-
raise MemoryError_("%s is not valid JSON (%s): fix it, or 'state.py memory disable'"
|
|
124
|
-
% (POLICY_REL, exc)) from exc
|
|
125
|
-
if not isinstance(loaded, dict):
|
|
126
|
-
raise MemoryError_("%s must be a JSON object" % POLICY_REL)
|
|
127
|
-
schema = loaded.get("schema", 1)
|
|
128
|
-
if isinstance(schema, int) and not isinstance(schema, bool) and schema > SCHEMA:
|
|
129
|
-
raise MemoryError_("%s has schema %s; this state.py knows %d — update the plugin"
|
|
130
|
-
% (POLICY_REL, schema, SCHEMA))
|
|
131
|
-
policy.update(loaded)
|
|
132
|
-
check_policy(policy)
|
|
133
|
-
local = {}
|
|
134
|
-
with contextlib.suppress(OSError, ValueError):
|
|
135
|
-
found = read_json(os.path.join(root, LOCAL_REL, "config.json"))
|
|
136
|
-
if isinstance(found, dict):
|
|
137
|
-
local = found
|
|
138
|
-
where = os.path.join(LOCAL_REL, "config.json")
|
|
139
|
-
if local.get("author") is not None and not (isinstance(local["author"], str)
|
|
140
|
-
and mf.AUTHOR_RE.match(local["author"])):
|
|
141
|
-
raise MemoryError_("%s: author %r is not a slug of [a-z0-9-]" % (where, local["author"]))
|
|
142
|
-
if local.get("index", "none") not in ("none", "sqlite"):
|
|
143
|
-
raise MemoryError_("%s: index takes none or sqlite" % where)
|
|
144
|
-
return MemoryConfig(root, policy, local, present)
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
def check_policy(policy):
|
|
148
|
-
"""Every key of memory.json with its type (§I6): a wrong one is a refusal
|
|
149
|
-
naming the key, never a traceback inside close."""
|
|
150
|
-
def is_int(value, low):
|
|
151
|
-
return isinstance(value, int) and not isinstance(value, bool) and value >= low
|
|
152
|
-
problems = []
|
|
153
|
-
if not is_int(policy.get("schema"), 1):
|
|
154
|
-
problems.append("schema must be an integer (1)")
|
|
155
|
-
if policy.get("mode") not in ("files", "packages"):
|
|
156
|
-
problems.append('mode must be "files" or "packages"')
|
|
157
|
-
if policy.get("period") not in mf.PERIODS:
|
|
158
|
-
problems.append("period must be month, quarter or year")
|
|
159
|
-
if not is_int(policy.get("package_max_bytes"), 1024):
|
|
160
|
-
problems.append("package_max_bytes must be an integer of at least 1024")
|
|
161
|
-
if not isinstance(policy.get("cleanup"), bool):
|
|
162
|
-
problems.append("cleanup must be true or false")
|
|
163
|
-
if not is_int(policy.get("excerpt_max_bytes"), 0):
|
|
164
|
-
problems.append("excerpt_max_bytes must be an integer >= 0")
|
|
165
|
-
if policy.get("keep_rules") != 1:
|
|
166
|
-
problems.append("keep_rules must be 1")
|
|
167
|
-
recall = policy.get("recall")
|
|
168
|
-
if not (isinstance(recall, dict) and is_int(recall.get("budget_tokens", 1500), 1)
|
|
169
|
-
and is_int(recall.get("max_items", 40), 1)):
|
|
170
|
-
problems.append('recall must be {"budget_tokens": N, "max_items": N}')
|
|
171
|
-
if problems:
|
|
172
|
-
raise MemoryError_("%s: %s" % (POLICY_REL, "; ".join(problems)))
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
def mode_of(root):
|
|
176
|
-
"""'files' or 'packages' without raising: close must archive whatever happens."""
|
|
177
|
-
try:
|
|
178
|
-
return load_config(root).mode
|
|
179
|
-
except MemoryError_:
|
|
180
|
-
return "files"
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
# ------------------------------------------------------------------ small I/O
|
|
184
|
-
def atomic_write(path, data):
|
|
185
|
-
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
186
|
-
handle, tmp = tempfile.mkstemp(prefix=".tmp-", dir=os.path.dirname(path))
|
|
187
|
-
try:
|
|
188
|
-
with os.fdopen(handle, "wb") as fh:
|
|
189
|
-
fh.write(data if isinstance(data, bytes) else data.encode("utf-8"))
|
|
190
|
-
fh.flush()
|
|
191
|
-
os.fsync(fh.fileno())
|
|
192
|
-
os.replace(tmp, path)
|
|
193
|
-
except BaseException:
|
|
194
|
-
with contextlib.suppress(OSError):
|
|
195
|
-
os.remove(tmp)
|
|
196
|
-
raise
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
@contextlib.contextmanager
|
|
200
|
-
def locked(root):
|
|
201
|
-
"""flock on .ai/local/memory/lock: two closes in one clone serialise here."""
|
|
202
|
-
os.makedirs(os.path.join(root, LOCAL_REL), exist_ok=True)
|
|
203
|
-
fd = os.open(os.path.join(root, LOCAL_REL, "lock"), os.O_RDWR | os.O_CREAT, 0o644)
|
|
204
|
-
try:
|
|
205
|
-
if fcntl is not None:
|
|
206
|
-
fcntl.flock(fd, fcntl.LOCK_EX)
|
|
207
|
-
yield
|
|
208
|
-
finally:
|
|
209
|
-
os.close(fd)
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
def git(root, *args, stdin=None, binary=False):
|
|
213
|
-
done = subprocess.run(["git", "-C", root, *args], input=stdin, capture_output=True, check=False,
|
|
214
|
-
text=not binary)
|
|
215
|
-
return done.returncode, done.stdout, done.stderr
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
def current_branch(root):
|
|
219
|
-
code, out, _ = git(root, "symbolic-ref", "--short", "-q", "HEAD")
|
|
220
|
-
if code == 0 and out.strip():
|
|
221
|
-
return out.strip()
|
|
222
|
-
code, out, _ = git(root, "rev-parse", "--short=8", "HEAD")
|
|
223
|
-
return "HEAD@%s" % (out.strip() if code == 0 and out.strip() else "unborn")
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
def memory_dir(root):
|
|
227
|
-
return os.path.join(root, MEMORY_REL)
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
def local_path(root, *parts):
|
|
231
|
-
return os.path.join(root, LOCAL_REL, *parts)
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
def read_bytes(path):
|
|
235
|
-
try:
|
|
236
|
-
with open(path, "rb") as fh:
|
|
237
|
-
return fh.read()
|
|
238
|
-
except OSError:
|
|
239
|
-
return None
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
# ------------------------------------------------------------------ packages on disk
|
|
243
|
-
def load_packages(root):
|
|
244
|
-
"""{name: (header, entries, errors)} for every package in the working tree."""
|
|
245
|
-
out = {}
|
|
246
|
-
base = memory_dir(root)
|
|
247
|
-
for name in sorted(os.listdir(base)) if os.path.isdir(base) else []:
|
|
248
|
-
if name.endswith(".jsonl"):
|
|
249
|
-
out[name] = mf.parse_package(read_bytes(os.path.join(base, name)) or b"", name)
|
|
250
|
-
return out
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
def all_rids(packages):
|
|
254
|
-
return {e["rid"] for _, entries, _ in packages.values() for e in entries}
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
def seals_by_key(packages):
|
|
258
|
-
out = collections.defaultdict(list)
|
|
259
|
-
for name, (_, entries, _) in packages.items():
|
|
260
|
-
for e in entries:
|
|
261
|
-
if e.get("kind") == "task":
|
|
262
|
-
out[mf.key_of(e)].append((name, e))
|
|
263
|
-
return out
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
def read_batches(root):
|
|
267
|
-
with contextlib.suppress(OSError, ValueError):
|
|
268
|
-
data = read_json(local_path(root, "batches.json"))
|
|
269
|
-
if isinstance(data, dict) and isinstance(data.get("open"), dict):
|
|
270
|
-
return data
|
|
271
|
-
return {"schema": 1, "open": {}}
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
def write_batches(root, data):
|
|
275
|
-
atomic_write(local_path(root, "batches.json"), json.dumps(data, sort_keys=True, indent=1) + "\n")
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
def new_batch_name(root, period, author):
|
|
279
|
-
rng = random.SystemRandom()
|
|
280
|
-
for _ in range(64):
|
|
281
|
-
batch = "".join(rng.choice(BATCH_CHARS) for _ in range(4))
|
|
282
|
-
name = mf.package_name(period, author, batch)
|
|
283
|
-
if not os.path.exists(os.path.join(memory_dir(root), name)):
|
|
284
|
-
return name
|
|
285
|
-
raise MemoryError_("no free batch name for %s.%s" % (period, author), 3)
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
def open_batch(root, cfg, stamp=None):
|
|
289
|
-
"""The branch's open batch, reused while it is in the tree, the author's,
|
|
290
|
-
of the current period and under package_max_bytes; else a new name.
|
|
291
|
-
Called under the lock."""
|
|
292
|
-
stamp = stamp or now()
|
|
293
|
-
period = mf.period_of(stamp, cfg.period)
|
|
294
|
-
branch = current_branch(root)
|
|
295
|
-
books = read_batches(root)
|
|
296
|
-
name = books["open"].get(branch)
|
|
297
|
-
if name:
|
|
298
|
-
parts = mf.parse_package_name(name)
|
|
299
|
-
path = os.path.join(memory_dir(root), name)
|
|
300
|
-
header = None
|
|
301
|
-
data = read_bytes(path)
|
|
302
|
-
if data:
|
|
303
|
-
header, _, _ = mf.parse_package(data.split(b"\n", 1)[0] + b"\n", name)
|
|
304
|
-
if parts and data and header and header.get("author") == cfg.author and parts["period"] == period \
|
|
305
|
-
and parts["batch"] and len(data) < cfg.package_max_bytes:
|
|
306
|
-
return name, branch, books
|
|
307
|
-
name = new_batch_name(root, period, cfg.author)
|
|
308
|
-
return name, branch, books
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
# ------------------------------------------------------------------ export (§D5)
|
|
312
|
-
def outbox_path(root, key):
|
|
313
|
-
task, inst = key.rsplit("#", 1)
|
|
314
|
-
return local_path(root, "outbox", "%s~%s.jsonl" % (task, inst))
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
def exportable(compactions):
|
|
318
|
-
return [c for c in compactions if c.seal and c.seal["result"] in ("closed", "abandoned", "restarted")]
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
ExportResult = collections.namedtuple("ExportResult",
|
|
322
|
-
"package added sealed coverage failures conflicts already added_by_key")
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
def compact_for_export(root, task_id, cfg):
|
|
326
|
-
found = mc.compact_instances(FsReader(root), task_id, "close", cfg.excerpt_max_bytes)
|
|
327
|
-
problems = [p for c in found for p in c.problems if "torn journal" not in p]
|
|
328
|
-
if problems:
|
|
329
|
-
raise MemoryError_(problems[0], 3)
|
|
330
|
-
ready = exportable(found)
|
|
331
|
-
if not ready:
|
|
332
|
-
raise MemoryError_("%s has no closed instance to export" % task_id, 3)
|
|
333
|
-
return ready
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
def write_outbox(root, compaction):
|
|
337
|
-
key = mf.key_of(compaction.seal)
|
|
338
|
-
lines = [mf.canonical(e) for e in compaction.entries + [compaction.seal]]
|
|
339
|
-
atomic_write(outbox_path(root, key), b"\n".join(lines) + b"\n")
|
|
340
|
-
return key
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
def read_outbox(root):
|
|
344
|
-
base = local_path(root, "outbox")
|
|
345
|
-
out = {}
|
|
346
|
-
for name in sorted(os.listdir(base)) if os.path.isdir(base) else []:
|
|
347
|
-
if not name.endswith(".jsonl"):
|
|
348
|
-
continue
|
|
349
|
-
entries = []
|
|
350
|
-
for raw in (read_bytes(os.path.join(base, name)) or b"").split(b"\n"):
|
|
351
|
-
if raw.strip():
|
|
352
|
-
with contextlib.suppress(ValueError, UnicodeDecodeError):
|
|
353
|
-
entries.append(json.loads(raw.decode("utf-8")))
|
|
354
|
-
seals = [e for e in entries if e.get("kind") == "task"]
|
|
355
|
-
if seals:
|
|
356
|
-
out[mf.key_of(seals[-1])] = entries
|
|
357
|
-
return out
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
def export(root, ids, cfg=None, stamp=None):
|
|
361
|
-
"""B2–B4 for every id (or the pending ones); returns ExportResult. A
|
|
362
|
-
failure of one task keeps its outbox and is reported, never raised."""
|
|
363
|
-
cfg = cfg or load_config(root)
|
|
364
|
-
if not cfg.packages:
|
|
365
|
-
raise MemoryError_("files mode: nothing is exported ('state.py memory enable' turns packages on)")
|
|
366
|
-
failures, coverage, sealed, already, added_by_key = [], {}, [], [], {}
|
|
367
|
-
if ids == "pending":
|
|
368
|
-
ids = pending_ids(root, cfg)
|
|
369
|
-
ids = list(ids)
|
|
370
|
-
# A key with a seal in the tree is never sealed again from other sources: a
|
|
371
|
-
# second compaction of changed local sources (a restarted id, a cleaned-up
|
|
372
|
-
# directory) would add a degraded seal to an append-only record (R8, R9).
|
|
373
|
-
# The same seal is a repair: the rids it lacks come back, nothing else (V4, V5).
|
|
374
|
-
sealed_here = seals_by_key(load_packages(root))
|
|
375
|
-
for task_id in ids:
|
|
376
|
-
try:
|
|
377
|
-
for compaction in compact_for_export(root, task_id, cfg):
|
|
378
|
-
key = mf.key_of(compaction.seal)
|
|
379
|
-
coverage[key] = compaction.coverage
|
|
380
|
-
stored = {s["rid"] for _, s in sealed_here.get(key, [])}
|
|
381
|
-
if stored and compaction.seal["rid"] not in stored:
|
|
382
|
-
already.append(key)
|
|
383
|
-
continue
|
|
384
|
-
write_outbox(root, compaction)
|
|
385
|
-
except MemoryError_ as exc:
|
|
386
|
-
failures.append((task_id, str(exc)))
|
|
387
|
-
except OSError as exc:
|
|
388
|
-
failures.append((task_id, "the outbox could not be written (%s)" % exc))
|
|
389
|
-
crash("B2")
|
|
390
|
-
added, conflicts, name = 0, [], None
|
|
391
|
-
try:
|
|
392
|
-
with locked(root):
|
|
393
|
-
outbox = read_outbox(root) # under the lock: another close may have taken it
|
|
394
|
-
packages = load_packages(root)
|
|
395
|
-
have = all_rids(packages)
|
|
396
|
-
seals = seals_by_key(packages)
|
|
397
|
-
recorded = {e.get("key"): set(e.get("rids") or []) for _, entries, _ in packages.values()
|
|
398
|
-
for e in entries if e.get("kind") == "conflict"}
|
|
399
|
-
new, keep = [], {}
|
|
400
|
-
for key, entries in outbox.items():
|
|
401
|
-
seal_rid = [e for e in entries if e.get("kind") == "task"][-1]["rid"]
|
|
402
|
-
stored = {s["rid"] for _, s in seals.get(key, [])}
|
|
403
|
-
missing = [e for e in entries if e["rid"] not in have]
|
|
404
|
-
if (stored and seal_rid not in stored) or not missing:
|
|
405
|
-
# Sealed meanwhile from other sources, or nothing lacking (a re-run, B4).
|
|
406
|
-
os.remove(outbox_path(root, key))
|
|
407
|
-
if key not in already:
|
|
408
|
-
already.append(key)
|
|
409
|
-
continue
|
|
410
|
-
new += missing
|
|
411
|
-
added_by_key[key] = len(missing)
|
|
412
|
-
keep[key] = entries
|
|
413
|
-
for key in coverage:
|
|
414
|
-
if key in seals and key not in keep and key not in already:
|
|
415
|
-
already.append(key) # another close took this outbox and sealed it (review L3)
|
|
416
|
-
for key, found in sorted(seals.items()):
|
|
417
|
-
# Two seals that arrived by a merge (one id closed in two clones):
|
|
418
|
-
# recorded side by side, never merged (R9).
|
|
419
|
-
rids = sorted({s["rid"] for _, s in found} - {s.get("supersedes") for _, s in found})
|
|
420
|
-
if len(rids) < 2 or key.rsplit("#", 1)[0] not in ids or set(rids) <= recorded.get(key, set()):
|
|
421
|
-
continue
|
|
422
|
-
seal = found[-1][1]
|
|
423
|
-
conflict = mf.with_rid({
|
|
424
|
-
"v": mf.FORMAT, "task": seal["task"], "inst": seal["inst"], "seq": seal["seq"] + 1,
|
|
425
|
-
"kind": "conflict", "ts": max(s["ts"] for _, s in found), "actor": "agent", "by": None,
|
|
426
|
-
"runtime": seal["runtime"], "src": ["export"], "key": key, "rids": rids,
|
|
427
|
-
"reason": "two different seals for one key — kept side by side, never merged"})
|
|
428
|
-
if conflict["rid"] not in have:
|
|
429
|
-
new.append(conflict)
|
|
430
|
-
conflicts.append(key)
|
|
431
|
-
if not new:
|
|
432
|
-
return ExportResult(None, 0, sealed, coverage, failures, conflicts, already, added_by_key)
|
|
433
|
-
name, branch, books = open_batch(root, cfg, stamp)
|
|
434
|
-
path = os.path.join(memory_dir(root), name)
|
|
435
|
-
header, current, _ = packages.get(name, (None, [], []))
|
|
436
|
-
parts = mf.parse_package_name(name)
|
|
437
|
-
header = header or mf.package_header(parts["period"], cfg.author, parts["batch"], stamp or now())
|
|
438
|
-
current = [mf.strip_private(e) for e in current]
|
|
439
|
-
outbox = keep
|
|
440
|
-
if new:
|
|
441
|
-
data = mf.render_package(header, current + new)
|
|
442
|
-
crash("B3")
|
|
443
|
-
atomic_write(path, data)
|
|
444
|
-
added = len(new)
|
|
445
|
-
books["open"][branch] = name
|
|
446
|
-
write_batches(root, books)
|
|
447
|
-
crash("B4")
|
|
448
|
-
check = load_packages(root)
|
|
449
|
-
present = all_rids(check)
|
|
450
|
-
for key, entries in outbox.items():
|
|
451
|
-
seal = [e for e in entries if e.get("kind") == "task"][-1]
|
|
452
|
-
if all(e["rid"] in present for e in entries):
|
|
453
|
-
os.remove(outbox_path(root, key))
|
|
454
|
-
sealed.append(key)
|
|
455
|
-
else:
|
|
456
|
-
failures.append((seal["task"], "read-back found %d of %d entries" % (
|
|
457
|
-
sum(e["rid"] in present for e in entries), len(entries))))
|
|
458
|
-
except OSError as exc:
|
|
459
|
-
failures.append((",".join(sorted(coverage)) or "the outbox",
|
|
460
|
-
"%s (%s)" % (exc.strerror or exc, getattr(exc, "filename", ""))))
|
|
461
|
-
if added and cfg.index == "sqlite":
|
|
462
|
-
crash("B5")
|
|
463
|
-
index_import(root, cfg)
|
|
464
|
-
return ExportResult(name, added, sealed, coverage, failures, conflicts, already, added_by_key)
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
def record_lines(result):
|
|
468
|
-
"""What close and `memory export` print: each key newly sealed with what it
|
|
469
|
-
added, each key that was already sealed, each conflict recorded (review L5)."""
|
|
470
|
-
out = []
|
|
471
|
-
for key in result.sealed:
|
|
472
|
-
cov = result.coverage.get(key) or {}
|
|
473
|
-
out.append("record: %s (+%d entries, %s sealed, coverage %s/%s lines)" % (
|
|
474
|
-
os.path.join(MEMORY_REL, result.package or "-"), result.added_by_key.get(key, 0),
|
|
475
|
-
key.rsplit("#", 1)[0], cov.get("accounted", "?"), cov.get("lines", "?")))
|
|
476
|
-
for key in result.already:
|
|
477
|
-
out.append("record: %s already sealed in %s — nothing added" % (key, MEMORY_REL))
|
|
478
|
-
for key in result.conflicts:
|
|
479
|
-
out.append("conflict: %s has two different seals — recorded side by side, never merged "
|
|
480
|
-
"('state.py memory show %s')" % (key, key.rsplit("#", 1)[0]))
|
|
481
|
-
return out
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
def index_import(root, cfg):
|
|
485
|
-
"""B5: the index learns the new package; any failure marks it stale."""
|
|
486
|
-
try:
|
|
487
|
-
import memory_store # pylint: disable=import-outside-toplevel
|
|
488
|
-
memory_store.SqliteHistory(root, cfg).refresh()
|
|
489
|
-
except Exception: # pylint: disable=broad-except # a stale index is rebuilt by the next read
|
|
490
|
-
with contextlib.suppress(Exception):
|
|
491
|
-
import memory_store # pylint: disable=import-outside-toplevel
|
|
492
|
-
memory_store.mark_stale(root)
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
def sealed_task_ids(packages):
|
|
496
|
-
return {e.get("task") for _, entries, _ in packages.values() for e in entries if e.get("kind") == "task"}
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
def pending_ids(root, cfg=None):
|
|
500
|
-
"""Closed new-shape tasks (and anything in the outbox) without a seal:
|
|
501
|
-
B1 left them closed but not exported. Legacy ids wait for the migration."""
|
|
502
|
-
sealed = sealed_task_ids(load_packages(root))
|
|
503
|
-
found = set(k.rsplit("#", 1)[0] for k in read_outbox(root))
|
|
504
|
-
logs = os.path.join(root, ".ai", "state", "logs")
|
|
505
|
-
for name in os.listdir(logs) if os.path.isdir(logs) else []:
|
|
506
|
-
if not mf.NEW_ID_RE.match(name) or name in sealed:
|
|
507
|
-
continue
|
|
508
|
-
with contextlib.suppress(OSError, ValueError):
|
|
509
|
-
state = read_json(os.path.join(logs, name, "state.json"))
|
|
510
|
-
if state.get("result") in ("closed", "abandoned"):
|
|
511
|
-
found.add(name)
|
|
512
|
-
del cfg
|
|
513
|
-
return sorted(found - sealed)
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
def after_close(root, task_id, out=print):
|
|
517
|
-
"""The one call close makes after archive_v7. Packages mode only; every
|
|
518
|
-
failure is one printed line and close still exits 0 (B4)."""
|
|
519
|
-
try:
|
|
520
|
-
cfg = load_config(root)
|
|
521
|
-
except MemoryError_ as exc:
|
|
522
|
-
out("memory: %s — the task is closed in files mode" % exc)
|
|
523
|
-
return None
|
|
524
|
-
if not cfg.packages:
|
|
525
|
-
return None
|
|
526
|
-
try:
|
|
527
|
-
pending = [t for t in pending_ids(root, cfg) if t != task_id][:PENDING_MAX]
|
|
528
|
-
result = export(root, [task_id] + pending, cfg)
|
|
529
|
-
except (MemoryError_, ValueError, OSError) as exc:
|
|
530
|
-
out("memory: export of %s failed (%s) — the task is closed; 'state.py memory export %s' retries"
|
|
531
|
-
% (task_id, exc, task_id))
|
|
532
|
-
return None
|
|
533
|
-
for failed, why in result.failures:
|
|
534
|
-
out("memory: export of %s failed (%s) — the task is closed; 'state.py memory export %s' retries"
|
|
535
|
-
% (failed, why, failed))
|
|
536
|
-
for line in record_lines(result):
|
|
537
|
-
out(line)
|
|
538
|
-
if cfg.cleanup and any(k.rsplit("#", 1)[0] == task_id for k in result.sealed + result.already):
|
|
539
|
-
crash("B6")
|
|
540
|
-
try:
|
|
541
|
-
done = cleanup_task(root, task_id, cfg)
|
|
542
|
-
out("cleanup: %s" % done)
|
|
543
|
-
except MemoryError_ as exc:
|
|
544
|
-
out("cleanup: %s kept in .ai/reports/ (%s)" % (task_id, exc))
|
|
545
|
-
return result
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
# ------------------------------------------------------------------ verify (§D6)
|
|
549
|
-
Finding = collections.namedtuple("Finding", "check path line what fix")
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
def kept_lines(events, schema=None):
|
|
553
|
-
"""The journal lines V5 requires to be the src of an entry."""
|
|
554
|
-
out = []
|
|
555
|
-
for number, event in events:
|
|
556
|
-
if not event:
|
|
557
|
-
continue
|
|
558
|
-
if event.get("event") == "task_started":
|
|
559
|
-
schema = (event.get("data") or {}).get("schema", schema)
|
|
560
|
-
rule = mf.classify(event.get("event"), schema)
|
|
561
|
-
if rule not in ("seal", "summary", "drop", "verify"):
|
|
562
|
-
out.append(number)
|
|
563
|
-
return out
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
def check_packages(packages, max_bytes, sizes=None, changed=None, ref_name=None):
|
|
567
|
-
"""V1, V3, V4, V8 over one tree's packages."""
|
|
568
|
-
findings = []
|
|
569
|
-
sizes = sizes or {}
|
|
570
|
-
for name, (_, _, errors) in sorted(packages.items()):
|
|
571
|
-
if changed is not None and name not in changed:
|
|
572
|
-
continue
|
|
573
|
-
for line, what in errors:
|
|
574
|
-
findings.append(Finding("V1", os.path.join(MEMORY_REL, name), line, what,
|
|
575
|
-
"restore the line from git ('git checkout -- %s') or 'state.py memory verify'"
|
|
576
|
-
% os.path.join(MEMORY_REL, name)))
|
|
577
|
-
by_key = collections.defaultdict(list)
|
|
578
|
-
conflicts = set()
|
|
579
|
-
redacted = {}
|
|
580
|
-
for name, (_, entries, _) in packages.items():
|
|
581
|
-
for e in entries:
|
|
582
|
-
by_key[mf.key_of(e)].append((name, e))
|
|
583
|
-
if e.get("kind") == "conflict":
|
|
584
|
-
conflicts.add((e.get("key"), tuple(e.get("rids") or [])))
|
|
585
|
-
if e.get("kind") == "redaction" and e.get("replacement"):
|
|
586
|
-
redacted[e["replacement"]] = e.get("target")
|
|
587
|
-
for key, items in sorted(by_key.items()):
|
|
588
|
-
seals = [(n, e) for n, e in items if e.get("kind") == "task"]
|
|
589
|
-
unique = {e["rid"]: (n, e) for n, e in seals}
|
|
590
|
-
chained = {e.get("supersedes") for _, e in unique.values() if e.get("supersedes")}
|
|
591
|
-
live = [rid for rid in unique if rid not in chained]
|
|
592
|
-
if len(live) > 1:
|
|
593
|
-
task = key.rsplit("#", 1)[0]
|
|
594
|
-
recorded = any(k == key and set(live) <= set(r) for k, r in conflicts)
|
|
595
|
-
if mf.NEW_ID_RE.match(task) and not recorded:
|
|
596
|
-
findings.append(Finding("V3", os.path.join(MEMORY_REL, unique[live[0]][0]), None,
|
|
597
|
-
"%s has %d seals not chained by supersedes" % (key, len(live)),
|
|
598
|
-
"'state.py memory export %s' records the two as a conflict" % task))
|
|
599
|
-
elif not recorded:
|
|
600
|
-
findings.append(Finding("V3", os.path.join(MEMORY_REL, unique[live[0]][0]), None,
|
|
601
|
-
"legacy %s has %d seals and no conflict entry" % (key, len(live)),
|
|
602
|
-
"'state.py memory export %s' records the conflict" % task))
|
|
603
|
-
rids = {e["rid"] for _, e in items if e.get("kind") not in ("task", "conflict", "redaction")}
|
|
604
|
-
if len(live) > 1:
|
|
605
|
-
continue # a recorded conflict: which entries are whose is not knowable
|
|
606
|
-
for rid in live:
|
|
607
|
-
name, seal = unique[rid]
|
|
608
|
-
own = sorted({redacted.get(r, r) for r in rids if _belongs(items, r, seal)})
|
|
609
|
-
if len(own) != seal.get("entries") or mf.seal_digest(own) != seal.get("digest"):
|
|
610
|
-
findings.append(Finding("V4", os.path.join(MEMORY_REL, name), seal.get("_line"),
|
|
611
|
-
"%s: the seal names %s entries (%s), the package has %d" % (
|
|
612
|
-
key, seal.get("entries"), (seal.get("digest") or "")[:19], len(own)),
|
|
613
|
-
"'state.py memory export %s' re-exports it" % seal.get("task")))
|
|
614
|
-
for name, size in sorted(sizes.items()):
|
|
615
|
-
if size > 2 * max_bytes:
|
|
616
|
-
findings.append(Finding("V8", os.path.join(MEMORY_REL, name), None,
|
|
617
|
-
"%d bytes, over twice package_max_bytes (%d)" % (size, max_bytes),
|
|
618
|
-
"a new batch opens on the next close; 'state.py memory pack' folds old ones"))
|
|
619
|
-
elif size > max_bytes:
|
|
620
|
-
findings.append(Finding("V8-warn", os.path.join(MEMORY_REL, name), None,
|
|
621
|
-
"%d bytes, over package_max_bytes (%d)" % (size, max_bytes),
|
|
622
|
-
"nothing to do now; the next close opens a new batch"))
|
|
623
|
-
del ref_name
|
|
624
|
-
return findings
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
def _belongs(items, rid, seal):
|
|
628
|
-
"""Entries of a key that came before its seal (seq below the seal's)."""
|
|
629
|
-
for _, e in items:
|
|
630
|
-
if e["rid"] == rid:
|
|
631
|
-
return (e.get("seq") or 0) < (seal.get("seq") or 0)
|
|
632
|
-
return False
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
def check_journals(read, paths, packages, cleanup_on=False):
|
|
636
|
-
"""V5 (and V7 with cleanup) against the committed journals."""
|
|
637
|
-
findings = []
|
|
638
|
-
seals = seals_by_key(packages)
|
|
639
|
-
sealed_tasks = {k.rsplit("#", 1)[0] for k in seals}
|
|
640
|
-
srcs = collections.defaultdict(set)
|
|
641
|
-
for _, entries, _ in packages.values():
|
|
642
|
-
for e in entries:
|
|
643
|
-
for s in e.get("src") or []:
|
|
644
|
-
srcs[mf.key_of(e)].add(s)
|
|
645
|
-
journals = sorted(p for p in paths if re.match(r"^\.ai/reports/[^/]+/events\.jsonl$", p))
|
|
646
|
-
for path in journals:
|
|
647
|
-
task = path.split("/")[2]
|
|
648
|
-
data = read(path) or b""
|
|
649
|
-
if b'"task_closed"' not in data:
|
|
650
|
-
continue
|
|
651
|
-
if not (mf.NEW_ID_RE.match(task) or task in sealed_tasks):
|
|
652
|
-
continue # an unmigrated legacy task: 'memory status' lists it
|
|
653
|
-
events = mc.parse_journal(data)
|
|
654
|
-
instances = mc.split_instances(events)
|
|
655
|
-
for inst, lines in enumerate(instances, 1):
|
|
656
|
-
key = "%s#%d" % (task, inst)
|
|
657
|
-
closed = any(e and e.get("event") == "task_closed" for _, e in lines)
|
|
658
|
-
if not closed and inst == len(instances):
|
|
659
|
-
continue
|
|
660
|
-
if key not in seals:
|
|
661
|
-
findings.append(Finding("V5", path, None, "%s is closed but has no seal" % key,
|
|
662
|
-
"state.py memory export %s" % task))
|
|
663
|
-
continue
|
|
664
|
-
missing = [n for n in kept_lines(lines) if "events.jsonl:%d" % n not in srcs[key]]
|
|
665
|
-
if missing:
|
|
666
|
-
findings.append(Finding("V5", path, missing[0], "%s: %d kept line(s) are no entry's src" % (
|
|
667
|
-
key, len(missing)), "state.py memory export %s" % task))
|
|
668
|
-
if cleanup_on and task in sealed_tasks:
|
|
669
|
-
findings.append(Finding("V7", "/".join(path.split("/")[:3]) + "/", None,
|
|
670
|
-
"cleanup is on and %s is sealed" % task, "state.py memory cleanup %s" % task))
|
|
671
|
-
return findings
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
def verify(root, scope="all", ids=None):
|
|
675
|
-
"""Findings for --all or ids over the working tree; [] is green."""
|
|
676
|
-
cfg = load_config(root)
|
|
677
|
-
packages = load_packages(root)
|
|
678
|
-
sizes = {n: os.path.getsize(os.path.join(memory_dir(root), n)) for n in packages}
|
|
679
|
-
findings = check_packages(packages, cfg.package_max_bytes, sizes)
|
|
680
|
-
reports = os.path.join(root, ".ai", "reports")
|
|
681
|
-
paths = []
|
|
682
|
-
for name in sorted(os.listdir(reports)) if os.path.isdir(reports) else []:
|
|
683
|
-
if ids and name not in ids:
|
|
684
|
-
continue
|
|
685
|
-
if os.path.exists(os.path.join(reports, name, "events.jsonl")):
|
|
686
|
-
paths.append(".ai/reports/%s/events.jsonl" % name)
|
|
687
|
-
findings += check_journals(lambda rel: read_bytes(os.path.join(root, rel)), paths, packages, False)
|
|
688
|
-
if ids:
|
|
689
|
-
findings = [f for f in findings if any(i in (f.path or "") or i in f.what for i in ids)]
|
|
690
|
-
del scope
|
|
691
|
-
return findings
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
def verify_fresh_clone(root):
|
|
695
|
-
"""§D6: a local clone of HEAD, rebuilt from packages alone, shows every
|
|
696
|
-
sealed key as this clone does."""
|
|
697
|
-
import memory_store # pylint: disable=import-outside-toplevel
|
|
698
|
-
findings = []
|
|
699
|
-
here = memory_store.FileHistory(root, load_config(root))
|
|
700
|
-
keys = sorted({mf.key_of(e) for _, entries, _ in load_packages(root).values() for e in entries
|
|
701
|
-
if e.get("kind") == "task"})
|
|
702
|
-
with tempfile.TemporaryDirectory(prefix="ai-memory-clone-") as tmp:
|
|
703
|
-
clone = os.path.join(tmp, "clone")
|
|
704
|
-
code, _, err = git(root, "clone", "-q", "--no-hardlinks", root, clone)
|
|
705
|
-
if code != 0:
|
|
706
|
-
raise MemoryError_("git clone failed: %s" % err.strip(), 3)
|
|
707
|
-
there = memory_store.FileHistory(clone, load_config(clone))
|
|
708
|
-
for key in keys:
|
|
709
|
-
a = "\n".join(mf.render_task_md(here.entries(key)))
|
|
710
|
-
b = "\n".join(mf.render_task_md(there.entries(key)))
|
|
711
|
-
if a != b:
|
|
712
|
-
findings.append(Finding("fresh-clone", key, None, "renders differently in a fresh clone",
|
|
713
|
-
"commit .ai/memory/ (the clone sees HEAD only)"))
|
|
714
|
-
return findings
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
def format_finding(f, ref=None):
|
|
718
|
-
where = f.path + (":%s" % f.line if f.line else "")
|
|
719
|
-
return "%s%s: %s: %s — fix: %s" % ("%s: " % ref if ref else "", f.check, where, f.what, f.fix)
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
# ------------------------------------------------------------------ status, show
|
|
723
|
-
def hook_state(root):
|
|
724
|
-
code, out, _ = git(root, "config", "--get", "core.hooksPath")
|
|
725
|
-
if code == 0 and out.strip():
|
|
726
|
-
return "hookspath"
|
|
727
|
-
path = hooks_path(root)
|
|
728
|
-
if not path:
|
|
729
|
-
return "absent"
|
|
730
|
-
hook = os.path.join(path, "pre-push")
|
|
731
|
-
text = read_bytes(hook)
|
|
732
|
-
if text is None:
|
|
733
|
-
return "absent"
|
|
734
|
-
if HOOK_MARKER.encode() in text:
|
|
735
|
-
decoded = text.decode("utf-8", "replace")
|
|
736
|
-
if not any(os.path.isfile(p) for p in shim_paths(decoded)):
|
|
737
|
-
return "installed, validator missing — pushes are NOT checked ('state.py memory enable' rewrites it)"
|
|
738
|
-
return "current" if decoded == shim_text() else "installed (stale)"
|
|
739
|
-
return "foreign"
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
def status_lines(root):
|
|
743
|
-
cfg = load_config(root)
|
|
744
|
-
packages = load_packages(root)
|
|
745
|
-
size = sum(os.path.getsize(os.path.join(memory_dir(root), n)) for n in packages)
|
|
746
|
-
books = read_batches(root)
|
|
747
|
-
branch = current_branch(root)
|
|
748
|
-
pending = pending_ids(root, cfg) if cfg.packages else []
|
|
749
|
-
legacy = unmigrated(root, packages)
|
|
750
|
-
lines = ["mode: %s%s" % (cfg.mode, "" if cfg.present else " (no %s)" % POLICY_REL),
|
|
751
|
-
"period: %s · author: %s · index: %s%s" % (cfg.period, cfg.author, cfg.index, fts_line(cfg)),
|
|
752
|
-
"open batch: %s" % (books["open"].get(branch) or "none") + " (branch %s)" % branch,
|
|
753
|
-
"packages: %d · %s bytes" % (len(packages), mf.fmt_n(size)),
|
|
754
|
-
"closed, not exported: %s" % (", ".join(pending) if pending else "none"),
|
|
755
|
-
"legacy, not migrated: %d%s" % (len(legacy), " — state.py memory migrate --report" if legacy else ""),
|
|
756
|
-
"pre-push: %s" % hook_state(root), "cleanup: %s" % ("on" if cfg.cleanup else "off")]
|
|
757
|
-
return lines
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
def fts_line(cfg):
|
|
761
|
-
if cfg.index != "sqlite":
|
|
762
|
-
return ""
|
|
763
|
-
import memory_store # pylint: disable=import-outside-toplevel
|
|
764
|
-
if memory_store.fts5_available():
|
|
765
|
-
return " · fts5: yes"
|
|
766
|
-
import sqlite3 # pylint: disable=import-outside-toplevel
|
|
767
|
-
return " · FTS5 unavailable in this SQLite %s: search scans; results identical, slower" % sqlite3.sqlite_version
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
def unmigrated(root, packages):
|
|
771
|
-
sealed = sealed_task_ids(packages)
|
|
772
|
-
reports = os.path.join(root, ".ai", "reports")
|
|
773
|
-
out = []
|
|
774
|
-
for name in sorted(os.listdir(reports)) if os.path.isdir(reports) else []:
|
|
775
|
-
path = os.path.join(reports, name, "events.jsonl")
|
|
776
|
-
if mf.LEGACY_ID_RE.match(name) and name not in sealed and b'"task_closed"' in (read_bytes(path) or b""):
|
|
777
|
-
out.append(name)
|
|
778
|
-
return out
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
def show(root, task_id, fmt="md", rid=None):
|
|
782
|
-
import memory_store # pylint: disable=import-outside-toplevel
|
|
783
|
-
cfg = load_config(root)
|
|
784
|
-
history = memory_store.open_history(root, cfg)
|
|
785
|
-
if rid:
|
|
786
|
-
found = [e for e in history.all_entries() if e.get("rid") == rid]
|
|
787
|
-
if not found:
|
|
788
|
-
raise MemoryError_("no entry %s" % rid, 1)
|
|
789
|
-
return [mf.canonical(found[0]).decode("utf-8")]
|
|
790
|
-
if task_id == memory_store.current_task(root):
|
|
791
|
-
found = mc.compact_instances(FsReader(root), task_id, "tracked", cfg.excerpt_max_bytes)
|
|
792
|
-
entries = [e for c in found for e in c.entries]
|
|
793
|
-
lines = ["# %s — in flight — live state ('state.py status' is the authority)" % task_id, ""]
|
|
794
|
-
return lines + mf.render_task_md(entries)[1:] if fmt == "md" else \
|
|
795
|
-
[mf.canonical(e).decode("utf-8") for e in entries]
|
|
796
|
-
entries = history.entries(task_id)
|
|
797
|
-
if not entries:
|
|
798
|
-
raise MemoryError_("nothing recorded for %s" % task_id, 1)
|
|
799
|
-
if fmt == "json":
|
|
800
|
-
return [mf.canonical(e).decode("utf-8") for e in entries]
|
|
801
|
-
out = []
|
|
802
|
-
for inst in sorted({e.get("inst") for e in entries}):
|
|
803
|
-
out += mf.render_task_md([e for e in entries if e.get("inst") == inst])
|
|
804
|
-
return out
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
# ------------------------------------------------------------------ risks (§D9)
|
|
808
|
-
def risk_add_files(root, title, where, label, what, likely, fix, status, found):
|
|
809
|
-
"""Files mode: append the template section (plus Label) to known-risks.md."""
|
|
810
|
-
import memory_compact # pylint: disable=import-outside-toplevel,reimported
|
|
811
|
-
path = os.path.join(root, ".ai", "project", "known-risks.md")
|
|
812
|
-
text = read_bytes(path)
|
|
813
|
-
text = text.decode("utf-8") if text is not None else "# Known risks\n"
|
|
814
|
-
risks, _ = memory_compact.parse_known_risks(text)
|
|
815
|
-
rkey = memory_compact.rkey_of(title)
|
|
816
|
-
if any(r["rkey"] == rkey for r in risks):
|
|
817
|
-
raise MemoryError_("a risk titled %r is already recorded (%s)" % (title, rkey), 2)
|
|
818
|
-
section = memory_compact.risk_section(title, where, label, what, likely, fix, status, found)
|
|
819
|
-
body = text.rstrip("\n") + "\n\n" + "\n".join(section) + "\n"
|
|
820
|
-
atomic_write(path, body)
|
|
821
|
-
return rkey
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
def risk_close_files(root, rkey, status, why):
|
|
825
|
-
import memory_compact # pylint: disable=import-outside-toplevel,reimported
|
|
826
|
-
path = os.path.join(root, ".ai", "project", "known-risks.md")
|
|
827
|
-
text = (read_bytes(path) or b"").decode("utf-8")
|
|
828
|
-
risks, leftovers = memory_compact.parse_known_risks(text)
|
|
829
|
-
hit = [r for r in risks if r["rkey"] == rkey]
|
|
830
|
-
if not hit:
|
|
831
|
-
raise MemoryError_("no risk %s in known-risks.md ('state.py memory risks --status all')" % rkey, 1)
|
|
832
|
-
lines = hit[0]["text"].split("\n")
|
|
833
|
-
for i, line in enumerate(lines):
|
|
834
|
-
if line.startswith("- **Status**:"):
|
|
835
|
-
lines[i] = "- **Status**: %s — %s" % (status, why)
|
|
836
|
-
break
|
|
837
|
-
else:
|
|
838
|
-
lines.append("- **Status**: %s — %s" % (status, why))
|
|
839
|
-
hit[0]["text"] = "\n".join(lines)
|
|
840
|
-
atomic_write(path, memory_compact.render_known_risks(risks, leftovers))
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
def risk_data(title, where, label, what, likely, fix, status, why=None, supersedes=None, by=None):
|
|
844
|
-
data = {"rkey": mc.rkey_of(title), "title": title, "where": where, "label": label, "what": what,
|
|
845
|
-
"likelihood": likely, "fix": fix, "status": status, "status_why": why, "supersedes": supersedes}
|
|
846
|
-
if by:
|
|
847
|
-
data["by"] = by
|
|
848
|
-
return data
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
def standalone_risk(root, cfg, data, runtime="unknown", by=None):
|
|
852
|
-
"""A risk with no task in flight: one entry, straight into the open batch."""
|
|
853
|
-
entry = {"v": mf.FORMAT, "task": None, "inst": 1, "seq": 0, "kind": "risk", "ts": now(), "actor": "agent",
|
|
854
|
-
"by": by, "runtime": runtime if runtime in mf.RUNTIMES else "unknown", "src": ["state.py memory risk"]}
|
|
855
|
-
entry.update({k: data.get(k) for k in ("rkey", "title", "where", "label", "what", "likelihood", "fix",
|
|
856
|
-
"status", "status_why")})
|
|
857
|
-
entry.update(found_by=None, text=None)
|
|
858
|
-
if data.get("supersedes"):
|
|
859
|
-
entry["supersedes"] = data["supersedes"]
|
|
860
|
-
entry = mf.with_rid(entry)
|
|
861
|
-
append_entries(root, cfg, [entry])
|
|
862
|
-
return entry
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
def append_entries(root, cfg, entries):
|
|
866
|
-
"""Entries outside a close (a standalone risk, a redaction) into the open batch."""
|
|
867
|
-
with locked(root):
|
|
868
|
-
name, branch, books = open_batch(root, cfg)
|
|
869
|
-
packages = load_packages(root)
|
|
870
|
-
have = all_rids(packages)
|
|
871
|
-
header, current, _ = packages.get(name, (None, [], []))
|
|
872
|
-
parts = mf.parse_package_name(name)
|
|
873
|
-
header = header or mf.package_header(parts["period"], cfg.author, parts["batch"], now())
|
|
874
|
-
new = [e for e in entries if e["rid"] not in have]
|
|
875
|
-
if new:
|
|
876
|
-
atomic_write(os.path.join(memory_dir(root), name),
|
|
877
|
-
mf.render_package(header, [mf.strip_private(e) for e in current] + new))
|
|
878
|
-
books["open"][branch] = name
|
|
879
|
-
write_batches(root, books)
|
|
880
|
-
return name
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
# ------------------------------------------------------------------ git trees (§D10)
|
|
884
|
-
class GitTreeReader:
|
|
885
|
-
"""Blobs of one commit, through one `git cat-file --batch`."""
|
|
886
|
-
|
|
887
|
-
def __init__(self, repo, sha):
|
|
888
|
-
self.repo, self.sha = repo, sha
|
|
889
|
-
self._cache = {}
|
|
890
|
-
self._paths = None
|
|
891
|
-
|
|
892
|
-
def paths(self):
|
|
893
|
-
if self._paths is None:
|
|
894
|
-
code, out, _ = git(self.repo, "ls-tree", "-r", "-z", "--name-only", self.sha, binary=True)
|
|
895
|
-
self._paths = [p.decode("utf-8", "replace") for p in out.split(b"\0") if p] if code == 0 else []
|
|
896
|
-
return self._paths
|
|
897
|
-
|
|
898
|
-
def read_many(self, rels):
|
|
899
|
-
wanted = [r for r in rels if r not in self._cache]
|
|
900
|
-
if wanted:
|
|
901
|
-
stdin = "".join("%s:%s\n" % (self.sha, r) for r in wanted).encode("utf-8")
|
|
902
|
-
done = subprocess.run(["git", "-C", self.repo, "cat-file", "--batch"], input=stdin,
|
|
903
|
-
capture_output=True, check=False)
|
|
904
|
-
data, pos = done.stdout, 0
|
|
905
|
-
for rel in wanted:
|
|
906
|
-
end = data.find(b"\n", pos)
|
|
907
|
-
head = data[pos:end].split(b" ")
|
|
908
|
-
pos = end + 1
|
|
909
|
-
if len(head) == 2 and head[1] == b"missing" or len(head) < 3:
|
|
910
|
-
self._cache[rel] = None
|
|
911
|
-
continue
|
|
912
|
-
size = int(head[2])
|
|
913
|
-
self._cache[rel] = data[pos:pos + size]
|
|
914
|
-
pos += size + 1
|
|
915
|
-
return {r: self._cache.get(r) for r in rels}
|
|
916
|
-
|
|
917
|
-
def read(self, rel):
|
|
918
|
-
return self.read_many([rel])[rel]
|
|
919
|
-
|
|
920
|
-
def listdir(self, rel):
|
|
921
|
-
prefix = rel.rstrip("/") + "/"
|
|
922
|
-
return sorted({p[len(prefix):].split("/")[0] for p in self.paths() if p.startswith(prefix)})
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
def rev(repo, name):
|
|
926
|
-
code, out, _ = git(repo, "rev-parse", "--verify", "-q", name + "^{commit}")
|
|
927
|
-
return out.strip() if code == 0 and out.strip() else None
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
def policy_at(repo, sha):
|
|
931
|
-
code, out, _ = git(repo, "show", "%s:%s" % (sha, POLICY_REL.replace(os.sep, "/")))
|
|
932
|
-
if code != 0:
|
|
933
|
-
return None
|
|
934
|
-
try:
|
|
935
|
-
data = json.loads(out)
|
|
936
|
-
except ValueError:
|
|
937
|
-
return {"mode": "invalid"}
|
|
938
|
-
return data if isinstance(data, dict) else {"mode": "invalid"}
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
def validate_range(repo, old, new, local=False):
|
|
942
|
-
"""(findings, stats) for the commits O..T (§D10). O may be None: then only
|
|
943
|
-
the structure of T is checked."""
|
|
944
|
-
started = time.time()
|
|
945
|
-
policy = policy_at(repo, new)
|
|
946
|
-
if policy is None or policy.get("mode", "files") == "files":
|
|
947
|
-
return None, {}
|
|
948
|
-
if policy.get("mode") == "invalid":
|
|
949
|
-
return [Finding("V1", POLICY_REL, None, "memory.json at %s is not valid JSON" % new[:9],
|
|
950
|
-
"fix the file, or 'state.py memory disable'")], {}
|
|
951
|
-
max_bytes = int(policy.get("package_max_bytes", DEFAULTS["package_max_bytes"]))
|
|
952
|
-
tree = GitTreeReader(repo, new)
|
|
953
|
-
paths = tree.paths()
|
|
954
|
-
pkg_paths = [p for p in paths if p.startswith(".ai/memory/") and p.endswith(".jsonl")]
|
|
955
|
-
blobs = tree.read_many(pkg_paths)
|
|
956
|
-
packages = {p.split("/", 2)[2]: mf.parse_package(blobs[p] or b"", p.split("/", 2)[2]) for p in pkg_paths}
|
|
957
|
-
sizes = {p.split("/", 2)[2]: len(blobs[p] or b"") for p in pkg_paths}
|
|
958
|
-
changed, added = None, []
|
|
959
|
-
findings = []
|
|
960
|
-
if old:
|
|
961
|
-
code, out, _ = git(repo, "diff", "--name-status", "--no-renames", old, new, "--", ".ai")
|
|
962
|
-
rows = [line.split("\t") for line in out.splitlines() if "\t" in line] if code == 0 else []
|
|
963
|
-
changed = {r[1].split("/", 2)[2] for r in rows if r[1].startswith(".ai/memory/")}
|
|
964
|
-
added = [r[1] for r in rows if r[0].startswith("A")]
|
|
965
|
-
before = GitTreeReader(repo, old)
|
|
966
|
-
old_pkgs = [p for p in before.paths() if p.startswith(".ai/memory/") and p.endswith(".jsonl")]
|
|
967
|
-
old_blobs = before.read_many(old_pkgs)
|
|
968
|
-
old_rids = set()
|
|
969
|
-
for p in old_pkgs:
|
|
970
|
-
_, entries, _ = mf.parse_package(old_blobs[p] or b"")
|
|
971
|
-
old_rids |= {e["rid"] for e in entries}
|
|
972
|
-
new_rids = all_rids(packages)
|
|
973
|
-
redacted = {e.get("target") for _, entries, _ in packages.values() for e in entries
|
|
974
|
-
if e.get("kind") == "redaction"}
|
|
975
|
-
lost = sorted(old_rids - new_rids - redacted)
|
|
976
|
-
if lost:
|
|
977
|
-
findings.append(Finding("V2", ".ai/memory/", None, "%d record(s) of %s are gone (%s…)" % (
|
|
978
|
-
len(lost), old[:9], lost[0]), "git checkout %s -- .ai/memory/ and merge the lines back" % old[:9]))
|
|
979
|
-
findings = check_packages(packages, max_bytes, sizes, changed) + findings
|
|
980
|
-
journals = [p for p in paths if re.match(r"^\.ai/reports/[^/]+/events\.jsonl$", p)]
|
|
981
|
-
journal_blobs = tree.read_many(journals) # one cat-file --batch for every journal (§D10)
|
|
982
|
-
findings += check_journals(journal_blobs.get, journals, packages, bool(policy.get("cleanup")))
|
|
983
|
-
for path in added:
|
|
984
|
-
if path.startswith(".ai/reports/") and path.endswith(".log"):
|
|
985
|
-
findings.append(Finding("V6", path, None, "a log is committed under .ai/reports/",
|
|
986
|
-
"git rm --cached %s (logs live in .ai/state/logs/)" % path))
|
|
987
|
-
for path in paths:
|
|
988
|
-
# The index lives under .ai/local/; a project's own SQLite file elsewhere is not ours.
|
|
989
|
-
if path.startswith(".ai/local/") or path.startswith(".ai/state/logs/") or \
|
|
990
|
-
(path.startswith(".ai/") and ".sqlite3" in path):
|
|
991
|
-
findings.append(Finding("V6", path, None, "local data is committed",
|
|
992
|
-
"git rm --cached %s (.ai/local/ and .ai/state/logs/ stay local)" % path))
|
|
993
|
-
if local:
|
|
994
|
-
outbox = local_path(repo, "outbox")
|
|
995
|
-
if os.path.isdir(outbox) and any(n.endswith(".jsonl") for n in os.listdir(outbox)):
|
|
996
|
-
findings.append(Finding("V9", os.path.join(LOCAL_REL, "outbox"), None, "an export is pending",
|
|
997
|
-
"state.py memory export --pending, commit, push again"))
|
|
998
|
-
stats = {"packages": len(packages), "tasks": len(journals), "ms": int((time.time() - started) * 1000)}
|
|
999
|
-
return findings, stats
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
# ------------------------------------------------------------------ the git hook (R17, §D10)
|
|
1003
|
-
def hooks_path(root):
|
|
1004
|
-
code, out, _ = git(root, "rev-parse", "--git-path", "hooks")
|
|
1005
|
-
if code != 0 or not out.strip():
|
|
1006
|
-
return None
|
|
1007
|
-
path = out.strip()
|
|
1008
|
-
return path if os.path.isabs(path) else os.path.join(root, path)
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
def shim_candidates(validator=None):
|
|
1012
|
-
"""Where the shim looks for the validator, in order: the hook beside this
|
|
1013
|
-
module, the hook in either runtime home, then this module itself — a Codex
|
|
1014
|
-
install copies skills/ai-task/ but not every hook."""
|
|
1015
|
-
return [validator or VALIDATOR, "$HOME/.claude/hooks/memory-pre-push.py",
|
|
1016
|
-
"${CODEX_HOME:-$HOME/.codex}/hooks/memory-pre-push.py", SELF,
|
|
1017
|
-
"$HOME/.claude/skills/ai-task/memory_sync.py",
|
|
1018
|
-
"${CODEX_HOME:-$HOME/.codex}/skills/ai-task/memory_sync.py"]
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
def shim_text(validator=None):
|
|
1022
|
-
return ("#!/bin/sh\n%s — written by 'state.py memory enable'; 'state.py memory disable' removes it\n"
|
|
1023
|
-
"for v in %s; do\n"
|
|
1024
|
-
" [ -f \"$v\" ] && exec python3 -I \"$v\" \"$@\"\n"
|
|
1025
|
-
"done\n"
|
|
1026
|
-
"echo \"riskrails memory: validator not found — this push is not checked\" >&2; exit 0\n"
|
|
1027
|
-
% (HOOK_MARKER, " ".join('"%s"' % c for c in shim_candidates(validator))))
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
def shim_paths(text):
|
|
1031
|
-
"""The shim's candidates as paths on this machine."""
|
|
1032
|
-
match = re.search(r"^for v in (.+); do$", text, re.M)
|
|
1033
|
-
home = os.path.expanduser("~")
|
|
1034
|
-
codex = os.environ.get("CODEX_HOME") or os.path.join(home, ".codex")
|
|
1035
|
-
return [c.replace("${CODEX_HOME:-$HOME/.codex}", codex).replace("$HOME", home)
|
|
1036
|
-
for c in re.findall(r'"([^"]+)"', match.group(1) if match else "")]
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
def snippet_text(validator=None):
|
|
1040
|
-
validator = validator or VALIDATOR
|
|
1041
|
-
return ("# riskrails memory: add to your own pre-push hook (stdin can be read only once)\n"
|
|
1042
|
-
"rr_in=$(mktemp) && cat > \"$rr_in\"\n"
|
|
1043
|
-
"python3 -I \"%s\" \"$@\" < \"$rr_in\" || { rr=$?; rm -f \"$rr_in\"; exit $rr; }\n"
|
|
1044
|
-
"# … then your hook, reading \"$rr_in\" where it read stdin …\n"
|
|
1045
|
-
"rm -f \"$rr_in\"\n" % validator)
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
def install_hook(root):
|
|
1049
|
-
"""installed | current | foreign | hookspath (R17)."""
|
|
1050
|
-
code, out, _ = git(root, "config", "--get", "core.hooksPath")
|
|
1051
|
-
if code == 0 and out.strip():
|
|
1052
|
-
return "hookspath"
|
|
1053
|
-
path = hooks_path(root)
|
|
1054
|
-
if not path:
|
|
1055
|
-
raise MemoryError_("not a git repository: no hooks path", 2)
|
|
1056
|
-
hook = os.path.join(path, "pre-push")
|
|
1057
|
-
text = read_bytes(hook)
|
|
1058
|
-
if text is not None and HOOK_MARKER.encode() not in text:
|
|
1059
|
-
return "foreign"
|
|
1060
|
-
if text is not None and text.decode("utf-8", "replace") == shim_text():
|
|
1061
|
-
return "current"
|
|
1062
|
-
os.makedirs(path, exist_ok=True)
|
|
1063
|
-
atomic_write(hook, shim_text())
|
|
1064
|
-
os.chmod(hook, 0o755)
|
|
1065
|
-
return "installed"
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
def remove_hook(root):
|
|
1069
|
-
path = hooks_path(root)
|
|
1070
|
-
hook = os.path.join(path, "pre-push") if path else None
|
|
1071
|
-
text = read_bytes(hook) if hook else None
|
|
1072
|
-
if text is not None and HOOK_MARKER.encode() in text:
|
|
1073
|
-
os.remove(hook)
|
|
1074
|
-
return "removed"
|
|
1075
|
-
return "foreign" if text is not None else "absent"
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
def write_policy(root, policy):
|
|
1079
|
-
atomic_write(os.path.join(root, POLICY_REL), json.dumps(policy, indent=2, ensure_ascii=False) + "\n")
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
def enable(root, author=None, period=None, index=None):
|
|
1083
|
-
cfg = load_config(root)
|
|
1084
|
-
policy = dict(DEFAULTS)
|
|
1085
|
-
policy.update(cfg.policy if cfg.present else {})
|
|
1086
|
-
policy["mode"] = "packages"
|
|
1087
|
-
if period:
|
|
1088
|
-
if period not in mf.PERIODS:
|
|
1089
|
-
raise MemoryError_("--period takes one of %s" % ", ".join(mf.PERIODS))
|
|
1090
|
-
policy["period"] = period
|
|
1091
|
-
write_policy(root, policy)
|
|
1092
|
-
local = dict(cfg.local)
|
|
1093
|
-
local["schema"] = 1
|
|
1094
|
-
local["author"] = author_slug(author) if author else cfg.author
|
|
1095
|
-
if index:
|
|
1096
|
-
if index not in ("none", "sqlite"):
|
|
1097
|
-
raise MemoryError_("--index takes none or sqlite")
|
|
1098
|
-
local["index"] = index
|
|
1099
|
-
local.setdefault("index", "none")
|
|
1100
|
-
atomic_write(local_path(root, "config.json"), json.dumps(local, sort_keys=True) + "\n")
|
|
1101
|
-
return install_hook(root)
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
def disable(root):
|
|
1105
|
-
cfg = load_config(root)
|
|
1106
|
-
policy = dict(cfg.policy)
|
|
1107
|
-
policy["mode"] = "files"
|
|
1108
|
-
write_policy(root, policy)
|
|
1109
|
-
return remove_hook(root)
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
# ------------------------------------------------------------------ migration (§D12, R25)
|
|
1113
|
-
def migration_plan(root):
|
|
1114
|
-
"""What --report shows and --apply does, from HEAD's tracked files only."""
|
|
1115
|
-
head = rev(root, "HEAD")
|
|
1116
|
-
if not head:
|
|
1117
|
-
raise MemoryError_("migration reads HEAD: commit first", 2)
|
|
1118
|
-
tree = GitTreeReader(root, head)
|
|
1119
|
-
packages = load_packages(root)
|
|
1120
|
-
sealed = {mf.key_of(e) for _, entries, _ in packages.values() for e in entries if e.get("kind") == "task"}
|
|
1121
|
-
tasks = sorted({p.split("/")[2] for p in tree.paths() if p.startswith(".ai/reports/T-")})
|
|
1122
|
-
plan = []
|
|
1123
|
-
for task in tasks:
|
|
1124
|
-
found = mc.compact_instances(tree, task, "tracked")
|
|
1125
|
-
journal = tree.read(".ai/reports/%s/events.jsonl" % task)
|
|
1126
|
-
names = {e.get("task") for _, e in mc.parse_journal(journal) if e} if journal else set()
|
|
1127
|
-
logs = [(p, len(tree.read(p) or b"")) for p in tree.paths()
|
|
1128
|
-
if p.startswith(".ai/reports/%s/" % task) and p.endswith(".log")]
|
|
1129
|
-
plan.append({"task": task, "compactions": found, "journal": journal is not None,
|
|
1130
|
-
"names": sorted(n for n in names if n), "logs": logs,
|
|
1131
|
-
"sealed": [mf.key_of(c.seal) for c in found if c.seal and mf.key_of(c.seal) in sealed]})
|
|
1132
|
-
return head, plan
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
def migrate_report(root):
|
|
1136
|
-
head, plan = migration_plan(root)
|
|
1137
|
-
kinds, dropped = collections.Counter(), collections.Counter()
|
|
1138
|
-
lines = ["# Migration report — %s" % now(), "", "HEAD %s; tracked files only, so every clone gives the same "
|
|
1139
|
-
"rids." % head[:12], "", "## Tasks", "",
|
|
1140
|
-
"| task | instances | journal | name in journal | coverage | result | sealed already |",
|
|
1141
|
-
"|---|---|---|---|---|---|---|"]
|
|
1142
|
-
total_logs = []
|
|
1143
|
-
for item in plan:
|
|
1144
|
-
found = item["compactions"]
|
|
1145
|
-
for c in found:
|
|
1146
|
-
if c.seal:
|
|
1147
|
-
for e in c.entries:
|
|
1148
|
-
kinds[e["kind"]] += 1
|
|
1149
|
-
dropped.update(c.seal["counts"]["dropped"])
|
|
1150
|
-
cov = ", ".join("%s/%s" % (c.coverage["accounted"], c.coverage["lines"]) for c in found) or "-"
|
|
1151
|
-
names = ", ".join(item["names"]) or "-"
|
|
1152
|
-
mismatch = " ≠ dir" if item["names"] and item["names"] != [item["task"]] else ""
|
|
1153
|
-
lines.append("| %s | %d | %s | %s%s | %s | %s | %s |" % (
|
|
1154
|
-
item["task"], len(found), "yes" if item["journal"] else "no (state-only)", names, mismatch, cov,
|
|
1155
|
-
", ".join(c.seal["result"] for c in found if c.seal) or "no sources",
|
|
1156
|
-
", ".join(item["sealed"]) or "-"))
|
|
1157
|
-
total_logs += item["logs"]
|
|
1158
|
-
lines += ["", "## Kept per kind", ""] + ["- %s: %d" % (k, n) for k, n in sorted(kinds.items())]
|
|
1159
|
-
lines += ["", "## Dropped per event (KEEP_RULES v1)", ""] + ["- %s: %d" % (k, n) for k, n in sorted(dropped.items())]
|
|
1160
|
-
lines += ["", "## Committed logs (%d, %s bytes)" % (len(total_logs), mf.fmt_n(sum(s for _, s in total_logs))), ""]
|
|
1161
|
-
lines += ["- %s (%s bytes)" % (p, mf.fmt_n(s)) for p, s in total_logs]
|
|
1162
|
-
text = (read_bytes(os.path.join(root, ".ai", "project", "known-risks.md")) or b"").decode("utf-8")
|
|
1163
|
-
risks, leftovers = mc.parse_known_risks(text)
|
|
1164
|
-
titles = collections.Counter(r["title"] for r in risks)
|
|
1165
|
-
lines += ["", "## known-risks.md", "", "- sections parsed: %d" % len(risks),
|
|
1166
|
-
"- lines outside a section: %d" % len(leftovers["lines"]),
|
|
1167
|
-
"- label derived (no Label line): %d" % sum(1 for r in risks if r["label_derived"]),
|
|
1168
|
-
"- renders byte for byte: %s" % ("yes" if mc.render_known_risks(risks, leftovers) == text else "NO"),
|
|
1169
|
-
"- duplicate titles: %s" % (", ".join(t for t, n in titles.items() if n > 1) or "none")]
|
|
1170
|
-
report = "\n".join(lines) + "\n"
|
|
1171
|
-
atomic_write(local_path(root, "migration-report.md"), report)
|
|
1172
|
-
return report
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
def backup(root):
|
|
1176
|
-
path = local_path(root, "backup-%s.tar" % datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ"))
|
|
1177
|
-
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
1178
|
-
with tarfile.open(path, "w") as tar:
|
|
1179
|
-
for rel in (".ai/reports", ".ai/project/known-risks.md"):
|
|
1180
|
-
if os.path.exists(os.path.join(root, rel)):
|
|
1181
|
-
tar.add(os.path.join(root, rel), arcname=rel)
|
|
1182
|
-
return path
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
def migrate_apply(root, risks=False):
|
|
1186
|
-
"""Closed legacy tasks into batches by their closed_at period; then, with
|
|
1187
|
-
risks, known-risks.md into risk entries and a stub."""
|
|
1188
|
-
cfg = load_config(root)
|
|
1189
|
-
saved = backup(root)
|
|
1190
|
-
out = ["backup: %s" % os.path.relpath(saved, root)]
|
|
1191
|
-
if risks:
|
|
1192
|
-
return out + migrate_risks(root, cfg)
|
|
1193
|
-
_, plan = migration_plan(root)
|
|
1194
|
-
groups = collections.defaultdict(list)
|
|
1195
|
-
skipped, incomplete = [], []
|
|
1196
|
-
for item in plan:
|
|
1197
|
-
for c in item["compactions"]:
|
|
1198
|
-
if not c.seal or c.seal["result"] not in ("closed", "abandoned", "restarted"):
|
|
1199
|
-
continue
|
|
1200
|
-
key = mf.key_of(c.seal)
|
|
1201
|
-
if key in item["sealed"]:
|
|
1202
|
-
skipped.append(key)
|
|
1203
|
-
continue
|
|
1204
|
-
if c.coverage["accounted"] != c.coverage["lines"] or c.coverage["skipped"] or c.problems:
|
|
1205
|
-
incomplete.append((key, c.coverage, c.problems))
|
|
1206
|
-
continue
|
|
1207
|
-
when = c.seal.get("closed_at") or c.seal.get("started_at") or c.seal["ts"]
|
|
1208
|
-
groups[mf.period_of(when, cfg.period)].append(c)
|
|
1209
|
-
written = 0
|
|
1210
|
-
with locked(root):
|
|
1211
|
-
for period in sorted(groups):
|
|
1212
|
-
# Whole tasks per batch, a new batch before package_max_bytes (V8).
|
|
1213
|
-
batches, size = [[]], 0
|
|
1214
|
-
for c in groups[period]:
|
|
1215
|
-
task_bytes = sum(len(mf.canonical(e)) + 1 for e in c.entries + [c.seal])
|
|
1216
|
-
if batches[-1] and size + task_bytes > cfg.package_max_bytes:
|
|
1217
|
-
batches.append([])
|
|
1218
|
-
size = 0
|
|
1219
|
-
batches[-1].append(c)
|
|
1220
|
-
size += task_bytes
|
|
1221
|
-
for batch in batches:
|
|
1222
|
-
name = new_batch_name(root, period, cfg.author)
|
|
1223
|
-
parts = mf.parse_package_name(name)
|
|
1224
|
-
entries = [e for c in batch for e in c.entries + [c.seal]]
|
|
1225
|
-
atomic_write(os.path.join(memory_dir(root), name),
|
|
1226
|
-
mf.render_package(mf.package_header(period, cfg.author, parts["batch"], now()),
|
|
1227
|
-
entries))
|
|
1228
|
-
written += len(batch)
|
|
1229
|
-
out.append("package: %s (%d tasks, %d entries)" % (os.path.join(MEMORY_REL, name), len(batch),
|
|
1230
|
-
len(entries)))
|
|
1231
|
-
findings = check_packages(load_packages(root), cfg.package_max_bytes)
|
|
1232
|
-
out.append("migrated %d task instance(s); %d already sealed; %d incomplete (reported, not exported)"
|
|
1233
|
-
% (written, len(skipped), len(incomplete)))
|
|
1234
|
-
out += ["incomplete: %s coverage %s/%s %s" % (k, cov["accounted"], cov["lines"], "; ".join(p))
|
|
1235
|
-
for k, cov, p in incomplete]
|
|
1236
|
-
out += [format_finding(f) for f in findings]
|
|
1237
|
-
return out
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
STUB = """# Known risks
|
|
1241
|
-
|
|
1242
|
-
<!-- The risks live in the record since {date}: `state.py memory risks --md` renders
|
|
1243
|
-
them, `state.py memory risk add|close` records them (spec compact-project-memory
|
|
1244
|
-
§D9, OQ4). A section written here is still read, as before the migration. -->
|
|
1245
|
-
"""
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
def migrate_risks(root, cfg):
|
|
1249
|
-
path = os.path.join(root, ".ai", "project", "known-risks.md")
|
|
1250
|
-
text = (read_bytes(path) or b"").decode("utf-8")
|
|
1251
|
-
import memory_store # pylint: disable=import-outside-toplevel
|
|
1252
|
-
entries = memory_store.risk_entries_from_md(text)
|
|
1253
|
-
risks, leftovers = mc.parse_known_risks(text)
|
|
1254
|
-
rendered = mc.render_known_risks([{"text": e["text"]} for e in entries], leftovers)
|
|
1255
|
-
if rendered != text:
|
|
1256
|
-
raise MemoryError_("known-risks.md does not render byte for byte from its entries — nothing changed", 3)
|
|
1257
|
-
entries = [dict(e, actor="migration", src=["known-risks.md#%s" % e["rkey"]]) for e in entries]
|
|
1258
|
-
entries = [mf.with_rid(e) for e in entries]
|
|
1259
|
-
name = append_entries(root, cfg, entries)
|
|
1260
|
-
atomic_write(path, STUB.format(date=now()[:10]))
|
|
1261
|
-
del risks
|
|
1262
|
-
return ["risks: %d entries into %s; known-risks.md is a stub" % (len(entries), os.path.join(MEMORY_REL, name))]
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
def move_logs(root):
|
|
1266
|
-
head = rev(root, "HEAD")
|
|
1267
|
-
tree = GitTreeReader(root, head) if head else None
|
|
1268
|
-
logs = [p for p in (tree.paths() if tree else []) if p.startswith(".ai/reports/") and p.endswith(".log")]
|
|
1269
|
-
moved = []
|
|
1270
|
-
for rel in logs:
|
|
1271
|
-
task = rel.split("/")[2]
|
|
1272
|
-
src = os.path.join(root, rel)
|
|
1273
|
-
if not os.path.exists(src):
|
|
1274
|
-
continue
|
|
1275
|
-
dest = os.path.join(root, ".ai", "state", "logs", task, "legacy", os.path.basename(rel))
|
|
1276
|
-
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
|
1277
|
-
shutil.move(src, dest)
|
|
1278
|
-
moved.append(rel)
|
|
1279
|
-
listing = local_path(root, "logs-to-rm.txt")
|
|
1280
|
-
atomic_write(listing, "".join("%s\n" % p for p in moved))
|
|
1281
|
-
return moved, "git rm --cached --quiet --pathspec-from-file=%s" % os.path.relpath(listing, root)
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
# ------------------------------------------------------------------ cleanup, restore, redact (§D6, §D13)
|
|
1285
|
-
def cleanup_ready(root, task_id, cfg, packages=None):
|
|
1286
|
-
"""The R22 conditions for one task, or the first that fails."""
|
|
1287
|
-
packages = packages if packages is not None else load_packages(root)
|
|
1288
|
-
seals = [(n, e) for n, (_, entries, _) in packages.items() for e in entries
|
|
1289
|
-
if e.get("kind") == "task" and e.get("task") == task_id]
|
|
1290
|
-
if not cfg.cleanup:
|
|
1291
|
-
return "cleanup is off in %s" % POLICY_REL
|
|
1292
|
-
if not seals:
|
|
1293
|
-
return "no seal for %s in a package in the working tree" % task_id
|
|
1294
|
-
findings = check_packages({n: packages[n] for n, _ in seals}, cfg.package_max_bytes)
|
|
1295
|
-
if findings:
|
|
1296
|
-
return "its package does not verify (%s)" % findings[0].what
|
|
1297
|
-
for _, seal in seals:
|
|
1298
|
-
if (seal.get("counts") or {}).get("skipped"):
|
|
1299
|
-
return "%s has %s torn journal line(s)" % (mf.key_of(seal), seal["counts"]["skipped"])
|
|
1300
|
-
journal = read_bytes(os.path.join(root, ".ai", "reports", task_id, "events.jsonl"))
|
|
1301
|
-
if journal is not None:
|
|
1302
|
-
found = check_journals(lambda rel: journal if rel.endswith("events.jsonl") else None,
|
|
1303
|
-
[".ai/reports/%s/events.jsonl" % task_id], packages)
|
|
1304
|
-
found = [f for f in found if f.check == "V5"]
|
|
1305
|
-
if found:
|
|
1306
|
-
return "coverage is incomplete (%s)" % found[0].what
|
|
1307
|
-
return None
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
def cleanup_task(root, task_id, cfg):
|
|
1311
|
-
"""B6: .ai/reports/<id> → .ai/state/logs/<id>/report; copy, verify,
|
|
1312
|
-
remove when the two are on different filesystems."""
|
|
1313
|
-
why = cleanup_ready(root, task_id, cfg)
|
|
1314
|
-
if why:
|
|
1315
|
-
raise MemoryError_(why, 1)
|
|
1316
|
-
src = os.path.join(root, ".ai", "reports", task_id)
|
|
1317
|
-
dest = os.path.join(root, ".ai", "state", "logs", task_id, "report")
|
|
1318
|
-
if not os.path.isdir(src):
|
|
1319
|
-
return "%s already cleaned up" % task_id
|
|
1320
|
-
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
|
1321
|
-
if os.path.exists(dest):
|
|
1322
|
-
raise MemoryError_("%s exists: 'state.py memory restore %s' first" % (os.path.relpath(dest, root), task_id), 3)
|
|
1323
|
-
try:
|
|
1324
|
-
if os.environ.get("AI_MEMORY_FORCE_EXDEV"):
|
|
1325
|
-
raise OSError(18, "Invalid cross-device link (forced)")
|
|
1326
|
-
os.replace(src, dest)
|
|
1327
|
-
except OSError as exc:
|
|
1328
|
-
if exc.errno != 18:
|
|
1329
|
-
raise
|
|
1330
|
-
shutil.copytree(src, dest)
|
|
1331
|
-
if tree_digest(src) != tree_digest(dest):
|
|
1332
|
-
shutil.rmtree(dest, ignore_errors=True)
|
|
1333
|
-
raise MemoryError_("the copy of %s does not match its source — nothing removed" % task_id, 3) \
|
|
1334
|
-
from exc
|
|
1335
|
-
shutil.rmtree(src)
|
|
1336
|
-
return "%s → %s" % (os.path.relpath(src, root), os.path.relpath(dest, root))
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
def tree_digest(path):
|
|
1340
|
-
import hashlib # pylint: disable=import-outside-toplevel
|
|
1341
|
-
h = hashlib.sha256()
|
|
1342
|
-
for dirpath, dirs, names in os.walk(path):
|
|
1343
|
-
dirs.sort()
|
|
1344
|
-
for name in sorted(names):
|
|
1345
|
-
full = os.path.join(dirpath, name)
|
|
1346
|
-
h.update(os.path.relpath(full, path).encode() + b"\0" + (read_bytes(full) or b"") + b"\0")
|
|
1347
|
-
return h.hexdigest()
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
def cleanup(root, ids):
|
|
1351
|
-
cfg = load_config(root)
|
|
1352
|
-
if not cfg.packages:
|
|
1353
|
-
raise MemoryError_("files mode: cleanup needs packages mode", 2)
|
|
1354
|
-
if not cfg.cleanup:
|
|
1355
|
-
raise MemoryError_("cleanup is off: set \"cleanup\": true in %s once 'memory verify --all' and "
|
|
1356
|
-
"'--fresh-clone' are green (§D6)" % POLICY_REL, 2)
|
|
1357
|
-
findings = [f for f in verify(root) if not f.check.endswith("-warn")]
|
|
1358
|
-
if findings:
|
|
1359
|
-
raise MemoryError_("'memory verify --all' is not green: %s" % format_finding(findings[0]), 1)
|
|
1360
|
-
packages = load_packages(root)
|
|
1361
|
-
if ids == "all":
|
|
1362
|
-
ids = sorted(sealed_task_ids(packages) & set(os.listdir(os.path.join(root, ".ai", "reports"))))
|
|
1363
|
-
out = []
|
|
1364
|
-
for task_id in ids:
|
|
1365
|
-
try:
|
|
1366
|
-
out.append(cleanup_task(root, task_id, cfg))
|
|
1367
|
-
except MemoryError_ as exc:
|
|
1368
|
-
out.append("%s kept: %s" % (task_id, exc))
|
|
1369
|
-
return out
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
def restore(root, ids):
|
|
1373
|
-
import memory_store # pylint: disable=import-outside-toplevel
|
|
1374
|
-
logs = os.path.join(root, ".ai", "state", "logs")
|
|
1375
|
-
if ids == "all":
|
|
1376
|
-
ids = sorted(n for n in os.listdir(logs) if os.path.isdir(os.path.join(logs, n, "report"))) \
|
|
1377
|
-
if os.path.isdir(logs) else []
|
|
1378
|
-
history = memory_store.FileHistory(root, load_config(root))
|
|
1379
|
-
ids = sorted(set(ids) | {s.get("task") for s in history.seals().values()
|
|
1380
|
-
if not os.path.isdir(os.path.join(root, ".ai", "reports", s.get("task") or "-"))
|
|
1381
|
-
and mf.NEW_ID_RE.match(s.get("task") or "")})
|
|
1382
|
-
out = []
|
|
1383
|
-
history = None
|
|
1384
|
-
for task_id in ids:
|
|
1385
|
-
dest = os.path.join(root, ".ai", "reports", task_id)
|
|
1386
|
-
raw = os.path.join(logs, task_id, "report")
|
|
1387
|
-
if os.path.isdir(dest):
|
|
1388
|
-
out.append("%s is in .ai/reports/ already" % task_id)
|
|
1389
|
-
continue
|
|
1390
|
-
if os.path.isdir(raw):
|
|
1391
|
-
os.replace(raw, dest)
|
|
1392
|
-
out.append("%s restored from %s" % (task_id, os.path.relpath(raw, root)))
|
|
1393
|
-
continue
|
|
1394
|
-
history = history or memory_store.FileHistory(root, load_config(root))
|
|
1395
|
-
entries = history.entries(task_id)
|
|
1396
|
-
if not entries:
|
|
1397
|
-
out.append("%s: nothing recorded" % task_id)
|
|
1398
|
-
continue
|
|
1399
|
-
os.makedirs(dest)
|
|
1400
|
-
md = ["<!-- restored from record %s — a reconstruction, not the original task.md -->" % now()]
|
|
1401
|
-
md += mf.render_task_md(entries)
|
|
1402
|
-
atomic_write(os.path.join(dest, "task.md"), "\n".join(md) + "\n")
|
|
1403
|
-
lines = []
|
|
1404
|
-
for e in entries:
|
|
1405
|
-
data = {k: v for k, v in e.items() if k not in mf.ENVELOPE}
|
|
1406
|
-
data["restored"] = True
|
|
1407
|
-
data["rid"] = e["rid"]
|
|
1408
|
-
lines.append(json.dumps({"ts": e["ts"], "task": task_id, "event": "restored_%s" % e["kind"],
|
|
1409
|
-
"actor": e["actor"], "runtime": e["runtime"], "stage": "",
|
|
1410
|
-
"detail": "restored from record", "data": data}, ensure_ascii=False))
|
|
1411
|
-
atomic_write(os.path.join(dest, "events.jsonl"), "\n".join(lines) + "\n")
|
|
1412
|
-
out.append("%s restored from record (task.md marked, events.jsonl with data.restored)" % task_id)
|
|
1413
|
-
return out
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
def redact(root, rid, field, why, by=None):
|
|
1417
|
-
"""The one exception to append-only (R8): the field becomes "[redacted]",
|
|
1418
|
-
the line gets a new rid, a redaction entry names the old one."""
|
|
1419
|
-
cfg = load_config(root)
|
|
1420
|
-
with locked(root):
|
|
1421
|
-
packages = load_packages(root)
|
|
1422
|
-
for name, (header, entries, _) in packages.items():
|
|
1423
|
-
hit = [e for e in entries if e["rid"] == rid]
|
|
1424
|
-
if not hit:
|
|
1425
|
-
continue
|
|
1426
|
-
entry = mf.strip_private(hit[0])
|
|
1427
|
-
if field in mf.ENVELOPE or field not in entry:
|
|
1428
|
-
raise MemoryError_("%s has no redactable field %r" % (rid, field), 2)
|
|
1429
|
-
new = dict(entry, **{field: "[redacted]"})
|
|
1430
|
-
new = mf.with_rid(new)
|
|
1431
|
-
note = mf.with_rid({"v": mf.FORMAT, "task": entry.get("task"), "inst": entry.get("inst") or 1,
|
|
1432
|
-
"seq": entry.get("seq") or 0, "kind": "redaction", "ts": now(), "actor": "human",
|
|
1433
|
-
"by": by, "runtime": "unknown", "src": ["state.py memory redact"],
|
|
1434
|
-
"target": rid, "field": field, "why": why, "replacement": new["rid"]})
|
|
1435
|
-
rest = [mf.strip_private(e) for e in entries if e["rid"] != rid]
|
|
1436
|
-
atomic_write(os.path.join(memory_dir(root), name), mf.render_package(header, rest + [new, note]))
|
|
1437
|
-
del cfg
|
|
1438
|
-
return new["rid"], name
|
|
1439
|
-
raise MemoryError_("no entry %s in .ai/memory/" % rid, 1)
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
# ------------------------------------------------------------------ pack (§D4, R7)
|
|
1443
|
-
def pack(root, period):
|
|
1444
|
-
cfg = load_config(root)
|
|
1445
|
-
if not re.match(r"^\d{4}(?:-\d{2}|-Q[1-4])?$", period or ""):
|
|
1446
|
-
raise MemoryError_("--period takes YYYY-MM, YYYY-Qn or YYYY", 2)
|
|
1447
|
-
grain = "quarter" if "-Q" in period else ("month" if "-" in period else "year")
|
|
1448
|
-
current = mf.period_of(now(), grain)
|
|
1449
|
-
if period >= current:
|
|
1450
|
-
raise MemoryError_("period %s has not ended (now %s)" % (period, current), 2)
|
|
1451
|
-
with locked(root):
|
|
1452
|
-
packages = load_packages(root)
|
|
1453
|
-
mine = [n for n in packages if (mf.parse_package_name(n) or {}).get("period") == period
|
|
1454
|
-
and (mf.parse_package_name(n) or {}).get("author") == cfg.author]
|
|
1455
|
-
if not mine:
|
|
1456
|
-
raise MemoryError_("no package of %s for %s" % (cfg.author, period), 1)
|
|
1457
|
-
target = mf.package_name(period, cfg.author)
|
|
1458
|
-
before = set()
|
|
1459
|
-
entries = {}
|
|
1460
|
-
for name in mine:
|
|
1461
|
-
for e in packages[name][1]:
|
|
1462
|
-
before.add(e["rid"])
|
|
1463
|
-
entries[e["rid"]] = mf.strip_private(e)
|
|
1464
|
-
seals = {mf.key_of(e): e for e in entries.values() if e.get("kind") == "task"}
|
|
1465
|
-
|
|
1466
|
-
def order(e):
|
|
1467
|
-
seal = seals.get(mf.key_of(e)) or {}
|
|
1468
|
-
return (seal.get("closed_at") or e.get("ts") or "", mf.key_of(e), e.get("seq") or 0, e["rid"])
|
|
1469
|
-
ordered = sorted(entries.values(), key=order)
|
|
1470
|
-
header = mf.package_header(period, cfg.author, None, now())
|
|
1471
|
-
data = b"\n".join([mf.canonical(header)] + [mf.canonical(e) for e in ordered]) + b"\n"
|
|
1472
|
-
atomic_write(os.path.join(memory_dir(root), target), data)
|
|
1473
|
-
_, check, errors = mf.parse_package(data, target)
|
|
1474
|
-
if errors or {e["rid"] for e in check} != before:
|
|
1475
|
-
raise MemoryError_("the pack does not hold the batches' rid set — batches kept", 3)
|
|
1476
|
-
removed = [n for n in mine if n != target]
|
|
1477
|
-
for name in removed:
|
|
1478
|
-
os.remove(os.path.join(memory_dir(root), name))
|
|
1479
|
-
books = read_batches(root)
|
|
1480
|
-
books["open"] = {b: n for b, n in books["open"].items() if n not in removed}
|
|
1481
|
-
write_batches(root, books)
|
|
1482
|
-
return target, removed, len(before)
|
|
18
|
+
import memory_io as mio # noqa: E402 pylint: disable=wrong-import-position
|
|
19
|
+
import memory_verify as mver # noqa: E402 pylint: disable=wrong-import-position
|
|
1483
20
|
|
|
1484
21
|
|
|
1485
22
|
# ------------------------------------------------------------------ the pre-push entry (§I5)
|
|
1486
23
|
def _git_out(repo, *args):
|
|
1487
|
-
code, out, _ = git(repo, *args)
|
|
24
|
+
code, out, _ = mio.git(repo, *args)
|
|
1488
25
|
return code, out.strip()
|
|
1489
26
|
|
|
1490
27
|
|
|
@@ -1516,14 +53,14 @@ def pre_push(argv, stdin):
|
|
|
1516
53
|
jobs, local = [], False
|
|
1517
54
|
if argv[:1] == ["--range"] and len(argv) == 2 and ".." in argv[1]:
|
|
1518
55
|
old, new = argv[1].split("..", 1)
|
|
1519
|
-
new_sha = rev(repo, new)
|
|
56
|
+
new_sha = mver.rev(repo, new)
|
|
1520
57
|
if not new_sha:
|
|
1521
|
-
raise MemoryError_("%s is not a commit" % new, 3)
|
|
1522
|
-
jobs.append((argv[1], rev(repo, old), new_sha))
|
|
58
|
+
raise mio.MemoryError_("%s is not a commit" % new, 3)
|
|
59
|
+
jobs.append((argv[1], mver.rev(repo, old), new_sha))
|
|
1523
60
|
elif argv[:1] == ["--tree"] and len(argv) == 2:
|
|
1524
|
-
sha = rev(repo, argv[1])
|
|
61
|
+
sha = mver.rev(repo, argv[1])
|
|
1525
62
|
if not sha:
|
|
1526
|
-
raise MemoryError_("%s is not a commit" % argv[1], 3)
|
|
63
|
+
raise mio.MemoryError_("%s is not a commit" % argv[1], 3)
|
|
1527
64
|
jobs.append((argv[1], None, sha))
|
|
1528
65
|
else:
|
|
1529
66
|
local = True
|
|
@@ -1533,20 +70,22 @@ def pre_push(argv, stdin):
|
|
|
1533
70
|
if len(parts) != 4:
|
|
1534
71
|
continue
|
|
1535
72
|
lref, lsha, _, rsha = parts
|
|
1536
|
-
if lsha == ZERO_SHA or lref.startswith("refs/tags/"):
|
|
73
|
+
if lsha == mio.ZERO_SHA or lref.startswith("refs/tags/"):
|
|
1537
74
|
continue # a deletion or a tag: nothing of ours travels
|
|
1538
|
-
known = rsha != ZERO_SHA and _git_out(repo, "cat-file", "-e", rsha + "^{commit}")[0] == 0
|
|
75
|
+
known = rsha != mio.ZERO_SHA and _git_out(repo, "cat-file", "-e", rsha + "^{commit}")[0] == 0
|
|
1539
76
|
jobs.append((lref, rsha if known else base_for_new_branch(repo, remote, lsha), lsha))
|
|
1540
77
|
failing, shown, packages, tasks, ms = [], 0, 0, 0, 0
|
|
78
|
+
if local and jobs and not mio.shared(repo):
|
|
79
|
+
return 0 # the hook form, a local record (.ai/memory/ git-ignored): nothing travels
|
|
1541
80
|
for ref, old, new in jobs:
|
|
1542
|
-
findings, stats = validate_range(repo, old, new, local=local)
|
|
81
|
+
findings, stats = mver.validate_range(repo, old, new, local=local)
|
|
1543
82
|
if findings is None:
|
|
1544
83
|
continue # files mode at T: not ours to check
|
|
1545
|
-
for f in findings[:FINDINGS_MAX]:
|
|
1546
|
-
print("memory-pre-push: %s" % format_finding(f, ref), file=sys.stderr)
|
|
1547
|
-
if len(findings) > FINDINGS_MAX:
|
|
84
|
+
for f in findings[:mio.FINDINGS_MAX]:
|
|
85
|
+
print("memory-pre-push: %s" % mver.format_finding(f, ref), file=sys.stderr)
|
|
86
|
+
if len(findings) > mio.FINDINGS_MAX:
|
|
1548
87
|
print("… %d more — state.py memory verify --range %s..%s" % (
|
|
1549
|
-
len(findings) - FINDINGS_MAX, (old or new)[:12], new[:12]), file=sys.stderr)
|
|
88
|
+
len(findings) - mio.FINDINGS_MAX, (old or new)[:12], new[:12]), file=sys.stderr)
|
|
1550
89
|
failing += [f for f in findings if not f.check.endswith("-warn")]
|
|
1551
90
|
shown += len(findings)
|
|
1552
91
|
packages += stats.get("packages", 0)
|