riskrails 4.1.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +2 -2
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +25 -15
- package/agents/ai-reviewer.md.tmpl +14 -1
- package/ai/agents/implementer.md +6 -0
- package/ai/agents/reviewer.md +8 -1
- package/bin/ansi.js +12 -0
- package/bin/menu.js +71 -12
- package/bin/riskrails.js +240 -14
- package/bin/uninstall.js +115 -0
- package/bin/wizard.js +72 -0
- package/hooks/AGENTS.md +0 -6
- package/hooks/CLAUDE.md +0 -6
- package/hooks/context-guard.py +11 -1
- package/hooks/dialog-gate.py +3 -2
- package/hooks/guard/decision.py +4 -3
- package/hooks/guard/dialog.py +12 -2
- package/hooks/guard/ere.py +29 -10
- package/hooks/guard/git.py +2 -1
- package/hooks/guard/paths.py +48 -1406
- package/hooks/guard/paths_base.py +66 -0
- package/hooks/guard/paths_bash.py +176 -0
- package/hooks/guard/paths_classify.py +205 -0
- package/hooks/guard/paths_fs.py +137 -0
- package/hooks/guard/paths_human.py +105 -0
- package/hooks/guard/paths_interp.py +162 -0
- package/hooks/guard/paths_movers.py +393 -0
- package/hooks/guard/paths_shell.py +288 -0
- package/hooks/runtime-gate.py +87 -0
- package/hooks/update_notice.py +23 -8
- package/package.json +1 -1
- package/skills/AGENTS.md +0 -6
- package/skills/CLAUDE.md +0 -6
- package/skills/ai-init/templates/.ai/policies/coding.md +9 -0
- package/skills/ai-init/templates/gitignore.snippet +2 -0
- package/skills/ai-task/memory_cleanup.py +213 -0
- package/skills/ai-task/memory_compact.py +25 -0
- package/skills/ai-task/memory_config.py +120 -0
- package/skills/ai-task/memory_export.py +246 -0
- package/skills/ai-task/memory_format.py +7 -3
- package/skills/ai-task/memory_hook.py +172 -0
- package/skills/ai-task/memory_io.py +122 -0
- package/skills/ai-task/memory_migrate.py +187 -0
- package/skills/ai-task/memory_packages.py +83 -0
- package/skills/ai-task/memory_risks.py +84 -0
- package/skills/ai-task/memory_status.py +76 -0
- package/skills/ai-task/memory_sync.py +20 -1481
- package/skills/ai-task/memory_verify.py +316 -0
- package/skills/ai-task/rr_state/cli.py +1 -0
- package/skills/ai-task/rr_state/close.py +2 -2
- package/skills/ai-task/rr_state/dialog.py +32 -7
- package/skills/ai-task/rr_state/dialog_cmd.py +38 -5
- package/skills/ai-task/rr_state/gate.py +10 -4
- package/skills/ai-task/rr_state/labels.py +33 -4
- package/skills/ai-task/rr_state/memory_cmd.py +84 -64
- package/skills/ai-task/rr_state/questions_cmd.py +5 -4
- package/skills/ai-task/rr_state/report.py +73 -10
- package/skills/ai-task/rr_state/report_cmd.py +2 -0
- package/skills/ai-task/rr_state/review.py +18 -14
- package/skills/ai-task/rr_state/start.py +12 -9
- package/skills/ai-task/rr_state/steps.py +16 -7
- package/skills/ai-task/rr_state/verify.py +17 -12
- package/skills/project-update/history/blobs/dac90781793b8a1c8e5f3e0307665b6394d922140c3831c5508d019f79e1ed0a +41 -0
- package/skills/project-update/history/blobs/fa0cb370b024e1a30e1355912032657b08f609e47773a603c193b1e609b0f997 +8 -0
- package/skills/project-update/history/index.json +8 -4
- package/skills/project-update/rr_update/__init__.py +1 -0
- package/skills/project-update/rr_update/apply.py +188 -0
- package/skills/project-update/rr_update/blocks.py +38 -0
- package/skills/project-update/rr_update/build.py +96 -0
- package/skills/project-update/rr_update/cleanup.py +194 -0
- package/skills/project-update/rr_update/cli.py +102 -0
- package/skills/project-update/rr_update/files.py +106 -0
- package/skills/project-update/rr_update/gate.py +60 -0
- package/skills/project-update/rr_update/hints.py +128 -0
- package/skills/project-update/rr_update/history.py +117 -0
- package/skills/project-update/rr_update/kept.py +120 -0
- package/skills/project-update/rr_update/paths.py +108 -0
- package/skills/project-update/rr_update/plan.py +123 -0
- package/skills/project-update/rr_update/report.py +51 -0
- package/skills/project-update/rr_update/rules.py +178 -0
- package/skills/project-update/rr_update/shared.py +74 -0
- package/skills/project-update/rr_update/siblings.py +37 -0
- package/skills/project-update/rr_update/weights.py +171 -0
- package/skills/project-update/update.py +8 -1682
- package/skills/rr-build/SKILL.md +5 -2
- package/skills/rr-setup/SKILL.md +4 -2
- package/skills/rr-setup/setup.py +13 -12
- package/skills/rr-ship/SKILL.md +2 -2
- package/skills/rr-spec/SKILL.md +6 -0
- package/skills/rr-status/SKILL.md +2 -0
- package/skills/rr-status/status.py +5 -3
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"""The git hook (R17, §D10): the installed shim and where it points, the policy file, and the enable and
|
|
2
|
+
disable commands that write and remove them."""
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
import memory_config as mcfg
|
|
8
|
+
import memory_format as mf
|
|
9
|
+
import memory_io as mio
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# ------------------------------------------------------------------ status, show
|
|
13
|
+
def hook_state(root):
|
|
14
|
+
code, out, _ = mio.git(root, "config", "--get", "core.hooksPath")
|
|
15
|
+
if code == 0 and out.strip():
|
|
16
|
+
return "hookspath"
|
|
17
|
+
path = hooks_path(root)
|
|
18
|
+
if not path:
|
|
19
|
+
return "absent"
|
|
20
|
+
hook = os.path.join(path, "pre-push")
|
|
21
|
+
text = mio.read_bytes(hook)
|
|
22
|
+
if text is None:
|
|
23
|
+
return "absent"
|
|
24
|
+
if mio.HOOK_MARKER.encode() in text:
|
|
25
|
+
decoded = text.decode("utf-8", "replace")
|
|
26
|
+
if not any(os.path.isfile(p) for p in shim_paths(decoded)):
|
|
27
|
+
return "installed, validator missing — pushes are NOT checked ('state.py memory enable' rewrites it)"
|
|
28
|
+
return "current" if decoded == shim_text() else "installed (stale)"
|
|
29
|
+
return "foreign"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
# ------------------------------------------------------------------ the git hook (R17, §D10)
|
|
33
|
+
def hooks_path(root):
|
|
34
|
+
code, out, _ = mio.git(root, "rev-parse", "--git-path", "hooks")
|
|
35
|
+
if code != 0 or not out.strip():
|
|
36
|
+
return None
|
|
37
|
+
path = out.strip()
|
|
38
|
+
return path if os.path.isabs(path) else os.path.join(root, path)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def shim_candidates(validator=None):
|
|
42
|
+
"""Where the shim looks for the validator, in order: the hook beside this
|
|
43
|
+
module, the hook in either runtime home, then this module itself — a Codex
|
|
44
|
+
install copies skills/ai-task/ but not every hook."""
|
|
45
|
+
return [validator or mio.VALIDATOR, "$HOME/.claude/hooks/memory-pre-push.py",
|
|
46
|
+
"${CODEX_HOME:-$HOME/.codex}/hooks/memory-pre-push.py", mio.SELF,
|
|
47
|
+
"$HOME/.claude/skills/ai-task/memory_sync.py",
|
|
48
|
+
"${CODEX_HOME:-$HOME/.codex}/skills/ai-task/memory_sync.py"]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def shim_text(validator=None):
|
|
52
|
+
return ("#!/bin/sh\n%s — written by 'state.py memory enable'; 'state.py memory disable' removes it\n"
|
|
53
|
+
"for v in %s; do\n"
|
|
54
|
+
" [ -f \"$v\" ] && exec python3 -I \"$v\" \"$@\"\n"
|
|
55
|
+
"done\n"
|
|
56
|
+
"echo \"riskrails memory: validator not found — this push is not checked\" >&2; exit 0\n"
|
|
57
|
+
% (mio.HOOK_MARKER, " ".join('"%s"' % c for c in shim_candidates(validator))))
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def shim_paths(text):
|
|
61
|
+
"""The shim's candidates as paths on this machine."""
|
|
62
|
+
match = re.search(r"^for v in (.+); do$", text, re.M)
|
|
63
|
+
home = os.path.expanduser("~")
|
|
64
|
+
codex = os.environ.get("CODEX_HOME") or os.path.join(home, ".codex")
|
|
65
|
+
return [c.replace("${CODEX_HOME:-$HOME/.codex}", codex).replace("$HOME", home)
|
|
66
|
+
for c in re.findall(r'"([^"]+)"', match.group(1) if match else "")]
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def snippet_text(validator=None):
|
|
70
|
+
validator = validator or mio.VALIDATOR
|
|
71
|
+
return ("# riskrails memory: add to your own pre-push hook (stdin can be read only once)\n"
|
|
72
|
+
"rr_in=$(mktemp) && cat > \"$rr_in\"\n"
|
|
73
|
+
"python3 -I \"%s\" \"$@\" < \"$rr_in\" || { rr=$?; rm -f \"$rr_in\"; exit $rr; }\n"
|
|
74
|
+
"# … then your hook, reading \"$rr_in\" where it read stdin …\n"
|
|
75
|
+
"rm -f \"$rr_in\"\n" % validator)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def install_hook(root):
|
|
79
|
+
"""installed | current | foreign | hookspath (R17)."""
|
|
80
|
+
code, out, _ = mio.git(root, "config", "--get", "core.hooksPath")
|
|
81
|
+
if code == 0 and out.strip():
|
|
82
|
+
return "hookspath"
|
|
83
|
+
path = hooks_path(root)
|
|
84
|
+
if not path:
|
|
85
|
+
raise mio.MemoryError_("not a git repository: no hooks path", 2)
|
|
86
|
+
hook = os.path.join(path, "pre-push")
|
|
87
|
+
text = mio.read_bytes(hook)
|
|
88
|
+
if text is not None and mio.HOOK_MARKER.encode() not in text:
|
|
89
|
+
return "foreign"
|
|
90
|
+
if text is not None and text.decode("utf-8", "replace") == shim_text():
|
|
91
|
+
return "current"
|
|
92
|
+
os.makedirs(path, exist_ok=True)
|
|
93
|
+
mio.atomic_write(hook, shim_text())
|
|
94
|
+
os.chmod(hook, 0o755)
|
|
95
|
+
return "installed"
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def remove_hook(root):
|
|
99
|
+
path = hooks_path(root)
|
|
100
|
+
hook = os.path.join(path, "pre-push") if path else None
|
|
101
|
+
text = mio.read_bytes(hook) if hook else None
|
|
102
|
+
if text is not None and mio.HOOK_MARKER.encode() in text:
|
|
103
|
+
os.remove(hook)
|
|
104
|
+
return "removed"
|
|
105
|
+
return "foreign" if text is not None else "absent"
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def write_policy(root, policy):
|
|
109
|
+
mio.atomic_write(os.path.join(root, mio.POLICY_REL), json.dumps(policy, indent=2, ensure_ascii=False) + "\n")
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
GITIGNORE_LOCAL = (".ai/memory/", ".ai/reports/")
|
|
113
|
+
GITIGNORE_SHARED = ("!.ai/memory/", "!.ai/reports/")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def append_gitignore(root, lines):
|
|
117
|
+
"""Append `lines` to .gitignore (created when absent), after a newline if the file lacks one; never rewrites."""
|
|
118
|
+
path = os.path.join(root, ".gitignore")
|
|
119
|
+
current = mio.read_bytes(path) or b""
|
|
120
|
+
sep = b"\n" if current and not current.endswith(b"\n") else b""
|
|
121
|
+
with open(path, "ab") as fh:
|
|
122
|
+
fh.write(sep + "".join(line + "\n" for line in lines).encode())
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def apply_share(root, share):
|
|
126
|
+
"""--share git: the negations (the record is shared through git); --share local: the ignore lines. Append only."""
|
|
127
|
+
if share == "git":
|
|
128
|
+
have = (mio.read_bytes(os.path.join(root, ".gitignore")) or b"").decode("utf-8", "replace").splitlines()
|
|
129
|
+
add = [line for line in GITIGNORE_SHARED if line not in have]
|
|
130
|
+
if not add and not mio.shared(root):
|
|
131
|
+
add = list(GITIGNORE_SHARED)
|
|
132
|
+
if add:
|
|
133
|
+
append_gitignore(root, add)
|
|
134
|
+
elif share == "local" and mio.shared(root):
|
|
135
|
+
append_gitignore(root, GITIGNORE_LOCAL)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def enable(root, author=None, period=None, index=None, share=None):
|
|
139
|
+
"""Packages mode on. The hook is installed only while the record is shared (.ai/memory/ not git-ignored);
|
|
140
|
+
a local record has no hook of ours and the state is reported as `local`."""
|
|
141
|
+
cfg = mcfg.load_config(root)
|
|
142
|
+
policy = dict(mcfg.DEFAULTS)
|
|
143
|
+
policy.update(cfg.policy if cfg.present else {})
|
|
144
|
+
policy["mode"] = "packages"
|
|
145
|
+
if period:
|
|
146
|
+
if period not in mf.PERIODS:
|
|
147
|
+
raise mio.MemoryError_("--period takes one of %s" % ", ".join(mf.PERIODS))
|
|
148
|
+
policy["period"] = period
|
|
149
|
+
write_policy(root, policy)
|
|
150
|
+
local = dict(cfg.local)
|
|
151
|
+
local["schema"] = 1
|
|
152
|
+
local["author"] = mcfg.author_slug(author) if author else cfg.author
|
|
153
|
+
if index:
|
|
154
|
+
if index not in ("none", "sqlite"):
|
|
155
|
+
raise mio.MemoryError_("--index takes none or sqlite")
|
|
156
|
+
local["index"] = index
|
|
157
|
+
local.setdefault("index", "none")
|
|
158
|
+
mio.atomic_write(mio.local_path(root, "config.json"), json.dumps(local, sort_keys=True) + "\n")
|
|
159
|
+
if share:
|
|
160
|
+
apply_share(root, share)
|
|
161
|
+
if not mio.shared(root):
|
|
162
|
+
remove_hook(root)
|
|
163
|
+
return "local"
|
|
164
|
+
return install_hook(root)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def disable(root):
|
|
168
|
+
cfg = mcfg.load_config(root)
|
|
169
|
+
policy = dict(cfg.policy)
|
|
170
|
+
policy["mode"] = "files"
|
|
171
|
+
write_policy(root, policy)
|
|
172
|
+
return remove_hook(root)
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""The small I/O of packages mode: the path constants, the refusal type, the clock, fault
|
|
2
|
+
injection, JSON and byte reads, the atomic write, the clone lock and the git runner. It imports no other
|
|
3
|
+
memory module; HERE is the one place that reads its own path (SELF stays the validator entry's path)."""
|
|
4
|
+
import contextlib
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import subprocess
|
|
8
|
+
import sys
|
|
9
|
+
import tempfile
|
|
10
|
+
from datetime import datetime, timezone
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
try:
|
|
14
|
+
import fcntl
|
|
15
|
+
except ImportError: # not POSIX: last writer wins within one clone (docs/local-memory.md)
|
|
16
|
+
fcntl = None
|
|
17
|
+
|
|
18
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
19
|
+
|
|
20
|
+
POLICY_REL = os.path.join(".ai", "policies", "memory.json")
|
|
21
|
+
MEMORY_REL = os.path.join(".ai", "memory")
|
|
22
|
+
LOCAL_REL = os.path.join(".ai", "local", "memory")
|
|
23
|
+
PENDING_MAX = 5
|
|
24
|
+
BATCH_CHARS = "0123456789abcdefghjkmnpqrstvwxyz"
|
|
25
|
+
HOOK_MARKER = "# riskrails-memory-pre-push 1"
|
|
26
|
+
VALIDATOR = os.path.normpath(os.path.join(HERE, "..", "..", "hooks", "memory-pre-push.py"))
|
|
27
|
+
SELF = os.path.join(HERE, "memory_sync.py") # the fallback validator: the entry, run as a script
|
|
28
|
+
FINDINGS_MAX = 40
|
|
29
|
+
ZERO_SHA = "0" * 40
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class MemoryError_(Exception): # pylint: disable=invalid-name # not the builtin MemoryError
|
|
33
|
+
"""A refusal with its exit code (§I4: 1 findings, 2 usage, 3 aborted, 5 human, 6 scope)."""
|
|
34
|
+
|
|
35
|
+
def __init__(self, message, code=2):
|
|
36
|
+
super().__init__(message)
|
|
37
|
+
self.code = code
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def now():
|
|
41
|
+
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def crash(point):
|
|
45
|
+
"""Fault injection (A9): AI_MEMORY_CRASH_AT=B2..B6 ends the process here."""
|
|
46
|
+
if os.environ.get("AI_MEMORY_CRASH_AT") == point:
|
|
47
|
+
sys.stdout.flush()
|
|
48
|
+
sys.stderr.flush()
|
|
49
|
+
os._exit(70) # pylint: disable=protected-access # a crash, not an exit
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def read_json(path):
|
|
53
|
+
with open(path, encoding="utf-8") as fh:
|
|
54
|
+
return json.load(fh)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
# ------------------------------------------------------------------ small I/O
|
|
58
|
+
def atomic_write(path, data):
|
|
59
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
60
|
+
handle, tmp = tempfile.mkstemp(prefix=".tmp-", dir=os.path.dirname(path))
|
|
61
|
+
try:
|
|
62
|
+
with os.fdopen(handle, "wb") as fh:
|
|
63
|
+
fh.write(data if isinstance(data, bytes) else data.encode("utf-8"))
|
|
64
|
+
fh.flush()
|
|
65
|
+
os.fsync(fh.fileno())
|
|
66
|
+
os.replace(tmp, path)
|
|
67
|
+
except BaseException:
|
|
68
|
+
with contextlib.suppress(OSError):
|
|
69
|
+
os.remove(tmp)
|
|
70
|
+
raise
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@contextlib.contextmanager
|
|
74
|
+
def locked(root):
|
|
75
|
+
"""flock on .ai/local/memory/lock: two closes in one clone serialise here."""
|
|
76
|
+
os.makedirs(os.path.join(root, LOCAL_REL), exist_ok=True)
|
|
77
|
+
fd = os.open(os.path.join(root, LOCAL_REL, "lock"), os.O_RDWR | os.O_CREAT, 0o644)
|
|
78
|
+
try:
|
|
79
|
+
if fcntl is not None:
|
|
80
|
+
fcntl.flock(fd, fcntl.LOCK_EX)
|
|
81
|
+
yield
|
|
82
|
+
finally:
|
|
83
|
+
os.close(fd)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def git(root, *args, stdin=None, binary=False):
|
|
87
|
+
done = subprocess.run(["git", "-C", root, *args], input=stdin, capture_output=True, check=False,
|
|
88
|
+
text=not binary)
|
|
89
|
+
return done.returncode, done.stdout, done.stderr
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def shared(root):
|
|
93
|
+
"""False only when git ignores .ai/memory/ (the record is local, nothing of it travels); True when it does not,
|
|
94
|
+
and when git cannot answer: the validator then keeps checking."""
|
|
95
|
+
try:
|
|
96
|
+
return git(root, "check-ignore", "-q", "--no-index", ".ai/memory/x")[0] != 0
|
|
97
|
+
except OSError:
|
|
98
|
+
return True
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def current_branch(root):
|
|
102
|
+
code, out, _ = git(root, "symbolic-ref", "--short", "-q", "HEAD")
|
|
103
|
+
if code == 0 and out.strip():
|
|
104
|
+
return out.strip()
|
|
105
|
+
code, out, _ = git(root, "rev-parse", "--short=8", "HEAD")
|
|
106
|
+
return "HEAD@%s" % (out.strip() if code == 0 and out.strip() else "unborn")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def memory_dir(root):
|
|
110
|
+
return os.path.join(root, MEMORY_REL)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def local_path(root, *parts):
|
|
114
|
+
return os.path.join(root, LOCAL_REL, *parts)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def read_bytes(path):
|
|
118
|
+
try:
|
|
119
|
+
with open(path, "rb") as fh:
|
|
120
|
+
return fh.read()
|
|
121
|
+
except OSError:
|
|
122
|
+
return None
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""Migration (§D12, R25): the plan and report of a move from files to packages, its backup and apply, the
|
|
2
|
+
risks stub and the move of agent logs."""
|
|
3
|
+
import collections
|
|
4
|
+
import os
|
|
5
|
+
import shutil
|
|
6
|
+
import tarfile
|
|
7
|
+
from datetime import datetime, timezone
|
|
8
|
+
|
|
9
|
+
import memory_compact as mc
|
|
10
|
+
import memory_config as mcfg
|
|
11
|
+
import memory_format as mf
|
|
12
|
+
import memory_io as mio
|
|
13
|
+
import memory_packages as mpkg
|
|
14
|
+
import memory_risks as mrisks
|
|
15
|
+
import memory_verify as mver
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
# ------------------------------------------------------------------ migration (§D12, R25)
|
|
19
|
+
def migration_plan(root):
|
|
20
|
+
"""What --report shows and --apply does, from HEAD's tracked files only."""
|
|
21
|
+
head = mver.rev(root, "HEAD")
|
|
22
|
+
if not head:
|
|
23
|
+
raise mio.MemoryError_("migration reads HEAD: commit first", 2)
|
|
24
|
+
tree = mver.GitTreeReader(root, head)
|
|
25
|
+
packages = mpkg.load_packages(root)
|
|
26
|
+
sealed = {mf.key_of(e) for _, entries, _ in packages.values() for e in entries if e.get("kind") == "task"}
|
|
27
|
+
tasks = sorted({p.split("/")[2] for p in tree.paths() if p.startswith(".ai/reports/T-")})
|
|
28
|
+
plan = []
|
|
29
|
+
for task in tasks:
|
|
30
|
+
found = mc.compact_instances(tree, task, "tracked")
|
|
31
|
+
journal = tree.read(".ai/reports/%s/events.jsonl" % task)
|
|
32
|
+
names = {e.get("task") for _, e in mc.parse_journal(journal) if e} if journal else set()
|
|
33
|
+
logs = [(p, len(tree.read(p) or b"")) for p in tree.paths()
|
|
34
|
+
if p.startswith(".ai/reports/%s/" % task) and p.endswith(".log")]
|
|
35
|
+
plan.append({"task": task, "compactions": found, "journal": journal is not None,
|
|
36
|
+
"names": sorted(n for n in names if n), "logs": logs,
|
|
37
|
+
"sealed": [mf.key_of(c.seal) for c in found if c.seal and mf.key_of(c.seal) in sealed]})
|
|
38
|
+
return head, plan
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def migrate_report(root):
|
|
42
|
+
head, plan = migration_plan(root)
|
|
43
|
+
kinds, dropped = collections.Counter(), collections.Counter()
|
|
44
|
+
lines = ["# Migration report — %s" % mio.now(), "", "HEAD %s; tracked files only, so every clone gives the same "
|
|
45
|
+
"rids." % head[:12], "", "## Tasks", "",
|
|
46
|
+
"| task | instances | journal | name in journal | coverage | result | sealed already |",
|
|
47
|
+
"|---|---|---|---|---|---|---|"]
|
|
48
|
+
total_logs = []
|
|
49
|
+
for item in plan:
|
|
50
|
+
found = item["compactions"]
|
|
51
|
+
for c in found:
|
|
52
|
+
if c.seal:
|
|
53
|
+
for e in c.entries:
|
|
54
|
+
kinds[e["kind"]] += 1
|
|
55
|
+
dropped.update(c.seal["counts"]["dropped"])
|
|
56
|
+
cov = ", ".join("%s/%s" % (c.coverage["accounted"], c.coverage["lines"]) for c in found) or "-"
|
|
57
|
+
names = ", ".join(item["names"]) or "-"
|
|
58
|
+
mismatch = " ≠ dir" if item["names"] and item["names"] != [item["task"]] else ""
|
|
59
|
+
lines.append("| %s | %d | %s | %s%s | %s | %s | %s |" % (
|
|
60
|
+
item["task"], len(found), "yes" if item["journal"] else "no (state-only)", names, mismatch, cov,
|
|
61
|
+
", ".join(c.seal["result"] for c in found if c.seal) or "no sources",
|
|
62
|
+
", ".join(item["sealed"]) or "-"))
|
|
63
|
+
total_logs += item["logs"]
|
|
64
|
+
lines += ["", "## Kept per kind", ""] + ["- %s: %d" % (k, n) for k, n in sorted(kinds.items())]
|
|
65
|
+
lines += ["", "## Dropped per event (KEEP_RULES v1)", ""] + ["- %s: %d" % (k, n) for k, n in sorted(dropped.items())]
|
|
66
|
+
lines += ["", "## Committed logs (%d, %s bytes)" % (len(total_logs), mf.fmt_n(sum(s for _, s in total_logs))), ""]
|
|
67
|
+
lines += ["- %s (%s bytes)" % (p, mf.fmt_n(s)) for p, s in total_logs]
|
|
68
|
+
text = (mio.read_bytes(os.path.join(root, ".ai", "project", "known-risks.md")) or b"").decode("utf-8")
|
|
69
|
+
risks, leftovers = mc.parse_known_risks(text)
|
|
70
|
+
titles = collections.Counter(r["title"] for r in risks)
|
|
71
|
+
lines += ["", "## known-risks.md", "", "- sections parsed: %d" % len(risks),
|
|
72
|
+
"- lines outside a section: %d" % len(leftovers["lines"]),
|
|
73
|
+
"- label derived (no Label line): %d" % sum(1 for r in risks if r["label_derived"]),
|
|
74
|
+
"- renders byte for byte: %s" % ("yes" if mc.render_known_risks(risks, leftovers) == text else "NO"),
|
|
75
|
+
"- duplicate titles: %s" % (", ".join(t for t, n in titles.items() if n > 1) or "none")]
|
|
76
|
+
report = "\n".join(lines) + "\n"
|
|
77
|
+
mio.atomic_write(mio.local_path(root, "migration-report.md"), report)
|
|
78
|
+
return report
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def backup(root):
|
|
82
|
+
path = mio.local_path(root, "backup-%s.tar" % datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ"))
|
|
83
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
84
|
+
with tarfile.open(path, "w") as tar:
|
|
85
|
+
for rel in (".ai/reports", ".ai/project/known-risks.md"):
|
|
86
|
+
if os.path.exists(os.path.join(root, rel)):
|
|
87
|
+
tar.add(os.path.join(root, rel), arcname=rel)
|
|
88
|
+
return path
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def migrate_apply(root, risks=False):
|
|
92
|
+
"""Closed legacy tasks into batches by their closed_at period; then, with
|
|
93
|
+
risks, known-risks.md into risk entries and a stub."""
|
|
94
|
+
cfg = mcfg.load_config(root)
|
|
95
|
+
saved = backup(root)
|
|
96
|
+
out = ["backup: %s" % os.path.relpath(saved, root)]
|
|
97
|
+
if risks:
|
|
98
|
+
return out + migrate_risks(root, cfg)
|
|
99
|
+
_, plan = migration_plan(root)
|
|
100
|
+
groups = collections.defaultdict(list)
|
|
101
|
+
skipped, incomplete = [], []
|
|
102
|
+
for item in plan:
|
|
103
|
+
for c in item["compactions"]:
|
|
104
|
+
if not c.seal or c.seal["result"] not in ("closed", "abandoned", "restarted"):
|
|
105
|
+
continue
|
|
106
|
+
key = mf.key_of(c.seal)
|
|
107
|
+
if key in item["sealed"]:
|
|
108
|
+
skipped.append(key)
|
|
109
|
+
continue
|
|
110
|
+
if c.coverage["accounted"] != c.coverage["lines"] or c.coverage["skipped"] or c.problems:
|
|
111
|
+
incomplete.append((key, c.coverage, c.problems))
|
|
112
|
+
continue
|
|
113
|
+
when = c.seal.get("closed_at") or c.seal.get("started_at") or c.seal["ts"]
|
|
114
|
+
groups[mf.period_of(when, cfg.period)].append(c)
|
|
115
|
+
written = 0
|
|
116
|
+
with mio.locked(root):
|
|
117
|
+
for period in sorted(groups):
|
|
118
|
+
# Whole tasks per batch, a new batch before package_max_bytes (V8).
|
|
119
|
+
batches, size = [[]], 0
|
|
120
|
+
for c in groups[period]:
|
|
121
|
+
task_bytes = sum(len(mf.canonical(e)) + 1 for e in c.entries + [c.seal])
|
|
122
|
+
if batches[-1] and size + task_bytes > cfg.package_max_bytes:
|
|
123
|
+
batches.append([])
|
|
124
|
+
size = 0
|
|
125
|
+
batches[-1].append(c)
|
|
126
|
+
size += task_bytes
|
|
127
|
+
for batch in batches:
|
|
128
|
+
name = mpkg.new_batch_name(root, period, cfg.author)
|
|
129
|
+
parts = mf.parse_package_name(name)
|
|
130
|
+
entries = [e for c in batch for e in c.entries + [c.seal]]
|
|
131
|
+
mio.atomic_write(os.path.join(mio.memory_dir(root), name),
|
|
132
|
+
mf.render_package(mf.package_header(period, cfg.author, parts["batch"], mio.now()),
|
|
133
|
+
entries))
|
|
134
|
+
written += len(batch)
|
|
135
|
+
out.append("package: %s (%d tasks, %d entries)" % (os.path.join(mio.MEMORY_REL, name), len(batch),
|
|
136
|
+
len(entries)))
|
|
137
|
+
findings = mver.check_packages(mpkg.load_packages(root), cfg.package_max_bytes)
|
|
138
|
+
out.append("migrated %d task instance(s); %d already sealed; %d incomplete (reported, not exported)"
|
|
139
|
+
% (written, len(skipped), len(incomplete)))
|
|
140
|
+
out += ["incomplete: %s coverage %s/%s %s" % (k, cov["accounted"], cov["lines"], "; ".join(p))
|
|
141
|
+
for k, cov, p in incomplete]
|
|
142
|
+
out += [mver.format_finding(f) for f in findings]
|
|
143
|
+
return out
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
STUB = """# Known risks
|
|
147
|
+
|
|
148
|
+
<!-- The risks live in the record since {date}: `state.py memory risks --md` renders
|
|
149
|
+
them, `state.py memory risk add|close` records them (spec compact-project-memory
|
|
150
|
+
§D9, OQ4). A section written here is still read, as before the migration. -->
|
|
151
|
+
"""
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def migrate_risks(root, cfg):
|
|
155
|
+
path = os.path.join(root, ".ai", "project", "known-risks.md")
|
|
156
|
+
text = (mio.read_bytes(path) or b"").decode("utf-8")
|
|
157
|
+
import memory_store # pylint: disable=import-outside-toplevel
|
|
158
|
+
entries = memory_store.risk_entries_from_md(text)
|
|
159
|
+
risks, leftovers = mc.parse_known_risks(text)
|
|
160
|
+
rendered = mc.render_known_risks([{"text": e["text"]} for e in entries], leftovers)
|
|
161
|
+
if rendered != text:
|
|
162
|
+
raise mio.MemoryError_("known-risks.md does not render byte for byte from its entries — nothing changed", 3)
|
|
163
|
+
entries = [dict(e, actor="migration", src=["known-risks.md#%s" % e["rkey"]]) for e in entries]
|
|
164
|
+
entries = [mf.with_rid(e) for e in entries]
|
|
165
|
+
name = mrisks.append_entries(root, cfg, entries)
|
|
166
|
+
mio.atomic_write(path, STUB.format(date=mio.now()[:10]))
|
|
167
|
+
del risks
|
|
168
|
+
return ["risks: %d entries into %s; known-risks.md is a stub" % (len(entries), os.path.join(mio.MEMORY_REL, name))]
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def move_logs(root):
|
|
172
|
+
head = mver.rev(root, "HEAD")
|
|
173
|
+
tree = mver.GitTreeReader(root, head) if head else None
|
|
174
|
+
logs = [p for p in (tree.paths() if tree else []) if p.startswith(".ai/reports/") and p.endswith(".log")]
|
|
175
|
+
moved = []
|
|
176
|
+
for rel in logs:
|
|
177
|
+
task = rel.split("/")[2]
|
|
178
|
+
src = os.path.join(root, rel)
|
|
179
|
+
if not os.path.exists(src):
|
|
180
|
+
continue
|
|
181
|
+
dest = os.path.join(root, ".ai", "state", "logs", task, "legacy", os.path.basename(rel))
|
|
182
|
+
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
|
183
|
+
shutil.move(src, dest)
|
|
184
|
+
moved.append(rel)
|
|
185
|
+
listing = mio.local_path(root, "logs-to-rm.txt")
|
|
186
|
+
mio.atomic_write(listing, "".join("%s\n" % p for p in moved))
|
|
187
|
+
return moved, "git rm --cached --quiet --pathspec-from-file=%s" % os.path.relpath(listing, root)
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""The packages on disk: loading them, the id and seal lookups over them, the batch ledger and the open
|
|
2
|
+
batch an export writes into."""
|
|
3
|
+
import collections
|
|
4
|
+
import contextlib
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import random
|
|
8
|
+
|
|
9
|
+
import memory_format as mf
|
|
10
|
+
import memory_io as mio
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
# ------------------------------------------------------------------ packages on disk
|
|
14
|
+
def load_packages(root):
|
|
15
|
+
"""{name: (header, entries, errors)} for every package in the working tree."""
|
|
16
|
+
out = {}
|
|
17
|
+
base = mio.memory_dir(root)
|
|
18
|
+
for name in sorted(os.listdir(base)) if os.path.isdir(base) else []:
|
|
19
|
+
if name.endswith(".jsonl"):
|
|
20
|
+
out[name] = mf.parse_package(mio.read_bytes(os.path.join(base, name)) or b"", name)
|
|
21
|
+
return out
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def all_rids(packages):
|
|
25
|
+
return {e["rid"] for _, entries, _ in packages.values() for e in entries}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def seals_by_key(packages):
|
|
29
|
+
out = collections.defaultdict(list)
|
|
30
|
+
for name, (_, entries, _) in packages.items():
|
|
31
|
+
for e in entries:
|
|
32
|
+
if e.get("kind") == "task":
|
|
33
|
+
out[mf.key_of(e)].append((name, e))
|
|
34
|
+
return out
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def read_batches(root):
|
|
38
|
+
with contextlib.suppress(OSError, ValueError):
|
|
39
|
+
data = mio.read_json(mio.local_path(root, "batches.json"))
|
|
40
|
+
if isinstance(data, dict) and isinstance(data.get("open"), dict):
|
|
41
|
+
return data
|
|
42
|
+
return {"schema": 1, "open": {}}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def write_batches(root, data):
|
|
46
|
+
mio.atomic_write(mio.local_path(root, "batches.json"), json.dumps(data, sort_keys=True, indent=1) + "\n")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def new_batch_name(root, period, author):
|
|
50
|
+
rng = random.SystemRandom()
|
|
51
|
+
for _ in range(64):
|
|
52
|
+
batch = "".join(rng.choice(mio.BATCH_CHARS) for _ in range(4))
|
|
53
|
+
name = mf.package_name(period, author, batch)
|
|
54
|
+
if not os.path.exists(os.path.join(mio.memory_dir(root), name)):
|
|
55
|
+
return name
|
|
56
|
+
raise mio.MemoryError_("no free batch name for %s.%s" % (period, author), 3)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def open_batch(root, cfg, stamp=None):
|
|
60
|
+
"""The branch's open batch, reused while it is in the tree, the author's,
|
|
61
|
+
of the current period and under package_max_bytes; else a new name.
|
|
62
|
+
Called under the lock."""
|
|
63
|
+
stamp = stamp or mio.now()
|
|
64
|
+
period = mf.period_of(stamp, cfg.period)
|
|
65
|
+
branch = mio.current_branch(root)
|
|
66
|
+
books = read_batches(root)
|
|
67
|
+
name = books["open"].get(branch)
|
|
68
|
+
if name:
|
|
69
|
+
parts = mf.parse_package_name(name)
|
|
70
|
+
path = os.path.join(mio.memory_dir(root), name)
|
|
71
|
+
header = None
|
|
72
|
+
data = mio.read_bytes(path)
|
|
73
|
+
if data:
|
|
74
|
+
header, _, _ = mf.parse_package(data.split(b"\n", 1)[0] + b"\n", name)
|
|
75
|
+
if parts and data and header and header.get("author") == cfg.author and parts["period"] == period \
|
|
76
|
+
and parts["batch"] and len(data) < cfg.package_max_bytes:
|
|
77
|
+
return name, branch, books
|
|
78
|
+
name = new_batch_name(root, period, cfg.author)
|
|
79
|
+
return name, branch, books
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def sealed_task_ids(packages):
|
|
83
|
+
return {e.get("task") for _, entries, _ in packages.values() for e in entries if e.get("kind") == "task"}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Risks in the record (§D9): the files-mode and packages-mode writers of a risk entry and the journal
|
|
2
|
+
append they share."""
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
import memory_compact as mc
|
|
6
|
+
import memory_format as mf
|
|
7
|
+
import memory_io as mio
|
|
8
|
+
import memory_packages as mpkg
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
# ------------------------------------------------------------------ risks (§D9)
|
|
12
|
+
def risk_add_files(root, title, where, label, what, likely, fix, status, found):
|
|
13
|
+
"""Files mode: append the template section (plus Label) to known-risks.md."""
|
|
14
|
+
import memory_compact # pylint: disable=import-outside-toplevel,reimported
|
|
15
|
+
path = os.path.join(root, ".ai", "project", "known-risks.md")
|
|
16
|
+
text = mio.read_bytes(path)
|
|
17
|
+
text = text.decode("utf-8") if text is not None else "# Known risks\n"
|
|
18
|
+
risks, _ = memory_compact.parse_known_risks(text)
|
|
19
|
+
rkey = memory_compact.rkey_of(title)
|
|
20
|
+
if any(r["rkey"] == rkey for r in risks):
|
|
21
|
+
raise mio.MemoryError_("a risk titled %r is already recorded (%s)" % (title, rkey), 2)
|
|
22
|
+
section = memory_compact.risk_section(title, where, label, what, likely, fix, status, found)
|
|
23
|
+
body = text.rstrip("\n") + "\n\n" + "\n".join(section) + "\n"
|
|
24
|
+
mio.atomic_write(path, body)
|
|
25
|
+
return rkey
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def risk_close_files(root, rkey, status, why):
|
|
29
|
+
import memory_compact # pylint: disable=import-outside-toplevel,reimported
|
|
30
|
+
path = os.path.join(root, ".ai", "project", "known-risks.md")
|
|
31
|
+
text = (mio.read_bytes(path) or b"").decode("utf-8")
|
|
32
|
+
risks, leftovers = memory_compact.parse_known_risks(text)
|
|
33
|
+
hit = [r for r in risks if r["rkey"] == rkey]
|
|
34
|
+
if not hit:
|
|
35
|
+
raise mio.MemoryError_("no risk %s in known-risks.md ('state.py memory risks --status all')" % rkey, 1)
|
|
36
|
+
lines = hit[0]["text"].split("\n")
|
|
37
|
+
for i, line in enumerate(lines):
|
|
38
|
+
if line.startswith("- **Status**:"):
|
|
39
|
+
lines[i] = "- **Status**: %s — %s" % (status, why)
|
|
40
|
+
break
|
|
41
|
+
else:
|
|
42
|
+
lines.append("- **Status**: %s — %s" % (status, why))
|
|
43
|
+
hit[0]["text"] = "\n".join(lines)
|
|
44
|
+
mio.atomic_write(path, memory_compact.render_known_risks(risks, leftovers))
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def risk_data(title, where, label, what, likely, fix, status, why=None, supersedes=None, by=None):
|
|
48
|
+
data = {"rkey": mc.rkey_of(title), "title": title, "where": where, "label": label, "what": what,
|
|
49
|
+
"likelihood": likely, "fix": fix, "status": status, "status_why": why, "supersedes": supersedes}
|
|
50
|
+
if by:
|
|
51
|
+
data["by"] = by
|
|
52
|
+
return data
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def standalone_risk(root, cfg, data, runtime="unknown", by=None):
|
|
56
|
+
"""A risk with no task in flight: one entry, straight into the open batch."""
|
|
57
|
+
entry = {"v": mf.FORMAT, "task": None, "inst": 1, "seq": 0, "kind": "risk", "ts": mio.now(), "actor": "agent",
|
|
58
|
+
"by": by, "runtime": runtime if runtime in mf.RUNTIMES else "unknown", "src": ["state.py memory risk"]}
|
|
59
|
+
entry.update({k: data.get(k) for k in ("rkey", "title", "where", "label", "what", "likelihood", "fix",
|
|
60
|
+
"status", "status_why")})
|
|
61
|
+
entry.update(found_by=None, text=None)
|
|
62
|
+
if data.get("supersedes"):
|
|
63
|
+
entry["supersedes"] = data["supersedes"]
|
|
64
|
+
entry = mf.with_rid(entry)
|
|
65
|
+
append_entries(root, cfg, [entry])
|
|
66
|
+
return entry
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def append_entries(root, cfg, entries):
|
|
70
|
+
"""Entries outside a close (a standalone risk, a redaction) into the open batch."""
|
|
71
|
+
with mio.locked(root):
|
|
72
|
+
name, branch, books = mpkg.open_batch(root, cfg)
|
|
73
|
+
packages = mpkg.load_packages(root)
|
|
74
|
+
have = mpkg.all_rids(packages)
|
|
75
|
+
header, current, _ = packages.get(name, (None, [], []))
|
|
76
|
+
parts = mf.parse_package_name(name)
|
|
77
|
+
header = header or mf.package_header(parts["period"], cfg.author, parts["batch"], mio.now())
|
|
78
|
+
new = [e for e in entries if e["rid"] not in have]
|
|
79
|
+
if new:
|
|
80
|
+
mio.atomic_write(os.path.join(mio.memory_dir(root), name),
|
|
81
|
+
mf.render_package(header, [mf.strip_private(e) for e in current] + new))
|
|
82
|
+
books["open"][branch] = name
|
|
83
|
+
mpkg.write_batches(root, books)
|
|
84
|
+
return name
|