polyctl 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- polyctl/__init__.py +7 -0
- polyctl/__main__.py +5 -0
- polyctl/a2a.py +226 -0
- polyctl/a2a_mcp.py +546 -0
- polyctl/acp.py +417 -0
- polyctl/adapters/README.md +101 -0
- polyctl/adapters/aider.json +32 -0
- polyctl/adapters/amazon-q.json +15 -0
- polyctl/adapters/amp.json +16 -0
- polyctl/adapters/antigravity.json +120 -0
- polyctl/adapters/claude-code.json +229 -0
- polyctl/adapters/cloud-generic.json +57 -0
- polyctl/adapters/codex.json +47 -0
- polyctl/adapters/command-code.json +45 -0
- polyctl/adapters/copilot.json +17 -0
- polyctl/adapters/cursor-agent.json +29 -0
- polyctl/adapters/deepseek.json +13 -0
- polyctl/adapters/droid.json +34 -0
- polyctl/adapters/gemini.json +30 -0
- polyctl/adapters/grok.json +22 -0
- polyctl/adapters/hermes.json +34 -0
- polyctl/adapters/kilocode.json +30 -0
- polyctl/adapters/kimi.json +23 -0
- polyctl/adapters/kiro.json +332 -0
- polyctl/adapters/ollama.json +29 -0
- polyctl/adapters/omp.json +34 -0
- polyctl/adapters/openai-api.json +68 -0
- polyctl/adapters/opencode.json +162 -0
- polyctl/adapters/pi.json +24 -0
- polyctl/adapters/pr-agent.json +54 -0
- polyctl/adapters/profiles.json +9 -0
- polyctl/adapters/qoder.json +163 -0
- polyctl/adapters/qwen.json +27 -0
- polyctl/adapters/reasonix.json +25 -0
- polyctl/adapters/rewriters/rtk.json +24 -0
- polyctl/adapters/trae.json +31 -0
- polyctl/adapters/vendors.json +42 -0
- polyctl/adapters/warp.json +88 -0
- polyctl/allowlist.py +1756 -0
- polyctl/api_review.py +920 -0
- polyctl/cli.py +835 -0
- polyctl/conformance_harness.py +9 -0
- polyctl/demo.py +503 -0
- polyctl/drivers.py +146 -0
- polyctl/email.py +160 -0
- polyctl/features.py +98 -0
- polyctl/journal.py +144 -0
- polyctl/language.py +945 -0
- polyctl/lib.py +1296 -0
- polyctl/matrix.py +660 -0
- polyctl/mcp.py +1362 -0
- polyctl/ntfy.py +169 -0
- polyctl/outbound.py +39 -0
- polyctl/parts/__init__.py +5 -0
- polyctl/parts/cli_authority.py +2615 -0
- polyctl/parts/cli_channels.py +1325 -0
- polyctl/parts/cli_checks.py +2293 -0
- polyctl/parts/cli_completion.py +686 -0
- polyctl/parts/cli_evidence.py +141 -0
- polyctl/parts/cli_gain.py +375 -0
- polyctl/parts/cli_harness.py +392 -0
- polyctl/parts/cli_hooks.py +1816 -0
- polyctl/parts/cli_install.py +638 -0
- polyctl/parts/cli_lanes.py +106 -0
- polyctl/parts/cli_maintenance.py +2536 -0
- polyctl/parts/cli_next.py +248 -0
- polyctl/parts/cli_pr.py +53 -0
- polyctl/parts/cli_project.py +2857 -0
- polyctl/parts/cli_review.py +5158 -0
- polyctl/parts/cli_status.py +898 -0
- polyctl/parts/lib_adapters.py +2933 -0
- polyctl/parts/lib_alarms.py +2106 -0
- polyctl/parts/lib_events.py +224 -0
- polyctl/parts/lib_evidence.py +412 -0
- polyctl/parts/lib_gates.py +2099 -0
- polyctl/parts/lib_lanes.py +829 -0
- polyctl/parts/lib_mail.py +1541 -0
- polyctl/parts/lib_next.py +652 -0
- polyctl/parts/lib_pr.py +331 -0
- polyctl/parts/lib_slices.py +3337 -0
- polyctl/parts/lib_state.py +2415 -0
- polyctl/parts/lib_transcript.py +4118 -0
- polyctl/pressure.py +325 -0
- polyctl/prices.json +47 -0
- polyctl/procs.py +1194 -0
- polyctl/settings.py +505 -0
- polyctl/skill/SKILL.md +365 -0
- polyctl/skillkit.py +503 -0
- polyctl/storage.py +2055 -0
- polyctl/telegram.py +327 -0
- polyctl/tiers.py +123 -0
- polyctl/verdicts.py +41 -0
- polyctl/watchdog.py +3887 -0
- polyctl/web.py +337 -0
- polyctl-0.7.0.dist-info/METADATA +235 -0
- polyctl-0.7.0.dist-info/RECORD +99 -0
- polyctl-0.7.0.dist-info/WHEEL +4 -0
- polyctl-0.7.0.dist-info/entry_points.txt +5 -0
- polyctl-0.7.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,3337 @@
|
|
|
1
|
+
"""Slices and projects: recall, worktrees, secondary projects, outcomes, bounded stores, hunting,
|
|
2
|
+
backup, content, boundaries.
|
|
3
|
+
|
|
4
|
+
A part of src/polyctl/lib.py (#44): moved out byte for byte and run in its namespace by `_part`,
|
|
5
|
+
where it stood; it is not importable on its own.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
# ---- recall: what was decided, found or learned before, in every project (#43) ----------
|
|
10
|
+
|
|
11
|
+
# Records are written in every language a project may choose, so the words that relate them are read in each.
|
|
12
|
+
_RECALL_STOP = frozenset(language.words("recall-stop"))
|
|
13
|
+
_RECALL_LETTERS = "".join(re.escape(letter) for letter in language.words("letters"))
|
|
14
|
+
_RECALL_WORD = re.compile(rf"[a-z{_RECALL_LETTERS}0-9][a-z{_RECALL_LETTERS}0-9_.-]*")
|
|
15
|
+
_FINDING_LINE = re.compile(r"^\s*- \[(BLOCKER|HIGH|MEDIUM|LOW)\]\s*(.+)$")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _recall_words(text):
|
|
19
|
+
words = (word.strip("._-") for word in _RECALL_WORD.findall(str(text or "").lower()))
|
|
20
|
+
return {word for word in words if len(word) >= 3 and word not in _RECALL_STOP}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _epoch(value):
|
|
24
|
+
"""A recorded time as seconds, whether polyctl wrote it as a number or as ISO text."""
|
|
25
|
+
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
|
26
|
+
return float(value)
|
|
27
|
+
try:
|
|
28
|
+
return datetime.fromisoformat(str(value)).timestamp()
|
|
29
|
+
except (TypeError, ValueError):
|
|
30
|
+
return None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def recall_roots(root=None):
|
|
34
|
+
"""(project, root) for every project this machine has registered, the one asking first."""
|
|
35
|
+
out, seen = [], set()
|
|
36
|
+
if root:
|
|
37
|
+
out.append((project_key(root), root))
|
|
38
|
+
seen.add(os.path.realpath(root))
|
|
39
|
+
for name, row in sorted(project_registry().items()):
|
|
40
|
+
path = row.get("root")
|
|
41
|
+
if path and os.path.isdir(path) and os.path.realpath(path) not in seen:
|
|
42
|
+
seen.add(os.path.realpath(path))
|
|
43
|
+
out.append((name, path))
|
|
44
|
+
return out
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def recall_entries(project, root):
|
|
48
|
+
"""What one project remembers, each with the file and row it came from (#43).
|
|
49
|
+
|
|
50
|
+
Questions asked and their answers, architect decisions, waivers, review
|
|
51
|
+
findings with their review's verdict, and lessons. A source that cannot be
|
|
52
|
+
read is passed over; the rest still answer.
|
|
53
|
+
"""
|
|
54
|
+
entries = []
|
|
55
|
+
|
|
56
|
+
def add(kind, ident, at, text, outcome, source):
|
|
57
|
+
entries.append({"project": project, "kind": kind, "id": ident, "at": _epoch(at), "text": text,
|
|
58
|
+
"outcome": outcome, "source": source})
|
|
59
|
+
|
|
60
|
+
for rec in decisions(root):
|
|
61
|
+
add("question", rec.get("id"), rec.get("answered_at") or rec.get("asked_at"),
|
|
62
|
+
" ".join(str(rec.get(key) or "") for key in ("question", "context", "answer")),
|
|
63
|
+
f"answered: {rec['answer']}" if rec.get("answer") else rec.get("state") or "open",
|
|
64
|
+
f"{DECISION_DIR}/{rec.get('id')}.json")
|
|
65
|
+
readers = ((decision_rows, "decision", ("decision", "why", "scope"), ".polyctl/ledger/decisions.jsonl"),
|
|
66
|
+
(waiver_rows, "waiver", ("gate", "slice", "why"), ".polyctl/ledger/waivers.jsonl"))
|
|
67
|
+
for reader, kind, fields, source in readers:
|
|
68
|
+
try:
|
|
69
|
+
rows = reader(root)
|
|
70
|
+
except Exception:
|
|
71
|
+
continue
|
|
72
|
+
for number, row in enumerate(rows, 1):
|
|
73
|
+
if isinstance(row, dict) and row.get("kind") == "proposal":
|
|
74
|
+
# A rule proposal is a row of the decision ledger, and no architect's decision (RULE-PROPOSALS);
|
|
75
|
+
# what a person answered is its question's, recalled above.
|
|
76
|
+
author = row.get("by") if isinstance(row.get("by"), dict) else {}
|
|
77
|
+
add("proposal", row.get("id"), row.get("at"),
|
|
78
|
+
" ".join(str(row.get(key) or "") for key in ("text", "why", "rule_file")),
|
|
79
|
+
f"proposed by {author.get('role') or 'a caller polyctl did not start'}", f"{source}:{number}")
|
|
80
|
+
elif isinstance(row, dict):
|
|
81
|
+
add(kind, row.get("id"), row.get("at"), " ".join(str(row.get(key) or "") for key in fields),
|
|
82
|
+
row.get("event") or ("recorded by " + str(row.get("by") or "architect")), f"{source}:{number}")
|
|
83
|
+
reviews_dir = "semantic-review"
|
|
84
|
+
try:
|
|
85
|
+
with open(os.path.join(root, ".polyctl", "config.json"), encoding=UTF8) as fh:
|
|
86
|
+
reviews_dir = json.load(fh).get("reviews") or reviews_dir
|
|
87
|
+
except (OSError, ValueError, AttributeError):
|
|
88
|
+
pass
|
|
89
|
+
directory = os.path.join(root, reviews_dir)
|
|
90
|
+
for name in sorted(os.listdir(directory)) if os.path.isdir(directory) else []:
|
|
91
|
+
path = os.path.join(directory, name)
|
|
92
|
+
if name.startswith(".") or not os.path.isfile(path):
|
|
93
|
+
continue
|
|
94
|
+
try:
|
|
95
|
+
with open(path, encoding=UTF8, errors="replace") as fh:
|
|
96
|
+
body = fh.read()
|
|
97
|
+
except OSError:
|
|
98
|
+
continue
|
|
99
|
+
verdict = _review_verdict(body)
|
|
100
|
+
for number, line in enumerate(body.splitlines(), 1):
|
|
101
|
+
found = _FINDING_LINE.match(line)
|
|
102
|
+
if found:
|
|
103
|
+
add("finding", name, os.path.getmtime(path), found.group(2), f"{found.group(1)} in a {verdict} review",
|
|
104
|
+
f"{reviews_dir}/{name}:{number}")
|
|
105
|
+
try:
|
|
106
|
+
with open(os.path.join(root, "docs", "lessons.md"), encoding=UTF8, errors="replace") as fh:
|
|
107
|
+
lessons = fh.read().splitlines()
|
|
108
|
+
except OSError:
|
|
109
|
+
lessons = []
|
|
110
|
+
starts = [number for number, line in enumerate(lessons) if line.startswith("## ")]
|
|
111
|
+
for index, start in enumerate(starts):
|
|
112
|
+
end = starts[index + 1] if index + 1 < len(starts) else len(lessons)
|
|
113
|
+
add("lesson", lessons[start][3:].strip(), None, "\n".join(lessons[start:end]), "a lesson",
|
|
114
|
+
f"docs/lessons.md:{start + 1}")
|
|
115
|
+
return entries
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def recall(text, root=None, limit=10, exclude=(), share=0.5):
|
|
119
|
+
"""Records in every registered project that share the words of `text`, best first (#43).
|
|
120
|
+
|
|
121
|
+
Plain words over the files polyctl already keeps: no index, no service, no network.
|
|
122
|
+
One or two words must all appear; longer text needs `share` of its words.
|
|
123
|
+
"""
|
|
124
|
+
words = _recall_words(text)
|
|
125
|
+
if not words:
|
|
126
|
+
return []
|
|
127
|
+
need = len(words) if len(words) <= 2 else max(2, int(len(words) * share + 0.999))
|
|
128
|
+
found = []
|
|
129
|
+
for project, path in recall_roots(root):
|
|
130
|
+
try:
|
|
131
|
+
entries = recall_entries(project, path)
|
|
132
|
+
except Exception:
|
|
133
|
+
continue
|
|
134
|
+
for entry in entries:
|
|
135
|
+
if entry["id"] in exclude:
|
|
136
|
+
continue
|
|
137
|
+
score = len(words & _recall_words(entry["text"]))
|
|
138
|
+
if score >= need:
|
|
139
|
+
found.append(dict(entry, score=score))
|
|
140
|
+
found.sort(key=lambda entry: (-entry["score"], -(entry["at"] or 0)))
|
|
141
|
+
return found[:limit]
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def architect_absence(root, cfg):
|
|
145
|
+
"""When the architect was last seen, and the questions waiting for its return (#84).
|
|
146
|
+
|
|
147
|
+
Seen means its session wrote, it recorded a decision, or it left mail still in
|
|
148
|
+
the mailbox, whichever is newest.
|
|
149
|
+
"""
|
|
150
|
+
seen = []
|
|
151
|
+
try:
|
|
152
|
+
# Its own session's last write. The newest transcript where the architect works is as likely
|
|
153
|
+
# the implementer's, when both run one harness there (SESSION-IDENTITY).
|
|
154
|
+
state = session_state(cfg, "architect")
|
|
155
|
+
if state is not None:
|
|
156
|
+
transcript = role_session_paths(dict(cfg, root=root), "architect")[0] if state["session"] else None
|
|
157
|
+
if transcript and os.path.exists(transcript):
|
|
158
|
+
seen.append(os.path.getmtime(transcript))
|
|
159
|
+
else:
|
|
160
|
+
found = discover_architect((cfg.get("architect") or {}).get("cwd") or root)
|
|
161
|
+
if found and found.get("age") is not None \
|
|
162
|
+
and found.get("session") != (session_state(cfg, "implementer") or {}).get("session"):
|
|
163
|
+
seen.append(time.time() - float(found["age"]))
|
|
164
|
+
except Exception:
|
|
165
|
+
pass
|
|
166
|
+
try:
|
|
167
|
+
# Its decisions: a rule proposal in the same ledger is no decision, whoever made it (RULE-PROPOSALS).
|
|
168
|
+
rows = [row for row in decision_rows(root) if row.get("kind") != "proposal"]
|
|
169
|
+
if rows and _epoch(rows[-1].get("at")):
|
|
170
|
+
seen.append(_epoch(rows[-1].get("at")))
|
|
171
|
+
except Exception:
|
|
172
|
+
pass
|
|
173
|
+
mailbox_dir = cfg.get("mailbox", "agent-mail")
|
|
174
|
+
for name in mailbox(root, mailbox_dir):
|
|
175
|
+
if from_architect(name, cfg):
|
|
176
|
+
try:
|
|
177
|
+
seen.append(os.path.getmtime(os.path.join(root, mailbox_dir, name)))
|
|
178
|
+
except OSError:
|
|
179
|
+
pass
|
|
180
|
+
waiting = sorted(decisions(root, "open"), key=lambda d: d.get("asked_at") or 0)
|
|
181
|
+
return {"seen_at": max(seen) if seen else None, "waiting": [d["id"] for d in waiting],
|
|
182
|
+
"oldest_at": waiting[0].get("asked_at") if waiting else None}
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
# ---- a worktree lives exactly as long as its slice (#42) --------------------------------
|
|
186
|
+
|
|
187
|
+
def default_branch(root):
|
|
188
|
+
"""The branch work lands on: origin's HEAD, else main, else master, else the main checkout's."""
|
|
189
|
+
remote = git_text(root, "symbolic-ref", "--quiet", "refs/remotes/origin/HEAD")
|
|
190
|
+
if remote.startswith("refs/remotes/origin/"):
|
|
191
|
+
return remote[len("refs/remotes/origin/"):]
|
|
192
|
+
for name in ("main", "master"):
|
|
193
|
+
if git_text(root, "rev-parse", "--verify", "--quiet", f"refs/heads/{name}"):
|
|
194
|
+
return name
|
|
195
|
+
trees = worktree_list(root)
|
|
196
|
+
return trees[0]["branch"] if trees else None
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def worktree_list(root):
|
|
200
|
+
"""Every worktree of the repository: {"path", "head", "branch", "prunable"}, the main one first."""
|
|
201
|
+
listed = _git_output(root, "worktree", "list", "--porcelain", "-z")
|
|
202
|
+
out = []
|
|
203
|
+
for record in listed.split(b"\0\0"):
|
|
204
|
+
fields = {}
|
|
205
|
+
for attribute in record.split(b"\0"):
|
|
206
|
+
key, _, value = os.fsdecode(attribute).partition(" ")
|
|
207
|
+
if key:
|
|
208
|
+
fields[key] = value
|
|
209
|
+
if "worktree" in fields:
|
|
210
|
+
branch = fields.get("branch", "")
|
|
211
|
+
out.append({"path": fields["worktree"], "head": fields.get("HEAD"),
|
|
212
|
+
"branch": branch[len("refs/heads/"):] if branch.startswith("refs/heads/") else None,
|
|
213
|
+
"prunable": "prunable" in fields})
|
|
214
|
+
return out
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def reviews_in_flight(root):
|
|
218
|
+
"""Submitted reviews in one checkout whose runner is alive or has only just started."""
|
|
219
|
+
directory = os.path.join(root, ".polyctl", "reviews")
|
|
220
|
+
try:
|
|
221
|
+
names = sorted(os.listdir(directory))
|
|
222
|
+
except OSError:
|
|
223
|
+
return []
|
|
224
|
+
flying = []
|
|
225
|
+
for name in names:
|
|
226
|
+
if not (name.startswith("R-") and name.endswith(".json")):
|
|
227
|
+
continue
|
|
228
|
+
try:
|
|
229
|
+
with open(os.path.join(directory, name), encoding=UTF8) as fh:
|
|
230
|
+
state = json.load(fh)
|
|
231
|
+
except (OSError, ValueError):
|
|
232
|
+
continue
|
|
233
|
+
if isinstance(state, dict) and state.get("state") == "running" and (
|
|
234
|
+
_pid_alive(state.get("pid")) if state.get("pid")
|
|
235
|
+
else time.time() - float(state.get("submitted_at") or 0) < 60):
|
|
236
|
+
flying.append(state.get("id") or name[:-5])
|
|
237
|
+
return flying
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _tree_bytes(path):
|
|
241
|
+
total = 0
|
|
242
|
+
for directory, subdirs, files in os.walk(path):
|
|
243
|
+
subdirs[:] = [d for d in subdirs if not os.path.islink(os.path.join(directory, d))]
|
|
244
|
+
for name in files:
|
|
245
|
+
try:
|
|
246
|
+
total += os.lstat(os.path.join(directory, name)).st_size
|
|
247
|
+
except OSError:
|
|
248
|
+
pass
|
|
249
|
+
return total
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def worktree_facts(root, cfg, sizes=False):
|
|
253
|
+
"""What decides whether each worktree may go (#42).
|
|
254
|
+
|
|
255
|
+
A worktree may go when its branch is merged into the default branch, when the
|
|
256
|
+
board rejected the slice that owns it (`worktree:` or `branch:` on the item),
|
|
257
|
+
or when its directory is already gone - and never while it holds product
|
|
258
|
+
changes nobody committed, a review in flight, or the command that is asking,
|
|
259
|
+
and never when it is a lane, which `polyctl lane remove` retires (LANE-START).
|
|
260
|
+
"""
|
|
261
|
+
target = default_branch(root)
|
|
262
|
+
board_items = board(root)
|
|
263
|
+
owners = []
|
|
264
|
+
for state, items in board_items.items():
|
|
265
|
+
for item in items:
|
|
266
|
+
owners.append((state, item))
|
|
267
|
+
here = os.path.realpath(root)
|
|
268
|
+
trees = worktree_list(root)
|
|
269
|
+
lanes = lane_keeps(root, trees)
|
|
270
|
+
out = []
|
|
271
|
+
for index, tree in enumerate(trees):
|
|
272
|
+
path, branch = tree["path"], tree["branch"]
|
|
273
|
+
real = os.path.realpath(path)
|
|
274
|
+
slice_state = next((state for state, item in owners
|
|
275
|
+
if (item["notes"].get("worktree") and os.path.realpath(item["notes"]["worktree"]) == real)
|
|
276
|
+
or (branch and item["notes"].get("branch") == branch)), None)
|
|
277
|
+
targets = [target, "origin/" + target] if target else []
|
|
278
|
+
merged_into = next((ref for ref in targets if branch and branch != target and subprocess.run(
|
|
279
|
+
[git_binary(), "merge-base", "--is-ancestor", tree["head"], ref], cwd=root,
|
|
280
|
+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0), None)
|
|
281
|
+
merged = merged_into is not None
|
|
282
|
+
exists = os.path.isdir(path)
|
|
283
|
+
keep = []
|
|
284
|
+
try:
|
|
285
|
+
dirty = product_dirty(path, cfg, strict=True) if exists else []
|
|
286
|
+
except RuntimeError:
|
|
287
|
+
dirty = []
|
|
288
|
+
keep.append("status unreadable; inspect the worktree and retry pruning")
|
|
289
|
+
flying = reviews_in_flight(path) if exists else []
|
|
290
|
+
if index == 0:
|
|
291
|
+
keep.append("the main checkout")
|
|
292
|
+
if real == here:
|
|
293
|
+
keep.append("the checkout asking")
|
|
294
|
+
if dirty:
|
|
295
|
+
keep.append(f"{len(dirty)} uncommitted product change(s)")
|
|
296
|
+
if flying:
|
|
297
|
+
keep.append(f"review in flight: {', '.join(flying)}")
|
|
298
|
+
if os.path.normcase(real) in lanes:
|
|
299
|
+
keep.append(lanes[os.path.normcase(real)])
|
|
300
|
+
why = "merged into " + merged_into if merged else "its slice was rejected" if slice_state == "rejected" \
|
|
301
|
+
else "its directory is gone" if not exists or tree["prunable"] else None
|
|
302
|
+
may_go = bool(why) and not keep
|
|
303
|
+
# Sizes are for what may go: walking every dependency tree on disk is not free.
|
|
304
|
+
out.append(dict(tree, merged=merged, slice_state=slice_state, dirty=len(dirty), in_flight=flying,
|
|
305
|
+
keep=keep, why=why, may_go=may_go,
|
|
306
|
+
bytes=_tree_bytes(path) if sizes and may_go and exists else None))
|
|
307
|
+
return out
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
class WorktreeCleanupRefused(RuntimeError):
|
|
311
|
+
"""Unsafe administration refuses cleanup; an ordinary Git failure remains distinct (PRUNE-LINKS)."""
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _check_worktree_cleanup(root):
|
|
315
|
+
"""Refuse linked or unreadable worktree administration before cleanup (PRUNE-LINKS)."""
|
|
316
|
+
import stat
|
|
317
|
+
try:
|
|
318
|
+
common = _git_output(root, "rev-parse", "--git-common-dir").decode(UTF8, "surrogateescape").strip()
|
|
319
|
+
except RuntimeError as exc:
|
|
320
|
+
raise WorktreeCleanupRefused(f"cannot locate the common git directory: {exc}; "
|
|
321
|
+
"repair it before polyctl prunes any worktree") from exc
|
|
322
|
+
if not common:
|
|
323
|
+
raise WorktreeCleanupRefused("cannot locate the common git directory; repair it before polyctl prunes any worktree")
|
|
324
|
+
common = os.path.join(root, common)
|
|
325
|
+
worktrees = os.path.join(common, "worktrees")
|
|
326
|
+
path = worktrees
|
|
327
|
+
try:
|
|
328
|
+
# Let the filesystem fold case as Git does; a name scan missed Worktrees links (PRUNE-LINKS).
|
|
329
|
+
try:
|
|
330
|
+
info = os.lstat(worktrees)
|
|
331
|
+
except FileNotFoundError:
|
|
332
|
+
info = None
|
|
333
|
+
if info is not None:
|
|
334
|
+
if stat.S_ISLNK(info.st_mode) or getattr(info, "st_file_attributes", 0) & 0x400:
|
|
335
|
+
raise WorktreeCleanupRefused(f"remove the link or junction at {path}; "
|
|
336
|
+
"polyctl prunes no worktree while it is there")
|
|
337
|
+
if not stat.S_ISDIR(info.st_mode):
|
|
338
|
+
raise WorktreeCleanupRefused(f"remove the non-directory entry at {path}; "
|
|
339
|
+
"polyctl prunes no worktree while it is there")
|
|
340
|
+
with os.scandir(worktrees) as entries:
|
|
341
|
+
for entry in entries:
|
|
342
|
+
path = entry.path
|
|
343
|
+
info = entry.stat(follow_symlinks=False)
|
|
344
|
+
if stat.S_ISLNK(info.st_mode) or getattr(info, "st_file_attributes", 0) & 0x400:
|
|
345
|
+
raise WorktreeCleanupRefused(f"remove the link or junction at {path}; "
|
|
346
|
+
"polyctl prunes no worktree while it is there")
|
|
347
|
+
except OSError as exc:
|
|
348
|
+
raise WorktreeCleanupRefused(f"cannot read {path}: {exc}; restore access or remove the unreadable entry; "
|
|
349
|
+
"polyctl prunes no worktree while it cannot be read") from exc
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _git_worktree_cleanup(root, *args):
|
|
353
|
+
"""Git follows linked worktree administration while deleting; refuse it before every cleanup (PRUNE-LINKS)."""
|
|
354
|
+
_check_worktree_cleanup(root)
|
|
355
|
+
return _git_output(root, "worktree", *args)
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def prune_worktree(root, fact, apply=False, now=None, lane=None):
|
|
359
|
+
"""Retire one worktree that may go: archive its coordination state and branch tip, then remove both (#42).
|
|
360
|
+
|
|
361
|
+
The `.polyctl/` state and review artefacts go to ~/.polyctl/archive/<project>/, and the
|
|
362
|
+
branch tip stays reachable as refs/polyctl/archive/<branch>-<stamp>, so what the
|
|
363
|
+
worktree held can still be read after it is gone.
|
|
364
|
+
"""
|
|
365
|
+
import shutil
|
|
366
|
+
|
|
367
|
+
def changes():
|
|
368
|
+
# A lane's prepared files stay exempt at both rechecks (TRIAGE-3-MEDIUM).
|
|
369
|
+
return lane_changes(fact["path"], lane) if lane is not None else \
|
|
370
|
+
product_dirty(fact["path"], load_config(root), strict=True)
|
|
371
|
+
|
|
372
|
+
# Refuse unsafe administration before a status failure can mask it (PRUNE-LINKS).
|
|
373
|
+
if apply:
|
|
374
|
+
_check_worktree_cleanup(root)
|
|
375
|
+
|
|
376
|
+
# Facts shown earlier are no permission to discard changes made since (TRIAGE-3-MEDIUM).
|
|
377
|
+
if apply and os.path.isdir(fact["path"]):
|
|
378
|
+
if changes():
|
|
379
|
+
raise RuntimeError("uncommitted product changes appeared; keep the worktree and commit or save them first")
|
|
380
|
+
stamp = datetime.fromtimestamp(time.time() if now is None else now).strftime("%Y%m%d-%H%M%S")
|
|
381
|
+
name = os.path.basename(os.path.normpath(fact["path"]))
|
|
382
|
+
archive = os.path.join(HOME, ".polyctl", "archive", project_key(root), f"worktree-{name}-{stamp}")
|
|
383
|
+
steps = []
|
|
384
|
+
for part in (".polyctl", "semantic-review"):
|
|
385
|
+
source = os.path.join(fact["path"], part)
|
|
386
|
+
if os.path.isdir(source):
|
|
387
|
+
steps.append(f"archive {part}/ to {archive}")
|
|
388
|
+
if apply:
|
|
389
|
+
shutil.copytree(source, os.path.join(archive, part), symlinks=True)
|
|
390
|
+
if fact["branch"]:
|
|
391
|
+
steps.append(f"keep {fact['branch']} at refs/polyctl/archive/{fact['branch']}-{stamp}")
|
|
392
|
+
if apply:
|
|
393
|
+
_git_output(root, "update-ref", f"refs/polyctl/archive/{fact['branch']}-{stamp}", fact["head"])
|
|
394
|
+
steps.append(f"remove the worktree {fact['path']}")
|
|
395
|
+
if apply and os.path.isdir(fact["path"]):
|
|
396
|
+
if changes():
|
|
397
|
+
raise RuntimeError("uncommitted product changes appeared while archiving; keep the worktree and save them")
|
|
398
|
+
_git_worktree_cleanup(root, "remove", "--force", fact["path"])
|
|
399
|
+
# Before the branch: git will not delete a branch a worktree it lists holds, and a worktree whose
|
|
400
|
+
# directory someone deleted stays listed until a prune, so its branch could never go (LANE-START).
|
|
401
|
+
steps.append("git worktree prune")
|
|
402
|
+
if apply:
|
|
403
|
+
_git_worktree_cleanup(root, "prune")
|
|
404
|
+
if fact["branch"]:
|
|
405
|
+
steps.append(f"delete the branch {fact['branch']}")
|
|
406
|
+
if apply:
|
|
407
|
+
_git_output(root, "branch", "-D", fact["branch"])
|
|
408
|
+
return steps
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
# ---- the secondary project: the same agent, another queue (#8, #22, #92) ----------------
|
|
412
|
+
|
|
413
|
+
HUMAN_WAITING = language.words("human")
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def secondary_projects(cfg):
|
|
417
|
+
"""The projects this one names as secondary (`"secondary": [{"root", "name"}]`), each with its config (#8)."""
|
|
418
|
+
out = []
|
|
419
|
+
for entry in (cfg or {}).get("secondary") or []:
|
|
420
|
+
other_root = entry.get("root") if isinstance(entry, dict) else entry
|
|
421
|
+
if not isinstance(other_root, str) or not os.path.isdir(os.path.join(other_root, ".polyctl")):
|
|
422
|
+
continue
|
|
423
|
+
try:
|
|
424
|
+
other = load_config(other_root, record=False)
|
|
425
|
+
except Exception:
|
|
426
|
+
continue
|
|
427
|
+
name = (entry.get("name") if isinstance(entry, dict) else None) or next_project_key(other_root)
|
|
428
|
+
out.append({"name": name, "root": other_root, "cfg": other})
|
|
429
|
+
return out
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def working_elsewhere(cfg, idle_seconds):
|
|
433
|
+
"""The secondary project the implementer is writing in now, if it is (#22).
|
|
434
|
+
|
|
435
|
+
While the implementer worked in a polyctl worktree, the watchdog of the project
|
|
436
|
+
it had left read the same agent as idle with a slice running and nudged it
|
|
437
|
+
back. Presence belongs to the agent: a transcript moving in a secondary
|
|
438
|
+
project is the agent working.
|
|
439
|
+
"""
|
|
440
|
+
newest, now = None, time.time()
|
|
441
|
+
for other in secondary_projects(cfg):
|
|
442
|
+
try:
|
|
443
|
+
transcript, _ = session_paths(other["cfg"])
|
|
444
|
+
age = now - last_write(transcript, transcript_shape(implementer_adapter(other["cfg"])))
|
|
445
|
+
except (OSError, TypeError, ValueError):
|
|
446
|
+
continue
|
|
447
|
+
# Configuration order said where the agent was, even with newer activity elsewhere (TRIAGE-2-LOW).
|
|
448
|
+
if age < idle_seconds and (newest is None or age < newest["age"]):
|
|
449
|
+
newest = {"name": other["name"], "root": other["root"], "age": age}
|
|
450
|
+
return newest
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def secondary_ready(cfg):
|
|
454
|
+
"""The first READY item of a secondary project, for an implementer with nothing READY here (#8)."""
|
|
455
|
+
for other in secondary_projects(cfg):
|
|
456
|
+
try:
|
|
457
|
+
items = ready(other["root"])
|
|
458
|
+
except Exception:
|
|
459
|
+
continue
|
|
460
|
+
if items:
|
|
461
|
+
return {"name": other["name"], "root": other["root"], "item": items[0]["id"]}
|
|
462
|
+
return None
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def human_waits(root):
|
|
466
|
+
"""Blocked board items the implementer declared as waiting on a person: `waiting: human` (#92)."""
|
|
467
|
+
return [item for item in board(root)["blocked"]
|
|
468
|
+
if (item["notes"].get("waiting") or "").strip().lower() in HUMAN_WAITING]
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
# ---- the process is measured by its outcomes (#49) ---------------------------------------
|
|
472
|
+
|
|
473
|
+
def _since_epoch(text):
|
|
474
|
+
try:
|
|
475
|
+
return datetime.strptime(str(text).strip()[:16], "%Y-%m-%d %H:%M").timestamp()
|
|
476
|
+
except ValueError:
|
|
477
|
+
return None
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def slice_outcomes(root, project=None):
|
|
481
|
+
"""What happened to every slice that landed, from what polyctl already recorded (#49).
|
|
482
|
+
|
|
483
|
+
On 2026-09-07 the round budget, the size guideline and the review pipeline
|
|
484
|
+
were all re-decided in one day on anecdote. Each outcome is read, never
|
|
485
|
+
typed: the verdicts in the order polyctl recorded them (a re-specification keeps
|
|
486
|
+
the history, it does not start a new one), rounds, ready-to-landed time, size
|
|
487
|
+
by kind, and whether a defect was later found in what landed - a
|
|
488
|
+
retrospective review that asked for changes, or a board item `fixes:` it.
|
|
489
|
+
"""
|
|
490
|
+
from .storage import read_chained_history
|
|
491
|
+
project = project or project_key(root)
|
|
492
|
+
reviews = [row for row in read_chained_history(review_ledger_path(root), REVIEW_CHAIN) if isinstance(row, dict)]
|
|
493
|
+
# Include grants retired into sealed history (LEDGER-HISTORY).
|
|
494
|
+
grants = [row for row in authority_history(root) if isinstance(row, dict) and row.get("granted") is True]
|
|
495
|
+
# Permission to commit was counted as a landing even when abandoned (TRIAGE-2-MEDIUM).
|
|
496
|
+
landed = {}
|
|
497
|
+
if grants:
|
|
498
|
+
for commit in landed_commits(root, "HEAD"):
|
|
499
|
+
tokens = _polyctl_trailers(commit["trailers"]).get("Polyctl-Grant", [])
|
|
500
|
+
if len(tokens) == 1:
|
|
501
|
+
landed.setdefault(tokens[0], commit["at"])
|
|
502
|
+
verifications = [row for row in verification_history(root) if isinstance(row, dict)]
|
|
503
|
+
waivers = {row.get("id"): row for row in read_chained_history(waivers_path(root), WAIVER_CHAIN, legacy_prefix=True)
|
|
504
|
+
if isinstance(row, dict) and row.get("event") == "waived"}
|
|
505
|
+
by_artefact = {row.get("artefact"): row for row in reviews}
|
|
506
|
+
items = {item["id"]: item for state_items in board(root).values() for item in state_items}
|
|
507
|
+
fixed = {item["notes"]["fixes"].strip() for item in items.values() if item["notes"].get("fixes")}
|
|
508
|
+
out = {}
|
|
509
|
+
for grant in grants:
|
|
510
|
+
if grant.get("token") not in landed:
|
|
511
|
+
continue
|
|
512
|
+
linked = by_artefact.get(grant.get("review")) or {}
|
|
513
|
+
slice_id = linked.get("slice") or (waivers.get(grant.get("waiver") or grant.get("gates_waiver")) or {}).get("slice")
|
|
514
|
+
if not slice_id or slice_id in out:
|
|
515
|
+
continue
|
|
516
|
+
mine = [row for row in reviews if row.get("slice") == slice_id]
|
|
517
|
+
prospective = [row for row in mine if row.get("kind") == "index-candidate"
|
|
518
|
+
and row.get("verdict") in ("APPROVED", "NEEDS_CHANGES")]
|
|
519
|
+
candidate = (grant.get("candidate") or {}).get("digest")
|
|
520
|
+
size = next((row.get("candidate_size") for row in reversed(verifications)
|
|
521
|
+
if (row.get("candidate") or {}).get("digest") == candidate and row.get("candidate_size")), None)
|
|
522
|
+
started = _since_epoch((items.get(slice_id) or {}).get("notes", {}).get("since", "")) \
|
|
523
|
+
or (min(float(row.get("at") or 0) for row in mine) if mine else None)
|
|
524
|
+
landed_at = landed[grant["token"]]
|
|
525
|
+
retro = [row for row in mine if row.get("kind") == "commit-range" and row.get("verdict") == "NEEDS_CHANGES"]
|
|
526
|
+
out[slice_id] = {
|
|
527
|
+
"project": project, "slice": slice_id, "verdicts": [row.get("verdict") for row in prospective],
|
|
528
|
+
"rounds": len(prospective), "first_pass": bool(prospective) and prospective[0].get("verdict") == "APPROVED",
|
|
529
|
+
"waived": not prospective and bool(grant.get("waiver"))
|
|
530
|
+
and (waivers.get(grant.get("waiver")) or {}).get("gate", "review") == "review",
|
|
531
|
+
"started_at": started, "landed_at": landed_at,
|
|
532
|
+
"hours": round((landed_at - started) / 3600, 2) if started and landed_at >= started else None,
|
|
533
|
+
"size": size, "defect_found": bool(retro) or slice_id in fixed,
|
|
534
|
+
# The tier of the review the grant stood on, so weaker independence is counted (REVIEW-TIERS).
|
|
535
|
+
"tier": linked.get("tier"),
|
|
536
|
+
}
|
|
537
|
+
return list(out.values())
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def outcome_stats(outcomes):
|
|
541
|
+
"""The distribution a process change is judged against: rounds, first-pass rate, time, size, defects."""
|
|
542
|
+
def spread(values):
|
|
543
|
+
values = sorted(v for v in values if v is not None)
|
|
544
|
+
if not values:
|
|
545
|
+
return None
|
|
546
|
+
from statistics import median
|
|
547
|
+
from math import ceil
|
|
548
|
+
middle = median(values)
|
|
549
|
+
if isinstance(middle, float) and middle.is_integer():
|
|
550
|
+
middle = int(middle)
|
|
551
|
+
return {"median": middle, "p90": values[ceil(len(values) * 0.9) - 1],
|
|
552
|
+
"n": len(values)}
|
|
553
|
+
reviewed = [o for o in outcomes if not o["waived"]]
|
|
554
|
+
return {
|
|
555
|
+
"slices": len(outcomes), "waived": len(outcomes) - len(reviewed),
|
|
556
|
+
"rounds": spread(o["rounds"] for o in reviewed),
|
|
557
|
+
"first_pass_pct": round(100 * sum(o["first_pass"] for o in reviewed) / len(reviewed)) if reviewed else None,
|
|
558
|
+
"hours": spread(o["hours"] for o in outcomes),
|
|
559
|
+
"product_lines": spread((o["size"]["kinds"]["product"]["added"] + o["size"]["kinds"]["product"]["deleted"])
|
|
560
|
+
if isinstance(o.get("size"), dict) else None for o in outcomes),
|
|
561
|
+
"defects_pct": round(100 * sum(o["defect_found"] for o in outcomes) / len(outcomes)) if outcomes else None,
|
|
562
|
+
"tiers": {tier: sum(1 for o in outcomes if o.get("tier") == tier)
|
|
563
|
+
for tier in sorted({o.get("tier") for o in outcomes if o.get("tier")})},
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
# ---- every store is bounded, without being asked (#50) -----------------------------------
|
|
568
|
+
|
|
569
|
+
OBSERVATION_LOGS = ("nudge-log", "watchdog-log", "refill-log", "escalate-log", "cycles") # PROJECT_FILES
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
def bound_store(path, kb, in_place=False):
|
|
573
|
+
"""Keep an observation store within its bound as it is written: past it, the oldest records go (#50).
|
|
574
|
+
|
|
575
|
+
Measured 2026-09-08: notices 3.2 MB, a nudge log 2.6 MB, and `polyctl prune`, which
|
|
576
|
+
existed and defaulted to a dry run, had never been run - the only outcome a
|
|
577
|
+
manual cleanup has. A store is trimmed once it is a quarter over its bound,
|
|
578
|
+
back to three quarters of it, at a line boundary, so it is not rewritten on
|
|
579
|
+
every write. Evidence is never trimmed here; it is sealed.
|
|
580
|
+
|
|
581
|
+
A log a started process writes is trimmed `in_place`, in the file it holds open: replaced
|
|
582
|
+
by a new file, the process wrote on into the old one, and what a wake that failed said
|
|
583
|
+
never reached the log its failure is read from (JOURNAL-3).
|
|
584
|
+
"""
|
|
585
|
+
limit = int(kb) * 1024
|
|
586
|
+
try:
|
|
587
|
+
size = os.path.getsize(path)
|
|
588
|
+
except OSError:
|
|
589
|
+
return False
|
|
590
|
+
if size <= limit * 1.25:
|
|
591
|
+
return False
|
|
592
|
+
if in_place:
|
|
593
|
+
return _trim_in_place(path, int(limit * 0.75))
|
|
594
|
+
from .storage import replace_file_durably
|
|
595
|
+
with open(path, "rb") as fh:
|
|
596
|
+
fh.seek(size - int(limit * 0.75))
|
|
597
|
+
tail = fh.read()
|
|
598
|
+
replace_file_durably(path, tail[tail.find(b"\n") + 1:])
|
|
599
|
+
return True
|
|
600
|
+
|
|
601
|
+
|
|
602
|
+
def _trim_in_place(path, keep):
|
|
603
|
+
"""Keep the last `keep` bytes of `path`, from a line boundary, in the same file (JOURNAL-3).
|
|
604
|
+
|
|
605
|
+
A writer appends to the file's end, so what it writes after the trim follows the kept tail;
|
|
606
|
+
what it appended while the tail was read is read on from where that read ended (JOURNAL-3-2).
|
|
607
|
+
No portable cut asks whether the end moved, so a line appended between that last read and the
|
|
608
|
+
cut is lost, and a writer whose handle does not append - a child's on Windows (CPython
|
|
609
|
+
gh-86772) - writes on past the new end: the watchdog leaves a log a process it started still
|
|
610
|
+
writes whole until that process ends.
|
|
611
|
+
"""
|
|
612
|
+
try:
|
|
613
|
+
with open(path, "r+b") as fh:
|
|
614
|
+
size = fh.seek(0, os.SEEK_END)
|
|
615
|
+
fh.seek(max(0, size - keep))
|
|
616
|
+
tail = fh.read()
|
|
617
|
+
tail = tail[tail.find(b"\n") + 1:]
|
|
618
|
+
tail += fh.read() # appended while the tail was read, read once (JOURNAL-3-2)
|
|
619
|
+
fh.seek(0)
|
|
620
|
+
fh.write(tail)
|
|
621
|
+
fh.truncate()
|
|
622
|
+
fh.flush()
|
|
623
|
+
os.fsync(fh.fileno())
|
|
624
|
+
except OSError:
|
|
625
|
+
return False
|
|
626
|
+
return True
|
|
627
|
+
|
|
628
|
+
|
|
629
|
+
def observation_stores(root, state_dir=None):
|
|
630
|
+
"""Every observation store of a project: its ledgers' and the watchdog's logs, as paths."""
|
|
631
|
+
key = project_key(root)
|
|
632
|
+
state_dir = state_dir or os.path.join(HOME, ".polyctl")
|
|
633
|
+
stores = [os.path.join(root, ".polyctl", "ledger", name) for name in ("notices.jsonl", "progress.jsonl")]
|
|
634
|
+
return stores + [os.path.join(state_dir, project_file_name(what, key)) for what in OBSERVATION_LOGS]
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
def bound_observation_logs(root, state_dir=None, busy=()):
|
|
638
|
+
"""Hold every observation store to its bound; the watchdog does this each cycle (#50).
|
|
639
|
+
|
|
640
|
+
The notices are folded first: what the bound trims still counts in a window (NOTICE-WINDOW). A
|
|
641
|
+
fold that could not be written leaves them whole, past their bound, where `polyctl doctor` names
|
|
642
|
+
them (NOTICE-WINDOW-2).
|
|
643
|
+
|
|
644
|
+
A log in `busy` is one a process the watchdog started may still write, and is left whole until
|
|
645
|
+
that process ends; on Windows a log is replaced, never cut under a handle that does not append,
|
|
646
|
+
and one another process holds stays over its bound this cycle while the rest are bounded
|
|
647
|
+
(JOURNAL-3-2).
|
|
648
|
+
"""
|
|
649
|
+
limit = settings.get(load_config(root), "retention.observation_kb")
|
|
650
|
+
notices = os.path.join(root, ".polyctl", "ledger", "notices.jsonl") if not fold_notice_times(root) else None
|
|
651
|
+
bounded = []
|
|
652
|
+
for path in observation_stores(root, state_dir):
|
|
653
|
+
if path == notices or path in busy:
|
|
654
|
+
continue
|
|
655
|
+
try:
|
|
656
|
+
# Elsewhere a log a started process may still write is trimmed in the file it holds (JOURNAL-3).
|
|
657
|
+
if bound_store(path, limit, in_place=path.endswith(".log") and os.name != "nt"):
|
|
658
|
+
bounded.append(path)
|
|
659
|
+
except OSError:
|
|
660
|
+
continue
|
|
661
|
+
return bounded
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def stores_over_bound(root, cfg, state_dir=None):
|
|
665
|
+
"""(path, amount, bound, unit) for observations and sealable evidence past their bounds (TRIAGE-1-LOW)."""
|
|
666
|
+
limit = settings.get(cfg, "retention.observation_kb") * 1024
|
|
667
|
+
out = []
|
|
668
|
+
for path in observation_stores(root, state_dir):
|
|
669
|
+
try:
|
|
670
|
+
size = os.path.getsize(path)
|
|
671
|
+
except OSError:
|
|
672
|
+
continue
|
|
673
|
+
if size > limit * 1.25:
|
|
674
|
+
out.append((path, size, limit, "bytes"))
|
|
675
|
+
from .storage import read_chained_jsonl
|
|
676
|
+
keep = settings.get(cfg, "retention.evidence_keep")
|
|
677
|
+
for name, chain, legacy in (("authority", AUTHORITY_CHAIN, False),
|
|
678
|
+
("verifications", VERIFICATION_CHAIN, True)):
|
|
679
|
+
path = os.path.join(root, ".polyctl", "ledger", name + ".jsonl")
|
|
680
|
+
count = len(read_chained_jsonl(path, chain, legacy_prefix=legacy))
|
|
681
|
+
if count > keep:
|
|
682
|
+
out.append((path, count, keep, "rows"))
|
|
683
|
+
return out
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
# ---- a standing bug-hunter: read-only, bounded, leads not verdicts (#45) -----------------
|
|
687
|
+
|
|
688
|
+
HUNT_CATEGORIES = ("correctness", "concurrency", "clock", "durability", "subprocess", "portability", "secrets",
|
|
689
|
+
"authority", "tests")
|
|
690
|
+
_LEAD = re.compile(r"^\s*-\s*\[([a-z]+)\]\s*([^\s:]+):(\d+)\s+(\S+)\s+[—-]+\s*(.+?)\s*$")
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
def hunter_ledger_path(root):
|
|
694
|
+
return os.path.join(root, ".polyctl", "ledger", "hunter.jsonl")
|
|
695
|
+
|
|
696
|
+
|
|
697
|
+
def hunter_record(root, event, **fields):
|
|
698
|
+
from .storage import append_jsonl
|
|
699
|
+
append_jsonl(hunter_ledger_path(root), scan_record(dict(fields, at=int(time.time()), event=event)))
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
def hunter_rows(root):
|
|
703
|
+
from .storage import read_jsonl
|
|
704
|
+
return [row for row in read_jsonl(hunter_ledger_path(root)) if isinstance(row, dict)]
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
def hunter_known(root):
|
|
708
|
+
"""Fingerprints already sent or discarded: a repeat is suppressed, a discard remembered (#45)."""
|
|
709
|
+
return {row.get("fingerprint") for row in hunter_rows(root) if row.get("event") in ("sent", "discarded")}
|
|
710
|
+
|
|
711
|
+
|
|
712
|
+
def _hunt_cursor(state_path, candidates):
|
|
713
|
+
"""(index, line) the next hunt starts at: the file the last one stopped before, and the line of it its next
|
|
714
|
+
piece begins with. The file is kept by name: a file added or removed above it moves no other into its place,
|
|
715
|
+
and where it is gone the hunt goes on with the one that follows it (BUG-HUNTER-3)."""
|
|
716
|
+
try:
|
|
717
|
+
with open(state_path, encoding=UTF8) as fh:
|
|
718
|
+
state = json.load(fh)
|
|
719
|
+
index, path, line = int(state.get("cursor") or 0), state.get("path"), int(state.get("line") or 1)
|
|
720
|
+
except (OSError, ValueError, TypeError, AttributeError):
|
|
721
|
+
return 0, 1
|
|
722
|
+
if not isinstance(path, str):
|
|
723
|
+
return index % len(candidates), 1 # a cursor written before BUG-HUNTER-3: a place in the list
|
|
724
|
+
if path in candidates:
|
|
725
|
+
return candidates.index(path), max(line, 1)
|
|
726
|
+
return next((at for at, each in enumerate(candidates) if each > path), 0), 1
|
|
727
|
+
|
|
728
|
+
|
|
729
|
+
def _hunt_lines(full, largest):
|
|
730
|
+
"""The lines of one file as a hunt reads them, numbered as an editor numbers them: [] where it cannot be read
|
|
731
|
+
as text, None where it is larger than `largest` bytes and so no source a hunt goes through (BUG-HUNTER-3)."""
|
|
732
|
+
try:
|
|
733
|
+
if os.path.getsize(full) > largest:
|
|
734
|
+
return None
|
|
735
|
+
with open(full, encoding=UTF8) as fh:
|
|
736
|
+
return fh.readlines()
|
|
737
|
+
except (OSError, ValueError):
|
|
738
|
+
return []
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
def _hunt_piece(lines, budget):
|
|
742
|
+
"""How many of `lines` one hunt reads of a file larger than its budget: those that fit, ending where a
|
|
743
|
+
blank line is followed by a line that starts at the margin - a definition, a heading - when the last
|
|
744
|
+
quarter of them holds such a place, so the next piece opens on a whole one (BUG-HUNTER-3)."""
|
|
745
|
+
taken = spent = 0
|
|
746
|
+
for line in lines:
|
|
747
|
+
spent += len(line.encode(UTF8))
|
|
748
|
+
if spent > budget:
|
|
749
|
+
break
|
|
750
|
+
taken += 1
|
|
751
|
+
for at in range(taken, taken - taken // 4, -1):
|
|
752
|
+
if at < len(lines) and not lines[at - 1].strip() and lines[at][:1].strip():
|
|
753
|
+
return at
|
|
754
|
+
return taken
|
|
755
|
+
|
|
756
|
+
|
|
757
|
+
def hunt_slice(root, cfg, passed_over=None):
|
|
758
|
+
"""The next bounded slice of tracked product files, from a cursor that goes round the tree (#45):
|
|
759
|
+
[(path, text, first line, last line, lines in the file)] and the place in the list the hunt started at.
|
|
760
|
+
|
|
761
|
+
Every line of every file up to `hunter.max_file_kb` is read in some hunt (BUG-HUNTER-3). A file that does
|
|
762
|
+
not fit what is left of a hunt's budget starts the next one; it was passed over, the cursor already beyond
|
|
763
|
+
it, and the same files could be missed on every round. A file larger than a whole hunt is read a piece a
|
|
764
|
+
hunt, cut at a line; it was read to its first `hunter.bytes_per_run` and never further, and polyctl's own
|
|
765
|
+
largest files are three times that. A line longer than a hunt is cut to it. A file past
|
|
766
|
+
`hunter.max_file_kb` is no source a hunt goes through: it is passed over, and named in `passed_over` where
|
|
767
|
+
the caller gives a list. A round that finds nothing it can read ends the hunt, where it went round for ever.
|
|
768
|
+
"""
|
|
769
|
+
try:
|
|
770
|
+
listed = [os.fsdecode(p) for p in _git_output(root, "ls-files", "-z").split(b"\0") if p]
|
|
771
|
+
except RuntimeError:
|
|
772
|
+
return [], 0
|
|
773
|
+
candidates = [p for p in listed if not _is_coordination_path(p, cfg) and os.path.isfile(os.path.join(root, p))]
|
|
774
|
+
if not candidates:
|
|
775
|
+
return [], 0
|
|
776
|
+
state_path = os.path.join(root, ".polyctl", "hunter.json")
|
|
777
|
+
index, line = _hunt_cursor(state_path, candidates)
|
|
778
|
+
cursor, pieces, spent, passed = index, [], 0, 0
|
|
779
|
+
budget, most = settings.get(cfg, "hunter.bytes_per_run"), settings.get(cfg, "hunter.files_per_run")
|
|
780
|
+
largest = settings.get(cfg, "hunter.max_file_kb") * 1024
|
|
781
|
+
while len(pieces) < most and passed < len(candidates):
|
|
782
|
+
path = candidates[index % len(candidates)]
|
|
783
|
+
lines = _hunt_lines(os.path.join(root, path), largest)
|
|
784
|
+
if lines is None and passed_over is not None:
|
|
785
|
+
passed_over.append(path)
|
|
786
|
+
rest = (lines or [])[line - 1:]
|
|
787
|
+
size = sum(len(each.encode(UTF8)) for each in rest)
|
|
788
|
+
if not rest or size <= budget - spent:
|
|
789
|
+
if rest:
|
|
790
|
+
pieces.append((path, "".join(rest), line, len(lines), len(lines)))
|
|
791
|
+
spent += size
|
|
792
|
+
index, line, passed = index + 1, 1, passed + 1
|
|
793
|
+
continue
|
|
794
|
+
if pieces:
|
|
795
|
+
break # it starts the next hunt, whole or by its next piece
|
|
796
|
+
taken = _hunt_piece(rest, budget)
|
|
797
|
+
if taken:
|
|
798
|
+
pieces.append((path, "".join(rest[:taken]), line, line + taken - 1, len(lines)))
|
|
799
|
+
else:
|
|
800
|
+
# One line longer than a whole hunt: what of it fits, a character the cut splits dropped.
|
|
801
|
+
taken = 1
|
|
802
|
+
pieces.append((path, rest[0].encode(UTF8)[:budget].decode(UTF8, "ignore"), line, line, len(lines)))
|
|
803
|
+
line += taken
|
|
804
|
+
if line > len(lines):
|
|
805
|
+
index, line = index + 1, 1
|
|
806
|
+
break
|
|
807
|
+
from .storage import replace_file_durably
|
|
808
|
+
at = index % len(candidates)
|
|
809
|
+
replace_file_durably(state_path, (json.dumps({"cursor": at, "path": candidates[at], "line": line}) + "\n")
|
|
810
|
+
.encode(UTF8))
|
|
811
|
+
return pieces, cursor
|
|
812
|
+
|
|
813
|
+
|
|
814
|
+
def parse_leads(out):
|
|
815
|
+
"""Leads a hunter wrote as `- [category] path:line symbol — what is wrong`, each with a fingerprint (#45)."""
|
|
816
|
+
leads = []
|
|
817
|
+
for line in str(out or "").splitlines():
|
|
818
|
+
found = _LEAD.match(line)
|
|
819
|
+
if not found or found.group(1) not in HUNT_CATEGORIES:
|
|
820
|
+
continue
|
|
821
|
+
category, path, number, symbol, text = found.groups()
|
|
822
|
+
fingerprint = hashlib.sha256(f"{path}|{symbol}|{category}".encode(UTF8)).hexdigest()[:16]
|
|
823
|
+
leads.append({"category": category, "path": path, "line": int(number), "symbol": symbol, "finding": text,
|
|
824
|
+
"fingerprint": fingerprint})
|
|
825
|
+
return leads
|
|
826
|
+
|
|
827
|
+
|
|
828
|
+
# ---- the governance survives the disk (#46) ---------------------------------------------
|
|
829
|
+
|
|
830
|
+
_BACKUP_SKIP = re.compile(r"(\.lock|\.tmp|\.pending|\.bak[^/]*|~)$|(^|/)\.polyctl/reviews/|(^|/)\.polyctl/hunter\.json$")
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
def governance_files(root, cfg):
|
|
834
|
+
"""Every file a project's control plane lives in, relative to the root (#46).
|
|
835
|
+
|
|
836
|
+
Config, authority, board, backlog, gates and sources; decisions, parked work and
|
|
837
|
+
every ledger with its sealed archives; the mailbox; and the review artefacts a
|
|
838
|
+
grant rests on. Locks, temporary files and config backups are not governance.
|
|
839
|
+
"""
|
|
840
|
+
out = []
|
|
841
|
+
for top in (".polyctl", cfg.get("mailbox", "agent-mail")):
|
|
842
|
+
base = os.path.join(root, top)
|
|
843
|
+
for directory, subdirs, files in os.walk(base):
|
|
844
|
+
subdirs[:] = [d for d in subdirs if not os.path.islink(os.path.join(directory, d))]
|
|
845
|
+
for name in files:
|
|
846
|
+
rel = os.path.relpath(os.path.join(directory, name), root).replace(os.sep, "/")
|
|
847
|
+
if not _BACKUP_SKIP.search(rel) and os.path.isfile(os.path.join(root, rel)):
|
|
848
|
+
out.append(rel)
|
|
849
|
+
reviews_dir = cfg.get("reviews", "semantic-review")
|
|
850
|
+
try:
|
|
851
|
+
# Every grant's review, a sealed grant's too, read from the archives held to their seal (LEDGER-HISTORY).
|
|
852
|
+
for name in {row.get("review") for row in authority_history(root)
|
|
853
|
+
if isinstance(row, dict) and row.get("granted") is True and row.get("review")}:
|
|
854
|
+
rel = f"{reviews_dir}/{name}"
|
|
855
|
+
if os.path.isfile(os.path.join(root, rel)):
|
|
856
|
+
out.append(rel)
|
|
857
|
+
except Exception:
|
|
858
|
+
# A chain that cannot be read cannot say which artefacts its grants rest on, and a backup that kept
|
|
859
|
+
# none of them was complete without a word: every one is kept instead (GOVERNANCE-BACKUP-2).
|
|
860
|
+
for directory, subdirs, files in os.walk(os.path.join(root, reviews_dir)):
|
|
861
|
+
subdirs[:] = [d for d in subdirs if not os.path.islink(os.path.join(directory, d))]
|
|
862
|
+
for name in files:
|
|
863
|
+
rel = os.path.relpath(os.path.join(directory, name), root).replace(os.sep, "/")
|
|
864
|
+
if not _BACKUP_SKIP.search(rel) and os.path.isfile(os.path.join(root, rel)):
|
|
865
|
+
out.append(rel)
|
|
866
|
+
return sorted(set(out))
|
|
867
|
+
|
|
868
|
+
|
|
869
|
+
def _manifest(root, files):
|
|
870
|
+
entries = {}
|
|
871
|
+
for rel in files:
|
|
872
|
+
with open(os.path.join(root, rel), "rb") as fh:
|
|
873
|
+
entries[rel] = "sha256:" + hashlib.sha256(fh.read()).hexdigest()
|
|
874
|
+
return {"schema": 1, "project": project_key(root), "at": int(time.time()),
|
|
875
|
+
"head": git_text(root, "rev-parse", "HEAD") or None, "files": entries}
|
|
876
|
+
|
|
877
|
+
|
|
878
|
+
def write_backup(root, cfg, destination):
|
|
879
|
+
"""Write the governance to a destination the project names: a directory, `ref`, or `remote:<name>` (#46).
|
|
880
|
+
|
|
881
|
+
A directory gets `<project>/<stamp>/` with a manifest of every file's digest. A
|
|
882
|
+
ref is a commit of the same files under refs/polyctl/backup/latest, made from blob
|
|
883
|
+
and tree objects without touching the index or the worktree. A remote gets that
|
|
884
|
+
ref pushed, and only after the host says the repository is private.
|
|
885
|
+
Returns the manifest with where it went, and it is recorded in the ledger.
|
|
886
|
+
"""
|
|
887
|
+
files = governance_files(root, cfg)
|
|
888
|
+
manifest = _manifest(root, files)
|
|
889
|
+
stamp = datetime.fromtimestamp(manifest["at"]).strftime("%Y%m%d-%H%M%S")
|
|
890
|
+
if destination.startswith("remote:"):
|
|
891
|
+
remote = destination.split(":", 1)[1]
|
|
892
|
+
if remote_is_private(root, remote) is not True:
|
|
893
|
+
raise RuntimeError(f"not pushing governance to {remote}: the host did not confirm it is private")
|
|
894
|
+
if destination == "ref" or destination.startswith("remote:"):
|
|
895
|
+
where = _backup_ref(root, manifest)
|
|
896
|
+
if destination.startswith("remote:"):
|
|
897
|
+
_git_output(root, "push", remote, "refs/polyctl/backup/latest:refs/polyctl/backup/latest", timeout=300)
|
|
898
|
+
where = f"{remote} {where}"
|
|
899
|
+
else:
|
|
900
|
+
target = os.path.join(os.path.expanduser(destination), manifest["project"], stamp)
|
|
901
|
+
from .storage import replace_file_durably
|
|
902
|
+
for rel in files:
|
|
903
|
+
with open(os.path.join(root, rel), "rb") as fh:
|
|
904
|
+
replace_file_durably(os.path.join(target, rel), fh.read())
|
|
905
|
+
replace_file_durably(os.path.join(target, "manifest.json"),
|
|
906
|
+
(json.dumps(manifest, indent=1, sort_keys=True) + "\n").encode(UTF8))
|
|
907
|
+
where = target
|
|
908
|
+
manifest["where"] = where
|
|
909
|
+
from .storage import append_jsonl
|
|
910
|
+
append_jsonl(os.path.join(root, ".polyctl", "ledger", "backups.jsonl"),
|
|
911
|
+
{"at": manifest["at"], "where": where, "files": len(files), "head": manifest["head"]})
|
|
912
|
+
return manifest
|
|
913
|
+
|
|
914
|
+
|
|
915
|
+
def _backup_ref(root, manifest):
|
|
916
|
+
"""A commit holding the governance files and the manifest, made without an index (#46)."""
|
|
917
|
+
tree = {}
|
|
918
|
+
for rel in list(manifest["files"]) + ["polyctl-backup-manifest.json"]:
|
|
919
|
+
data = (json.dumps(manifest, indent=1, sort_keys=True) + "\n").encode(UTF8) if rel == "polyctl-backup-manifest.json" \
|
|
920
|
+
else open(os.path.join(root, rel), "rb").read()
|
|
921
|
+
blob = subprocess.run([git_binary(), "hash-object", "-w", "--stdin"], cwd=root, input=data,
|
|
922
|
+
stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True).stdout.decode().strip()
|
|
923
|
+
node = tree
|
|
924
|
+
parts = rel.split("/")
|
|
925
|
+
for part in parts[:-1]:
|
|
926
|
+
node = node.setdefault(part, {})
|
|
927
|
+
node[parts[-1]] = blob
|
|
928
|
+
|
|
929
|
+
def write(node):
|
|
930
|
+
lines = []
|
|
931
|
+
for name in sorted(node):
|
|
932
|
+
if isinstance(node[name], dict):
|
|
933
|
+
lines.append(f"040000 tree {write(node[name])}\t{name}")
|
|
934
|
+
else:
|
|
935
|
+
lines.append(f"100644 blob {node[name]}\t{name}")
|
|
936
|
+
return subprocess.run([git_binary(), "mktree"], cwd=root, input=("\n".join(lines) + "\n").encode(UTF8),
|
|
937
|
+
stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True).stdout.decode().strip()
|
|
938
|
+
|
|
939
|
+
commit = subprocess.run([git_binary(), "-c", "user.name=polyctl", "-c", "user.email=polyctl@localhost", "commit-tree",
|
|
940
|
+
write(tree), "-m", f"polyctl backup of {manifest['project']} governance"], cwd=root,
|
|
941
|
+
stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True).stdout.decode().strip()
|
|
942
|
+
_git_output(root, "update-ref", "refs/polyctl/backup/latest", commit)
|
|
943
|
+
return f"refs/polyctl/backup/latest {commit[:12]}"
|
|
944
|
+
|
|
945
|
+
|
|
946
|
+
def remote_is_private(root, remote):
|
|
947
|
+
"""True only when the host says the remote's repository is private; None when it cannot say (#46, #83).
|
|
948
|
+
|
|
949
|
+
The remote is judged by the URL `git push` reaches, its pushurl where one is set, and GitHub is asked
|
|
950
|
+
only about a repository on github.com itself (MAIL-SYNC-3).
|
|
951
|
+
"""
|
|
952
|
+
try:
|
|
953
|
+
url = _push_url(root, remote)
|
|
954
|
+
except RuntimeError:
|
|
955
|
+
return None
|
|
956
|
+
return _github_says_private(url)
|
|
957
|
+
|
|
958
|
+
|
|
959
|
+
def _backup_path(base, rel):
|
|
960
|
+
"""`rel` under `base`, or None where a manifest's words or a link would leave it (GOVERNANCE-BACKUP-2).
|
|
961
|
+
|
|
962
|
+
A manifest is not authenticated: `../x` or an absolute path in it was written wherever it
|
|
963
|
+
pointed. polyctl writes every path relative, with forward slashes and no `.` or `..` part.
|
|
964
|
+
"""
|
|
965
|
+
words = str(rel).split("/")
|
|
966
|
+
if not rel or "\\" in rel or os.path.isabs(rel) or os.path.splitdrive(rel)[0] \
|
|
967
|
+
or any(word in ("", ".", "..") for word in words):
|
|
968
|
+
return None
|
|
969
|
+
path = os.path.join(base, *words)
|
|
970
|
+
return path if _within(path, base) else None
|
|
971
|
+
|
|
972
|
+
|
|
973
|
+
def restore_backup(root, source):
|
|
974
|
+
"""Put the governance back from a backup directory, verifying every file against its manifest (#46).
|
|
975
|
+
|
|
976
|
+
Returns (restored, unverified): a file whose bytes do not match the digest it was
|
|
977
|
+
backed up with is not restored, and is named, and so is a path that would leave the
|
|
978
|
+
checkout or the backup (GOVERNANCE-BACKUP-2).
|
|
979
|
+
"""
|
|
980
|
+
with open(os.path.join(source, "manifest.json"), encoding=UTF8) as fh:
|
|
981
|
+
manifest = json.load(fh)
|
|
982
|
+
from .storage import replace_file_durably
|
|
983
|
+
restored, unverified = [], []
|
|
984
|
+
for rel, digest in sorted(manifest["files"].items()):
|
|
985
|
+
path, target = _backup_path(source, rel), _backup_path(root, rel)
|
|
986
|
+
if path is None or target is None:
|
|
987
|
+
unverified.append(f"{rel}: not a path inside the checkout")
|
|
988
|
+
continue
|
|
989
|
+
try:
|
|
990
|
+
with open(path, "rb") as fh:
|
|
991
|
+
data = fh.read()
|
|
992
|
+
except OSError:
|
|
993
|
+
unverified.append(f"{rel}: missing from the backup")
|
|
994
|
+
continue
|
|
995
|
+
if "sha256:" + hashlib.sha256(data).hexdigest() != digest:
|
|
996
|
+
unverified.append(f"{rel}: its bytes do not match the manifest")
|
|
997
|
+
continue
|
|
998
|
+
replace_file_durably(target, data)
|
|
999
|
+
restored.append(rel)
|
|
1000
|
+
return restored, unverified
|
|
1001
|
+
|
|
1002
|
+
|
|
1003
|
+
RESTORE_CHAIN = "polyctl-restore-row-v1"
|
|
1004
|
+
|
|
1005
|
+
|
|
1006
|
+
def backup_manifest_digest(source):
|
|
1007
|
+
"""The SHA-256 digest of a backup directory's `manifest.json` bytes (LEDGER-ATTESTED)."""
|
|
1008
|
+
with open(os.path.join(source, "manifest.json"), "rb") as fh:
|
|
1009
|
+
return "sha256:" + hashlib.sha256(fh.read()).hexdigest()
|
|
1010
|
+
|
|
1011
|
+
|
|
1012
|
+
def record_restore_attestation(root, by, manifest_digest, ledgers):
|
|
1013
|
+
"""Record a person's attestation of a restored backup in `.polyctl/ledger/restores.jsonl` (LEDGER-ATTESTED)."""
|
|
1014
|
+
from .storage import append_chained_in_tree
|
|
1015
|
+
user, interactive = _login_and_terminal()
|
|
1016
|
+
return append_chained_in_tree(
|
|
1017
|
+
root,
|
|
1018
|
+
".polyctl/ledger/restores.jsonl",
|
|
1019
|
+
{"at": int(time.time()), "by": str(by).strip(), "user": user, "interactive": interactive,
|
|
1020
|
+
"manifest": manifest_digest, "ledgers": {rel: rows for rel, rows, *_ in ledgers}},
|
|
1021
|
+
RESTORE_CHAIN,
|
|
1022
|
+
)
|
|
1023
|
+
|
|
1024
|
+
|
|
1025
|
+
def latest_restore_attestation(root):
|
|
1026
|
+
"""The newest recorded restore attestation in `.polyctl/ledger/restores.jsonl`, or None (LEDGER-ATTESTED)."""
|
|
1027
|
+
from .storage import read_chained_in_tree
|
|
1028
|
+
rows = [row for row in read_chained_in_tree(root, ".polyctl/ledger/restores.jsonl", RESTORE_CHAIN)
|
|
1029
|
+
if isinstance(row, dict)]
|
|
1030
|
+
return rows[-1] if rows else None
|
|
1031
|
+
|
|
1032
|
+
|
|
1033
|
+
def _restored_ledgers(root, restored, *, attest=False):
|
|
1034
|
+
"""Validate each restored chained ledger against its seal and archives; when `attest=True`, record this machine's
|
|
1035
|
+
checkpoint for each one that validates, and return `[(rel, rows, digest)]` of the gate ledgers attested (or, when
|
|
1036
|
+
`attest=False`, those that need attestation on this machine) (LEDGER-ATTESTED)."""
|
|
1037
|
+
from .storage import GATE_LEDGERS, LedgerCorruption, _load_seal, attest_restored_ledger, seal_path
|
|
1038
|
+
gate_rels = {rel for _, rel, _, _, _ in GATE_LEDGERS}
|
|
1039
|
+
chains = tuple((rel, chain, legacy) for _, rel, chain, legacy, _ in GATE_LEDGERS) + (
|
|
1040
|
+
(".polyctl/ledger/merges.jsonl", MERGE_CHAIN, False),
|
|
1041
|
+
(".polyctl/ledger/slices.jsonl", SLICE_CHAIN, False),
|
|
1042
|
+
(".polyctl/ledger/mail-store.jsonl", MAIL_STORE_CHAIN, False),
|
|
1043
|
+
(".polyctl/ledger/repairs.jsonl", REPAIR_CHAIN, False),
|
|
1044
|
+
(".polyctl/ledger/restores.jsonl", RESTORE_CHAIN, False),
|
|
1045
|
+
)
|
|
1046
|
+
verified = set(restored)
|
|
1047
|
+
unattested, errors = [], []
|
|
1048
|
+
for rel, chain, legacy in chains:
|
|
1049
|
+
if rel not in verified:
|
|
1050
|
+
continue
|
|
1051
|
+
path = os.path.join(root, *rel.split("/"))
|
|
1052
|
+
if os.path.lexists(seal_path(path)):
|
|
1053
|
+
if f"{rel}.seal.json" not in verified:
|
|
1054
|
+
errors.append(f"{rel}: its seal was not verified from the backup")
|
|
1055
|
+
continue
|
|
1056
|
+
try:
|
|
1057
|
+
seal = _load_seal(path)
|
|
1058
|
+
except Exception as exc:
|
|
1059
|
+
errors.append(str(exc))
|
|
1060
|
+
continue
|
|
1061
|
+
if seal and any(f".polyctl/ledger/sealed/{name}" not in verified for name in (seal.get("archives") or [])):
|
|
1062
|
+
errors.append(f"{rel}: its sealed archives were not verified from the backup")
|
|
1063
|
+
continue
|
|
1064
|
+
try:
|
|
1065
|
+
is_gate = rel in gate_rels
|
|
1066
|
+
rows, digest, already = attest_restored_ledger(path, chain, legacy_prefix=legacy, record=attest)
|
|
1067
|
+
if is_gate and rows > 0 and (attest or not already):
|
|
1068
|
+
unattested.append((rel, rows, digest))
|
|
1069
|
+
except Exception as exc:
|
|
1070
|
+
errors.append(str(exc))
|
|
1071
|
+
if errors:
|
|
1072
|
+
raise LedgerCorruption("; ".join(errors))
|
|
1073
|
+
return unattested
|
|
1074
|
+
|
|
1075
|
+
|
|
1076
|
+
|
|
1077
|
+
def backup_age(root):
|
|
1078
|
+
"""(seconds since the newest backup, where it went), or None when there is none (#46)."""
|
|
1079
|
+
from .storage import read_jsonl
|
|
1080
|
+
rows = [row for row in read_jsonl(os.path.join(root, ".polyctl", "ledger", "backups.jsonl")) if isinstance(row, dict)]
|
|
1081
|
+
return (time.time() - float(rows[-1]["at"]), rows[-1].get("where")) if rows else None
|
|
1082
|
+
|
|
1083
|
+
|
|
1084
|
+
# ---- the harness content seam: borrow, pin, verify (#14, #15) ----------------------------
|
|
1085
|
+
|
|
1086
|
+
CONTENT_TEXT = (".md", ".txt", ".json", ".yaml", ".yml")
|
|
1087
|
+
# Each kind of content a source keeps, and the key naming an entry of its list in .polyctl/content.json.
|
|
1088
|
+
CONTENT_KINDS = (("skills", "skill"), ("steering", "steering"), ("agents", "agent"))
|
|
1089
|
+
CONTENT_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*")
|
|
1090
|
+
# A tree entry of one of these modes is not a text file, whatever its name says.
|
|
1091
|
+
_NOT_TEXT_MODES = {"120000": "a symbolic link", "160000": "a submodule", "100755": "executable"}
|
|
1092
|
+
|
|
1093
|
+
|
|
1094
|
+
def content_manifest_path(root):
|
|
1095
|
+
return os.path.join(root, ".polyctl", "content.json")
|
|
1096
|
+
|
|
1097
|
+
|
|
1098
|
+
def content_manifest(root):
|
|
1099
|
+
"""{kind: [entry]} as .polyctl/content.json records them; a kind it lacks, or holds as no list, is empty."""
|
|
1100
|
+
try:
|
|
1101
|
+
with open(content_manifest_path(root), encoding=UTF8) as fh:
|
|
1102
|
+
data = json.load(fh)
|
|
1103
|
+
except (OSError, ValueError):
|
|
1104
|
+
data = {}
|
|
1105
|
+
data = data if isinstance(data, dict) else {}
|
|
1106
|
+
return {kind: [entry for entry in data[kind] if isinstance(entry, dict)] if isinstance(data.get(kind), list)
|
|
1107
|
+
else [] for kind, _ in CONTENT_KINDS}
|
|
1108
|
+
|
|
1109
|
+
|
|
1110
|
+
def fetch_pinned(source, pin, paths, workdir):
|
|
1111
|
+
"""{path: (mode, blob id)} for each file under `paths` of `source` at commit `pin` (#14).
|
|
1112
|
+
|
|
1113
|
+
Nothing unpinned is ever fetched, and nothing is checked out: a file is read by its
|
|
1114
|
+
blob id when it is wanted (`pinned_blobs`). So its bytes are the commit's whatever this
|
|
1115
|
+
machine converts - Git for Windows checks text out with CRLF by default, and neither
|
|
1116
|
+
the files nor their digests were the commit's (#71) - and a symbolic link is a mode
|
|
1117
|
+
here, never a path polyctl follows to a file of this machine. A path the commit does not
|
|
1118
|
+
hold is absent from the answer.
|
|
1119
|
+
"""
|
|
1120
|
+
if not re.fullmatch(r"[0-9a-f]{40}", str(pin or "")):
|
|
1121
|
+
raise ValueError(f"{pin!r} is not a pin: name a full 40-character commit id, never a branch or a tag")
|
|
1122
|
+
git = git_binary()
|
|
1123
|
+
subprocess.run([git, "init", "-q", workdir], check=True, capture_output=True)
|
|
1124
|
+
subprocess.run([git, "-C", workdir, "fetch", "-q", "--depth", "1", source, pin], check=True, capture_output=True,
|
|
1125
|
+
timeout=300)
|
|
1126
|
+
fetched = subprocess.run([git, "-C", workdir, "rev-parse", "FETCH_HEAD"], check=True, capture_output=True,
|
|
1127
|
+
text=True, encoding=UTF8, errors="replace").stdout.strip()
|
|
1128
|
+
if fetched != pin:
|
|
1129
|
+
raise RuntimeError(f"{source} answered {fetched[:12]}, not the pinned {pin[:12]}")
|
|
1130
|
+
if not paths:
|
|
1131
|
+
return {}
|
|
1132
|
+
listing = subprocess.run([git, "-C", workdir, "--literal-pathspecs", "ls-tree", "-r", "-z", pin, "--", *paths],
|
|
1133
|
+
check=True, capture_output=True, timeout=300).stdout
|
|
1134
|
+
files = {}
|
|
1135
|
+
for record in listing.split(b"\0"):
|
|
1136
|
+
meta, _, path = record.partition(b"\t")
|
|
1137
|
+
fields = meta.decode(UTF8, "replace").split()
|
|
1138
|
+
if path and len(fields) == 3:
|
|
1139
|
+
files[path.decode(UTF8, "surrogateescape")] = (fields[0], fields[2])
|
|
1140
|
+
return files
|
|
1141
|
+
|
|
1142
|
+
|
|
1143
|
+
def pinned_blobs(workdir, ids):
|
|
1144
|
+
"""{blob id: bytes} for each id, read from the fetched commit by one `git cat-file --batch` (#15)."""
|
|
1145
|
+
wanted = sorted(set(ids))
|
|
1146
|
+
if not wanted:
|
|
1147
|
+
return {}
|
|
1148
|
+
out = subprocess.run([git_binary(), "-C", workdir, "cat-file", "--batch"],
|
|
1149
|
+
input="".join(f"{oid}\n" for oid in wanted).encode(UTF8), check=True, capture_output=True,
|
|
1150
|
+
timeout=300).stdout
|
|
1151
|
+
blobs, at = {}, 0
|
|
1152
|
+
for oid in wanted:
|
|
1153
|
+
end = out.find(b"\n", at)
|
|
1154
|
+
header = out[at:end].split() if end >= 0 else []
|
|
1155
|
+
if len(header) != 3 or header[1] != b"blob":
|
|
1156
|
+
raise RuntimeError(f"the pinned commit holds no blob {oid[:12]}")
|
|
1157
|
+
size = int(header[2])
|
|
1158
|
+
blobs[oid] = out[end + 1:end + 1 + size]
|
|
1159
|
+
at = end + 1 + size + 1
|
|
1160
|
+
return blobs
|
|
1161
|
+
|
|
1162
|
+
|
|
1163
|
+
def _not_borrowed(path, mode, data=None):
|
|
1164
|
+
"""Why a file of a pinned tree is not borrowed, or None when it is text (#14).
|
|
1165
|
+
|
|
1166
|
+
The mode is the commit's, so an executable is one on every platform: Windows keeps
|
|
1167
|
+
no execute bit on disk, and os.access says every file there has one (#71).
|
|
1168
|
+
"""
|
|
1169
|
+
if mode in _NOT_TEXT_MODES:
|
|
1170
|
+
return f"{_NOT_TEXT_MODES[mode]}: only text is borrowed"
|
|
1171
|
+
if not path.lower().endswith(CONTENT_TEXT):
|
|
1172
|
+
return "not Markdown, text, JSON or YAML: only text is borrowed"
|
|
1173
|
+
if data is not None and data[:2] == b"#!":
|
|
1174
|
+
return "a script: only text is borrowed"
|
|
1175
|
+
return None
|
|
1176
|
+
|
|
1177
|
+
|
|
1178
|
+
def _front_matter(data, field):
|
|
1179
|
+
"""What a Markdown file's front matter says for a top-level `field`, or None."""
|
|
1180
|
+
block = re.match(rb"\A---\r?\n(.*?)\r?\n---(?:\r?\n|\Z)", data, re.S)
|
|
1181
|
+
found = block and re.search(rb"^" + re.escape(field.encode(UTF8)) + rb"[ \t]*:[ \t]*['\"]?([^'\"\s#]+)",
|
|
1182
|
+
block.group(1), re.M)
|
|
1183
|
+
return found.group(1).decode(UTF8, "replace") if found else None
|
|
1184
|
+
|
|
1185
|
+
|
|
1186
|
+
def _declared(*blocks):
|
|
1187
|
+
"""One {name: text} from each block that is a dict, a later block's names winning."""
|
|
1188
|
+
merged = {}
|
|
1189
|
+
for block in blocks:
|
|
1190
|
+
merged.update(block if isinstance(block, dict) else {})
|
|
1191
|
+
return merged
|
|
1192
|
+
|
|
1193
|
+
|
|
1194
|
+
def _kept_files(root):
|
|
1195
|
+
"""Files an adapter says a harness or polyctl keeps for itself; borrowed content replaces none (#15).
|
|
1196
|
+
|
|
1197
|
+
A playbook, a coordination file, an MCP registration, settings whose hooks run
|
|
1198
|
+
commands: a borrowed file landing on one would speak for polyctl or run what nobody
|
|
1199
|
+
pinned. A path ending in `/` stands for everything under it.
|
|
1200
|
+
"""
|
|
1201
|
+
kept = set()
|
|
1202
|
+
for adapter in list(package_adapters().values()) + [entry["adapter"] for entry in adapter_catalog(root).values()]:
|
|
1203
|
+
adapter = adapter if isinstance(adapter, dict) else {}
|
|
1204
|
+
directives = adapter.get("directives") if isinstance(adapter.get("directives"), dict) else {}
|
|
1205
|
+
named = [_declared(directives.get("playbook")).get("path"), directives.get("coordination"),
|
|
1206
|
+
directives.get("config"), directives.get("mcp"), _declared(adapter.get("mcp")).get("file")]
|
|
1207
|
+
for field in ("polyctl_files", "rule_files", "steering_files"):
|
|
1208
|
+
named += directives.get(field) if isinstance(directives.get(field), list) else []
|
|
1209
|
+
named += _declared(directives.get("command_hooks")).get("files") or []
|
|
1210
|
+
kept.update(rel for rel in named if isinstance(rel, str) and rel)
|
|
1211
|
+
return kept
|
|
1212
|
+
|
|
1213
|
+
|
|
1214
|
+
def _content_refusal(root, harness, target, kept):
|
|
1215
|
+
"""Why polyctl must not write `target` for `harness`, or None (#15).
|
|
1216
|
+
|
|
1217
|
+
Borrowed content lands in the harness's own directory, as the package's adapter
|
|
1218
|
+
declares it in `detect.dirs`, and nowhere else. A user's or a project's adapter layer
|
|
1219
|
+
is writable by the agents it describes, so a directory it names elsewhere - the
|
|
1220
|
+
product, polyctl's state, another harness's settings - is refused, as is a symbolic link
|
|
1221
|
+
on the way and a file a harness or polyctl keeps for itself.
|
|
1222
|
+
"""
|
|
1223
|
+
homes = [str(d).strip("/") for d in ((package_adapters().get(harness) or {}).get("detect") or {}).get("dirs") or []
|
|
1224
|
+
if str(d).strip("/")]
|
|
1225
|
+
steps = _plain_steps(target)
|
|
1226
|
+
if not steps:
|
|
1227
|
+
return f"{target} is not a path inside the project"
|
|
1228
|
+
if not any(target.startswith(home + "/") for home in homes):
|
|
1229
|
+
return f"{target} is outside {harness}'s own directory ({', '.join(homes) or 'the package declares none'})"
|
|
1230
|
+
# A content import has the same owners on case-sensitive and insensitive volumes (TRIAGE-4-REST).
|
|
1231
|
+
folded = target.casefold()
|
|
1232
|
+
if any(folded == rel.casefold() or (rel.endswith("/") and folded.startswith(rel.casefold())) for rel in kept):
|
|
1233
|
+
return f"{target} is a file a harness or polyctl keeps for itself"
|
|
1234
|
+
real = os.path.realpath(root)
|
|
1235
|
+
if os.path.normcase(os.path.realpath(os.path.join(root, *steps))) != os.path.normcase(os.path.join(real, *steps)):
|
|
1236
|
+
return f"{target} passes through a symbolic link"
|
|
1237
|
+
return None
|
|
1238
|
+
|
|
1239
|
+
|
|
1240
|
+
def _content_item(kind, name):
|
|
1241
|
+
return {"kind": kind, "name": name, "files": {}, "skipped": [], "unsupported": [], "notes": []}
|
|
1242
|
+
|
|
1243
|
+
|
|
1244
|
+
def _skill_item(name, head, tree, blobs, declared):
|
|
1245
|
+
"""A skill's text files for each harness that takes skills, or its SKILL.md as a steering file (#14)."""
|
|
1246
|
+
item, texts = _content_item("skills", name), {}
|
|
1247
|
+
for path, (mode, oid) in sorted(tree.items()):
|
|
1248
|
+
if not path.startswith(head):
|
|
1249
|
+
continue
|
|
1250
|
+
rel = path[len(head):]
|
|
1251
|
+
steps = rel.split("/")
|
|
1252
|
+
if "hooks" in steps[:-1]:
|
|
1253
|
+
hooks = ("/".join(steps[:steps.index("hooks") + 1]) + "/", "hooks are never imported")
|
|
1254
|
+
if hooks not in item["skipped"]:
|
|
1255
|
+
item["skipped"].append(hooks)
|
|
1256
|
+
continue
|
|
1257
|
+
why = _not_borrowed(rel, mode, blobs.get(oid))
|
|
1258
|
+
if why:
|
|
1259
|
+
item["skipped"].append((rel, why))
|
|
1260
|
+
else:
|
|
1261
|
+
texts[rel] = blobs[oid]
|
|
1262
|
+
for harness, (directives, _, _) in declared.items():
|
|
1263
|
+
if directives.get("skills_dir"):
|
|
1264
|
+
item["files"].update({f"{directives['skills_dir']}/{name}/{rel}": (harness, data)
|
|
1265
|
+
for rel, data in texts.items()})
|
|
1266
|
+
elif directives.get("steering_dir") and "SKILL.md" in texts:
|
|
1267
|
+
body = re.sub(r"\A---\n.*?\n---\n+", "", texts["SKILL.md"].decode(UTF8, "replace"), flags=re.S)
|
|
1268
|
+
header = str(_declared(directives.get("steering_inclusion")).get("skill_header") or "")
|
|
1269
|
+
item["files"][f"{directives['steering_dir']}/{name}.md"] = (harness, (header + body).encode(UTF8))
|
|
1270
|
+
return item
|
|
1271
|
+
|
|
1272
|
+
|
|
1273
|
+
def _steering_item(name, path, tree, blobs, declared):
|
|
1274
|
+
"""A steering file, byte for byte and front matter and all, for each harness that reads steering (#15)."""
|
|
1275
|
+
item = _content_item("steering", name)
|
|
1276
|
+
mode, oid = tree[path]
|
|
1277
|
+
why = _not_borrowed(path, mode, blobs.get(oid))
|
|
1278
|
+
if why:
|
|
1279
|
+
item["skipped"].append((f"steering/{name}.md", why))
|
|
1280
|
+
return item
|
|
1281
|
+
for harness, (directives, _, known) in declared.items():
|
|
1282
|
+
if directives.get("steering_dir"):
|
|
1283
|
+
item["files"][f"{directives['steering_dir']}/{name}.md"] = (harness, blobs[oid])
|
|
1284
|
+
elif known:
|
|
1285
|
+
item["notes"].append(f"{harness} reads no steering files")
|
|
1286
|
+
return item
|
|
1287
|
+
|
|
1288
|
+
|
|
1289
|
+
def _resource_outside(value):
|
|
1290
|
+
"""Whether an agent's resource names a file outside the project's tree: a `file://` path that is absolute,
|
|
1291
|
+
starts at a home (`~`), names a drive, or steps up with `..` (KIRO-CONTENT-2)."""
|
|
1292
|
+
text = str(value)
|
|
1293
|
+
if not text.startswith("file://"):
|
|
1294
|
+
return False
|
|
1295
|
+
path = text[len("file://"):].replace("\\", "/")
|
|
1296
|
+
return path.startswith(("/", "~")) or path[1:2] == ":" or ".." in path.split("/")
|
|
1297
|
+
|
|
1298
|
+
|
|
1299
|
+
def _agent_item(name, path, tree, blobs, declared, takes_agents):
|
|
1300
|
+
"""An agent definition in each harness's declared format, with no field in it that runs a command (#15).
|
|
1301
|
+
|
|
1302
|
+
A field the package's adapter or a layer over it names in `agent_format.commands` -
|
|
1303
|
+
hooks, servers the harness starts - is taken out and named, so a layer cannot take one
|
|
1304
|
+
back in. The fields that grant trust, and an empty field that loads context, are kept
|
|
1305
|
+
and named: whoever runs the agent should see them before it runs.
|
|
1306
|
+
"""
|
|
1307
|
+
item = _content_item("agents", name)
|
|
1308
|
+
for harness, (directives, shipped, known) in declared.items():
|
|
1309
|
+
if harness not in takes_agents:
|
|
1310
|
+
if known:
|
|
1311
|
+
item["notes"].append(f"{harness} reads no agent definitions polyctl can borrow")
|
|
1312
|
+
continue
|
|
1313
|
+
shape, (mode, oid) = _declared(directives.get("agent_format")), tree[path]
|
|
1314
|
+
why = _not_borrowed(path, mode, blobs.get(oid))
|
|
1315
|
+
try:
|
|
1316
|
+
definition = None if why else json.loads(blobs[oid].decode("utf-8-sig"))
|
|
1317
|
+
except ValueError:
|
|
1318
|
+
definition = None
|
|
1319
|
+
if not why and not isinstance(definition, dict):
|
|
1320
|
+
why = "not a JSON object, which an agent definition is"
|
|
1321
|
+
elif not why and definition.get("name", name) != name:
|
|
1322
|
+
why = f"it names itself {definition.get('name')!r}, and an agent is named as its file"
|
|
1323
|
+
if why:
|
|
1324
|
+
if (f"agents/{name}.json", why) not in item["skipped"]:
|
|
1325
|
+
item["skipped"].append((f"agents/{name}.json", why))
|
|
1326
|
+
continue
|
|
1327
|
+
commands = _declared(shape.get("commands"), _declared(shipped.get("agent_format")).get("commands"))
|
|
1328
|
+
taken = [field for field in sorted(commands) if definition.get(field)]
|
|
1329
|
+
for field in taken:
|
|
1330
|
+
definition.pop(field)
|
|
1331
|
+
item["skipped"].append((f"agents/{name}.json {field}", str(commands[field])))
|
|
1332
|
+
# A resource is context the agent loads before it runs: one outside the project's tree loads a file
|
|
1333
|
+
# of this machine the source chose - a key, a credential - so it is taken out and named, and what is
|
|
1334
|
+
# kept is named as a grant is (KIRO-CONTENT-2).
|
|
1335
|
+
context = _declared(shape.get("context"))
|
|
1336
|
+
resources = definition.get(context.get("field")) if context.get("field") else None
|
|
1337
|
+
if isinstance(resources, list) and any(_resource_outside(value) for value in resources):
|
|
1338
|
+
for value in resources:
|
|
1339
|
+
if _resource_outside(value):
|
|
1340
|
+
item["skipped"].append((f"agents/{name}.json {context['field']} {value}",
|
|
1341
|
+
"a file outside the project's tree, which the source chose: never "
|
|
1342
|
+
"imported"))
|
|
1343
|
+
definition[context["field"]] = [value for value in resources if not _resource_outside(value)]
|
|
1344
|
+
taken.append(context["field"])
|
|
1345
|
+
# So does a field whose `file://` value the agent loads as text - Kiro's prompt is inline or a file: one outside
|
|
1346
|
+
# the project's tree was kept as written, and loaded a file of this machine the source chose into the agent's
|
|
1347
|
+
# instructions. The package's list and a layer's are read together, so a layer cannot take one back in
|
|
1348
|
+
# (REVIEW-PATH-HIGHS).
|
|
1349
|
+
loads = _declared(shape.get("loads"), _declared(shipped.get("agent_format")).get("loads"))
|
|
1350
|
+
for field in sorted(loads):
|
|
1351
|
+
value = definition.get(field)
|
|
1352
|
+
if isinstance(value, str) and _resource_outside(value):
|
|
1353
|
+
definition.pop(field)
|
|
1354
|
+
item["skipped"].append((f"agents/{name}.json {field} {value}",
|
|
1355
|
+
f"{loads[field]}, outside the project's tree, which the source chose: never "
|
|
1356
|
+
"imported"))
|
|
1357
|
+
taken.append(field)
|
|
1358
|
+
target = f"{directives['agents_dir']}/{name}.json"
|
|
1359
|
+
item["files"][target] = (harness, (json.dumps(definition, indent=2, ensure_ascii=False) + "\n").encode(UTF8)
|
|
1360
|
+
if taken else blobs[oid])
|
|
1361
|
+
for field, meaning in sorted(_declared(shape.get("grants")).items()):
|
|
1362
|
+
value = definition.get(field)
|
|
1363
|
+
if value:
|
|
1364
|
+
shown = value if isinstance(value, list) else sorted(value) if isinstance(value, dict) else [value]
|
|
1365
|
+
item["notes"].append(f"{target} keeps {field} {', '.join(map(str, shown))}: {meaning}")
|
|
1366
|
+
if context.get("field") and not definition.get(context["field"]):
|
|
1367
|
+
item["notes"].append(f"{target} names no {context['field']}: {context.get('empty')}")
|
|
1368
|
+
elif context.get("field") and isinstance(definition.get(context["field"]), list):
|
|
1369
|
+
item["notes"].append(f"{target} keeps {context['field']} "
|
|
1370
|
+
f"{', '.join(map(str, definition[context['field']]))}: the context it loads")
|
|
1371
|
+
return item
|
|
1372
|
+
|
|
1373
|
+
|
|
1374
|
+
def _unsupported_inclusion(item, declared):
|
|
1375
|
+
"""Name each steering file whose inclusion its harness does not honour; the file is written as it is (#15)."""
|
|
1376
|
+
for target, (harness, data) in sorted(item["files"].items()):
|
|
1377
|
+
directives = declared[harness][0]
|
|
1378
|
+
rules = _declared(directives.get("steering_inclusion"))
|
|
1379
|
+
if not (rules.get("field") and directives.get("steering_dir")
|
|
1380
|
+
and target.startswith(f"{directives['steering_dir']}/")):
|
|
1381
|
+
continue
|
|
1382
|
+
mode = _front_matter(data, rules["field"]) or str(rules.get("default") or "")
|
|
1383
|
+
if mode not in (rules.get("honoured") or []):
|
|
1384
|
+
item["unsupported"].append((target, f"{rules['field']}: {mode} - {rules.get('unhonoured')}"))
|
|
1385
|
+
|
|
1386
|
+
|
|
1387
|
+
def _content_refusals(root, items):
|
|
1388
|
+
"""Every reason a planned write must not be made; one of them stops them all (#15)."""
|
|
1389
|
+
record, kept, refused, writers = content_manifest(root), _kept_files(root), [], {}
|
|
1390
|
+
for item in items:
|
|
1391
|
+
key = dict(CONTENT_KINDS)[item["kind"]]
|
|
1392
|
+
label = f"{key} {item['name']}"
|
|
1393
|
+
for target, (harness, data) in sorted(item["files"].items()):
|
|
1394
|
+
why = _content_refusal(root, harness, target, kept)
|
|
1395
|
+
folded = target.casefold()
|
|
1396
|
+
if not why and folded in writers:
|
|
1397
|
+
why = f"{target} would be written by both {writers[folded]} and {label}"
|
|
1398
|
+
writers.setdefault(folded, label)
|
|
1399
|
+
# A file another entry vendored is that entry's, whatever its bytes: equal bytes left two entries
|
|
1400
|
+
# owning one file, and removing either took it from the other (KIRO-CONTENT-2).
|
|
1401
|
+
owner = next((f"{dict(CONTENT_KINDS)[kind]} {entry.get(dict(CONTENT_KINDS)[kind])}"
|
|
1402
|
+
for kind, entries in record.items() for entry in entries
|
|
1403
|
+
if any(target.casefold() == rel.casefold() for rel in (entry.get("files") or {}))
|
|
1404
|
+
and not (kind == item["kind"] and entry.get(key) == item["name"])), None)
|
|
1405
|
+
if not why and owner:
|
|
1406
|
+
why = f"{target} was vendored as {owner}: remove that first"
|
|
1407
|
+
path = os.path.join(root, *target.split("/"))
|
|
1408
|
+
if not why and os.path.isdir(path):
|
|
1409
|
+
why = f"{target} is a directory"
|
|
1410
|
+
elif not why and os.path.exists(path):
|
|
1411
|
+
with open(path, "rb") as fh:
|
|
1412
|
+
current = fh.read()
|
|
1413
|
+
# A manifest names an owner, not permission to discard a later hand edit (TRIAGE-4-REST).
|
|
1414
|
+
recorded = [digest for entry in record[item["kind"]] if entry.get(key) == item["name"]
|
|
1415
|
+
for rel, digest in (entry.get("files") or {}).items()
|
|
1416
|
+
if target.casefold() == rel.casefold()]
|
|
1417
|
+
digest = "sha256:" + hashlib.sha256(current).hexdigest()
|
|
1418
|
+
if recorded and any(previous != digest for previous in recorded):
|
|
1419
|
+
why = f"{target} changed since it was vendored: move it aside first"
|
|
1420
|
+
elif current != data and not recorded:
|
|
1421
|
+
why = f"{target} exists and polyctl did not vendor it as {label}: move it aside first"
|
|
1422
|
+
if why:
|
|
1423
|
+
refused.append(why)
|
|
1424
|
+
return refused
|
|
1425
|
+
|
|
1426
|
+
|
|
1427
|
+
def vendor_content(root, source, pin, harnesses, skills=(), steering=(), agents=(), base="", dry_run=False):
|
|
1428
|
+
"""Borrow skills, steering files and agent definitions pinned to a commit, text only, into each harness (#14, #15).
|
|
1429
|
+
|
|
1430
|
+
A source keeps `skills/<name>/`, `steering/<name>.md` and `agents/<name>.json` under
|
|
1431
|
+
`base`, its root unless named: a repository that ships one harness's layer keeps it in
|
|
1432
|
+
that harness's own directory. Each harness takes what its adapter declares. Only text
|
|
1433
|
+
is borrowed and hooks never are, and both are named; what a harness does not honour,
|
|
1434
|
+
such as an inclusion only its IDE reads, is written as it is and named unsupported,
|
|
1435
|
+
never faked. Every write is checked before any is made, one refusal writes nothing,
|
|
1436
|
+
and a dry run writes nothing at all. Each written file's digest is recorded in
|
|
1437
|
+
.polyctl/content.json.
|
|
1438
|
+
|
|
1439
|
+
Returns {"items", "hooks", "refused"}: each item {kind, name, files: {target: (harness,
|
|
1440
|
+
bytes)}, skipped: [(path, why)], unsupported: [(target, why)], notes: [text]}, each hook
|
|
1441
|
+
(path, why), and each refusal a sentence.
|
|
1442
|
+
"""
|
|
1443
|
+
import tempfile
|
|
1444
|
+
for kind, names in (("skills", skills), ("steering", steering), ("agents", agents)):
|
|
1445
|
+
for name in names:
|
|
1446
|
+
if not CONTENT_NAME.fullmatch(name):
|
|
1447
|
+
raise ValueError(f"{name!r} is not a {dict(CONTENT_KINDS)[kind]} name")
|
|
1448
|
+
base = str(base or "").strip("/")
|
|
1449
|
+
if base and not _plain_steps(base):
|
|
1450
|
+
raise ValueError(f"{base!r} is not a directory inside the source: names joined by `/`, with no `.` or `..`")
|
|
1451
|
+
prefix = f"{base}/" if base else ""
|
|
1452
|
+
declared = {} # harness: (its directives, the package's directives for it, has it an adapter)
|
|
1453
|
+
for harness in harnesses:
|
|
1454
|
+
adapter = load_adapter(harness, root)
|
|
1455
|
+
declared[harness] = (_declared(adapter.get("directives")),
|
|
1456
|
+
_declared((package_adapters().get(harness) or {}).get("directives")), bool(adapter))
|
|
1457
|
+
takes_agents = [harness for harness, (directives, _, _) in declared.items() if directives.get("agents_dir")
|
|
1458
|
+
and _declared(directives.get("agent_format")).get("extension") == ".json"]
|
|
1459
|
+
wanted = [f"{prefix}skills/{name}" for name in skills] + [f"{prefix}steering/{name}.md" for name in steering]
|
|
1460
|
+
wanted += [f"{prefix}agents/{name}.json" for name in agents] if takes_agents else []
|
|
1461
|
+
wanted += [f"{prefix}hooks"] if steering or agents else []
|
|
1462
|
+
where = f" at {pin[:12]}" + (f" in {base}" if base else "")
|
|
1463
|
+
with tempfile.TemporaryDirectory(prefix="polyctl-content-") as work:
|
|
1464
|
+
tree = fetch_pinned(source, pin, wanted, work)
|
|
1465
|
+
missing = [f"{name} is not a skill{where}" for name in skills
|
|
1466
|
+
if not any(path.startswith(f"{prefix}skills/{name}/") for path in tree)]
|
|
1467
|
+
missing += [f"{name} is not a steering file{where}" for name in steering
|
|
1468
|
+
if f"{prefix}steering/{name}.md" not in tree]
|
|
1469
|
+
missing += [f"{name} is not an agent{where}" for name in agents
|
|
1470
|
+
if takes_agents and f"{prefix}agents/{name}.json" not in tree]
|
|
1471
|
+
if missing:
|
|
1472
|
+
raise ValueError("; ".join(missing))
|
|
1473
|
+
blobs = pinned_blobs(work, [oid for path, (mode, oid) in tree.items()
|
|
1474
|
+
if not path.startswith(f"{prefix}hooks/") and not _not_borrowed(path, mode)])
|
|
1475
|
+
items = [_skill_item(name, f"{prefix}skills/{name}/", tree, blobs, declared) for name in skills]
|
|
1476
|
+
items += [_steering_item(name, f"{prefix}steering/{name}.md", tree, blobs, declared) for name in steering]
|
|
1477
|
+
items += [_agent_item(name, f"{prefix}agents/{name}.json", tree, blobs, declared, takes_agents) for name in agents]
|
|
1478
|
+
for item in items:
|
|
1479
|
+
_unsupported_inclusion(item, declared)
|
|
1480
|
+
hooks = []
|
|
1481
|
+
for path in sorted(path for path in tree if path.startswith(f"{prefix}hooks/")):
|
|
1482
|
+
rel, why = path[len(prefix):], ["hooks are never imported"]
|
|
1483
|
+
for harness, (directives, _, _) in declared.items():
|
|
1484
|
+
formats = _declared(directives.get("hook_files"))
|
|
1485
|
+
suffix = max((suffix for suffix in formats if rel.endswith(suffix)), key=len, default=None)
|
|
1486
|
+
if suffix:
|
|
1487
|
+
why.append(f"{harness}: {formats[suffix]}")
|
|
1488
|
+
hooks.append((rel, "; ".join(why)))
|
|
1489
|
+
plan = {"items": items, "hooks": hooks, "refused": _content_refusals(root, items)}
|
|
1490
|
+
if plan["refused"] or dry_run:
|
|
1491
|
+
return plan
|
|
1492
|
+
|
|
1493
|
+
from .storage import replace_file_durably
|
|
1494
|
+
record = content_manifest(root)
|
|
1495
|
+
for item in items:
|
|
1496
|
+
key, digests = dict(CONTENT_KINDS)[item["kind"]], {}
|
|
1497
|
+
for target, (_, data) in sorted(item["files"].items()):
|
|
1498
|
+
replace_file_durably(os.path.join(root, *target.split("/")), data)
|
|
1499
|
+
digests[target] = "sha256:" + hashlib.sha256(data).hexdigest()
|
|
1500
|
+
record[item["kind"]] = [entry for entry in record[item["kind"]] if entry.get(key) != item["name"]]
|
|
1501
|
+
record[item["kind"]].append({key: item["name"], "source": source, "pin": pin, "files": digests,
|
|
1502
|
+
"skipped": [f"{path} ({why})" for path, why in item["skipped"]]})
|
|
1503
|
+
replace_file_durably(content_manifest_path(root), (json.dumps(record, indent=1, sort_keys=True) + "\n").encode(UTF8))
|
|
1504
|
+
return plan
|
|
1505
|
+
|
|
1506
|
+
|
|
1507
|
+
def verify_content(root):
|
|
1508
|
+
"""Vendored files whose bytes no longer match the digest they were vendored with (#14, #15)."""
|
|
1509
|
+
drift = []
|
|
1510
|
+
for kind, entries in content_manifest(root).items():
|
|
1511
|
+
key = dict(CONTENT_KINDS)[kind]
|
|
1512
|
+
for entry in entries:
|
|
1513
|
+
# A skill is named as it always was; the kinds borrowed since say which kind they are.
|
|
1514
|
+
label = f"{'' if kind == 'skills' else key + ' '}{entry.get(key)}@{str(entry.get('pin') or '')[:12]}"
|
|
1515
|
+
for target, digest in sorted((entry.get("files") or {}).items()):
|
|
1516
|
+
try:
|
|
1517
|
+
with open(os.path.join(root, target), "rb") as fh:
|
|
1518
|
+
actual = "sha256:" + hashlib.sha256(fh.read()).hexdigest()
|
|
1519
|
+
except OSError:
|
|
1520
|
+
drift.append(f"{target} ({label}) is missing")
|
|
1521
|
+
continue
|
|
1522
|
+
if actual != digest:
|
|
1523
|
+
drift.append(f"{target} ({label}) changed since it was vendored")
|
|
1524
|
+
return drift
|
|
1525
|
+
|
|
1526
|
+
|
|
1527
|
+
_UNSAFE_HOOK = re.compile(r"\b(curl|wget)\b[^|;&]*\|\s*(sudo\s+)?(ba|z)?sh\b|\brm\s+-rf\s+(/|~|\$HOME)(\s|$)")
|
|
1528
|
+
_UNPINNED_RUNNER = ("npx", "bunx", "uvx", "pipx")
|
|
1529
|
+
|
|
1530
|
+
|
|
1531
|
+
def agent_config_files(root):
|
|
1532
|
+
"""Project files that configure an agent, as every adapter declares them, and AGENTS.md (#76)."""
|
|
1533
|
+
files = []
|
|
1534
|
+
|
|
1535
|
+
def add(rel):
|
|
1536
|
+
rel = str(rel or "")
|
|
1537
|
+
if rel and not rel.startswith("~") and not os.path.isabs(rel) and rel not in files:
|
|
1538
|
+
files.append(rel)
|
|
1539
|
+
|
|
1540
|
+
for _, entry in sorted(adapter_catalog(root).items()):
|
|
1541
|
+
adapter = entry["adapter"]
|
|
1542
|
+
directives = adapter.get("directives") or {}
|
|
1543
|
+
for rel in (directives.get("rule_files") or []) + (directives.get("steering_files") or []) \
|
|
1544
|
+
+ ((directives.get("command_hooks") or {}).get("files") or []):
|
|
1545
|
+
add(rel)
|
|
1546
|
+
add((adapter.get("mcp") or {}).get("file"))
|
|
1547
|
+
steering = directives.get("steering_dir")
|
|
1548
|
+
if steering and os.path.isdir(os.path.join(root, *steering.split("/"))):
|
|
1549
|
+
for name in sorted(os.listdir(os.path.join(root, *steering.split("/")))):
|
|
1550
|
+
if name.endswith(".md"):
|
|
1551
|
+
add(f"{steering}/{name}")
|
|
1552
|
+
add("AGENTS.md")
|
|
1553
|
+
return files
|
|
1554
|
+
|
|
1555
|
+
|
|
1556
|
+
def agent_config_findings(root):
|
|
1557
|
+
"""AgentShield's check categories over a project's agent configuration, ported natively (#14).
|
|
1558
|
+
|
|
1559
|
+
Secrets in agent files, allow rules that admit everything, permissions with no
|
|
1560
|
+
deny list, hooks that pipe a download into a shell or remove the home directory,
|
|
1561
|
+
and MCP servers run from an unpinned package. Only allow rules are scored, so a
|
|
1562
|
+
deny rule naming `--no-verify` is not a finding, and `env -u VAR` is not read as
|
|
1563
|
+
dumping the environment (the known false positives in docs/upstream.md).
|
|
1564
|
+
Returns [(category, text)].
|
|
1565
|
+
"""
|
|
1566
|
+
out = []
|
|
1567
|
+
files = agent_config_files(root)
|
|
1568
|
+
for rel in files:
|
|
1569
|
+
try:
|
|
1570
|
+
with open(os.path.join(root, rel), encoding=UTF8, errors="replace") as fh:
|
|
1571
|
+
text = fh.read()
|
|
1572
|
+
except OSError:
|
|
1573
|
+
continue
|
|
1574
|
+
hits = scan_evidence(text)[1]
|
|
1575
|
+
if hits:
|
|
1576
|
+
out.append(("secrets", f"{rel} holds what looks like a credential: {', '.join(sorted(set(hits)))}"))
|
|
1577
|
+
if not rel.endswith(".json"):
|
|
1578
|
+
continue
|
|
1579
|
+
try:
|
|
1580
|
+
document = json.loads(text)
|
|
1581
|
+
except ValueError:
|
|
1582
|
+
continue
|
|
1583
|
+
permissions = document.get("permissions") if isinstance(document, dict) else None
|
|
1584
|
+
if isinstance(permissions, dict):
|
|
1585
|
+
broad = [rule for rule in permissions.get("allow") or [] if str(rule).strip() in ("*", "Bash", "Bash(*)", "Bash(*:*)")]
|
|
1586
|
+
if broad:
|
|
1587
|
+
out.append(("permissive-allow", f"{rel} allows {', '.join(broad)}: every command, a hook bypass and a push "
|
|
1588
|
+
"among them"))
|
|
1589
|
+
if permissions.get("allow") and not permissions.get("deny"):
|
|
1590
|
+
out.append(("missing-deny", f"{rel} allows tools and denies nothing"))
|
|
1591
|
+
hooks = document.get("hooks") if isinstance(document, dict) else None
|
|
1592
|
+
for entries in (hooks.values() if isinstance(hooks, dict) else []):
|
|
1593
|
+
for entry in entries if isinstance(entries, list) else []:
|
|
1594
|
+
for hook in (entry.get("hooks") or []) if isinstance(entry, dict) else []:
|
|
1595
|
+
command = str((hook or {}).get("command") or "") if isinstance(hook, dict) else ""
|
|
1596
|
+
if _UNSAFE_HOOK.search(command):
|
|
1597
|
+
out.append(("hook-safety", f"{rel} runs a hook that pipes a download into a shell or removes "
|
|
1598
|
+
f"a home: {command[:80]}"))
|
|
1599
|
+
servers = document.get("mcpServers") if isinstance(document, dict) else None
|
|
1600
|
+
for name, server in (servers.items() if isinstance(servers, dict) else []):
|
|
1601
|
+
command = os.path.basename(str((server or {}).get("command") or ""))
|
|
1602
|
+
args = [str(arg) for arg in (server or {}).get("args") or []]
|
|
1603
|
+
packages = [arg for arg in args if not arg.startswith("-")]
|
|
1604
|
+
if command in _UNPINNED_RUNNER and packages and not re.search(r".@\d", packages[0]):
|
|
1605
|
+
out.append(("mcp-hygiene", f"{rel} runs MCP server {name} from {command} {packages[0]} with no "
|
|
1606
|
+
"version pinned"))
|
|
1607
|
+
return out
|
|
1608
|
+
|
|
1609
|
+
|
|
1610
|
+
# ---- asking the codebase a question is a capability, not a copy (#81) --------------------
|
|
1611
|
+
|
|
1612
|
+
# The folder of the copy of the working tree that `polyctl ask --codebase` runs its provider in, and names {root}
|
|
1613
|
+
# (CODEBASE-COPY).
|
|
1614
|
+
CODEBASE_TREE_DIR = "tree"
|
|
1615
|
+
|
|
1616
|
+
|
|
1617
|
+
def ask_codebase(root, cfg, question, timeout=300):
|
|
1618
|
+
"""Answer a question about the code through a configured provider, never without citations (#81).
|
|
1619
|
+
|
|
1620
|
+
polyctl cannot embed a code-intelligence engine without breaking `dependencies = []`,
|
|
1621
|
+
and should not reimplement one. A provider is a command in `codebase.provider`
|
|
1622
|
+
(argv with {question} and {root}) that prints JSON: {"answer": text, "citations":
|
|
1623
|
+
[{"file": path, "lines": "a-b"}]}. With none configured, or an answer that cites
|
|
1624
|
+
nothing, the question is refused and what would satisfy it is named.
|
|
1625
|
+
Returns {"answer", "citations"} or raises ValueError.
|
|
1626
|
+
|
|
1627
|
+
The provider runs in a copy of the working tree, never in the tree itself (CODEBASE-COPY). It ran in the
|
|
1628
|
+
repository, so a provider that is a harness read the tree's own configuration - the hooks, plugins and MCP servers
|
|
1629
|
+
of .claude/settings.json, opencode.json and the like, files an agent writes - and ran them before any review had
|
|
1630
|
+
seen them. The tree as it stands, uncommitted changes and untracked files in and ignored files out, is written into
|
|
1631
|
+
a tree object through an index of the copy's own and unpacked outside the repository, less what any harness reads
|
|
1632
|
+
from a project, as a review's candidate is (REVIEW-TREE-3); {root} names that copy and the provider runs in it.
|
|
1633
|
+
Citations are read against the copy, and the copy is removed whatever happens: an answer, a refusal, an error or
|
|
1634
|
+
a timeout.
|
|
1635
|
+
"""
|
|
1636
|
+
import shutil
|
|
1637
|
+
import tempfile
|
|
1638
|
+
provider = (cfg.get("codebase") or {}).get("provider") or {}
|
|
1639
|
+
argv = provider.get("argv") if isinstance(provider, dict) else None
|
|
1640
|
+
if not argv:
|
|
1641
|
+
raise ValueError("no codebase provider is configured: set codebase.provider.argv to a command that takes "
|
|
1642
|
+
"{question} and {root} and prints {\"answer\", \"citations\"} as JSON (ctxman is the first "
|
|
1643
|
+
"provider documented in docs/upstream.md)")
|
|
1644
|
+
# Found before anything is copied: a provider named without a directory is the one PATH holds, never one the
|
|
1645
|
+
# current directory holds, which Windows looks in first (WINDOWS-NO-CWD-EXEC); one named by a path is that file.
|
|
1646
|
+
program = str(argv[0]).replace("{question}", question)
|
|
1647
|
+
if not os.path.dirname(program):
|
|
1648
|
+
found = program_path(program)
|
|
1649
|
+
if found is None:
|
|
1650
|
+
raise ValueError(f"the codebase provider {program} is not on PATH")
|
|
1651
|
+
program = found
|
|
1652
|
+
copy = tempfile.mkdtemp(prefix="polyctl-codebase-")
|
|
1653
|
+
try:
|
|
1654
|
+
if _within(copy, root):
|
|
1655
|
+
raise ValueError(f"the temporary directory {os.path.dirname(copy)} lies inside this repository, where a "
|
|
1656
|
+
"copy of the tree would be read as the repository itself: set TMPDIR to a directory "
|
|
1657
|
+
"outside it and ask again")
|
|
1658
|
+
try:
|
|
1659
|
+
snapshot = working_tree_snapshot(root, os.path.join(copy, "index"))
|
|
1660
|
+
except RuntimeError as exc:
|
|
1661
|
+
raise ValueError(f"the working tree could not be copied for the provider ({exc}); a provider runs in a "
|
|
1662
|
+
"copy of the tree, never in the repository") from exc
|
|
1663
|
+
tree = unpack_tree(root, snapshot, os.path.join(copy, CODEBASE_TREE_DIR))
|
|
1664
|
+
if tree is None:
|
|
1665
|
+
raise ValueError("the working tree could not be written out for the provider; a provider runs in a copy "
|
|
1666
|
+
"of the tree, never in the repository")
|
|
1667
|
+
rendered = [program] + [str(part).replace("{question}", question).replace("{root}", tree) for part in argv[1:]]
|
|
1668
|
+
code, stdout = _run_codebase_provider(rendered, tree, timeout)
|
|
1669
|
+
try:
|
|
1670
|
+
document = json.loads(stdout)
|
|
1671
|
+
except ValueError as exc:
|
|
1672
|
+
raise ValueError(f"the codebase provider did not answer in JSON (exit {code})") from exc
|
|
1673
|
+
answer = str((document or {}).get("answer") or "").strip() if isinstance(document, dict) else ""
|
|
1674
|
+
citations = [item for item in (document.get("citations") or [])
|
|
1675
|
+
if isinstance(item, dict) and item.get("file")] if isinstance(document, dict) else []
|
|
1676
|
+
if not answer or not citations:
|
|
1677
|
+
raise ValueError("the provider's answer cites no file and line range; an answer without citations is not "
|
|
1678
|
+
"an answer polyctl passes on")
|
|
1679
|
+
cited = [_cited_in_copy(tree, item["file"]) for item in citations]
|
|
1680
|
+
missing = [str(item["file"]) for item, place in zip(citations, cited) if place is None]
|
|
1681
|
+
if missing:
|
|
1682
|
+
withheld = sorted({project_config_member(name) for name in missing} - {None}, key=str.casefold)
|
|
1683
|
+
raise ValueError(f"the answer cites files that are not in this repository: {', '.join(missing[:3])}"
|
|
1684
|
+
+ (f". A provider is handed the tree without what a harness reads from a project as its "
|
|
1685
|
+
f"configuration or instructions ({', '.join(withheld)}); read those in the repository "
|
|
1686
|
+
"yourself" if withheld else ""))
|
|
1687
|
+
# The provider was told {root} is the copy; the person reads the repository's paths, the copy being gone.
|
|
1688
|
+
for spelling in sorted({tree, os.path.realpath(tree)}, key=len, reverse=True):
|
|
1689
|
+
answer = answer.replace(spelling, root)
|
|
1690
|
+
return {"answer": scan_evidence(answer)[0],
|
|
1691
|
+
"citations": [dict(item, file=place) for item, place in zip(citations, cited)]}
|
|
1692
|
+
finally:
|
|
1693
|
+
shutil.rmtree(copy, ignore_errors=True)
|
|
1694
|
+
|
|
1695
|
+
|
|
1696
|
+
def _cited_in_copy(tree, name):
|
|
1697
|
+
"""The path a citation names, relative to the copy of the tree, when the copy holds it; else None (CODEBASE-COPY).
|
|
1698
|
+
|
|
1699
|
+
A provider handed {root} may cite a file under it in full, which is read as the path below it. One that climbs out
|
|
1700
|
+
of the copy, or that the copy does not hold, is no file of this repository's: the copy is all the provider had.
|
|
1701
|
+
"""
|
|
1702
|
+
name = str(name)
|
|
1703
|
+
full = os.path.join(tree, name)
|
|
1704
|
+
if not name or not _lands_inside(tree, name) or not os.path.exists(full):
|
|
1705
|
+
return None
|
|
1706
|
+
return os.path.relpath(os.path.realpath(full), os.path.realpath(tree)).replace(os.sep, "/")
|
|
1707
|
+
|
|
1708
|
+
|
|
1709
|
+
def _run_codebase_provider(argv, cwd, timeout):
|
|
1710
|
+
"""(exit code, standard output) of a codebase provider run in `cwd`; ValueError when it does not start, or does not
|
|
1711
|
+
answer within `timeout` (CODEBASE-COPY).
|
|
1712
|
+
|
|
1713
|
+
It leads a process group of its own and is handed no git variable, as a reviewer is (#65): a timeout stops whatever
|
|
1714
|
+
it started, which would otherwise go on reading the copy - and on Windows keep it from being removed - and a git
|
|
1715
|
+
command it runs finds no repository above the copy.
|
|
1716
|
+
"""
|
|
1717
|
+
import signal
|
|
1718
|
+
env = {name: value for name, value in os.environ.items() if not name.upper().startswith("GIT_")}
|
|
1719
|
+
env.update(PWD=cwd, GIT_CEILING_DIRECTORIES=os.path.dirname(cwd))
|
|
1720
|
+
group = ({"creationflags": getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)} if os.name == "nt"
|
|
1721
|
+
else {"start_new_session": True})
|
|
1722
|
+
try:
|
|
1723
|
+
proc = subprocess.Popen(argv, cwd=cwd, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
|
1724
|
+
stderr=subprocess.PIPE, text=True, encoding=UTF8, errors="replace", **group)
|
|
1725
|
+
except OSError as exc:
|
|
1726
|
+
raise ValueError(f"the codebase provider did not run: {exc}") from exc
|
|
1727
|
+
try:
|
|
1728
|
+
stdout, _ = proc.communicate(timeout=timeout)
|
|
1729
|
+
except subprocess.TimeoutExpired:
|
|
1730
|
+
left = kill_turn(proc.pid, getattr(signal, "SIGKILL", signal.SIGTERM)) or []
|
|
1731
|
+
try:
|
|
1732
|
+
proc.kill()
|
|
1733
|
+
proc.communicate(timeout=5)
|
|
1734
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
1735
|
+
for stream in (proc.stdout, proc.stderr):
|
|
1736
|
+
try:
|
|
1737
|
+
stream.close()
|
|
1738
|
+
except OSError:
|
|
1739
|
+
pass
|
|
1740
|
+
if left:
|
|
1741
|
+
# It said whatever the provider started was stopped, whatever the stop did (WINDOWS-STOP-PROVEN).
|
|
1742
|
+
raise ValueError(f"the codebase provider did not answer within {timeout}s, and nothing was answered: it "
|
|
1743
|
+
f"was stopped, and {len(left)} process(es) it started still run: {left} - stop them by "
|
|
1744
|
+
"hand") from None
|
|
1745
|
+
raise ValueError(f"the codebase provider did not answer within {timeout}s: it was stopped, with whatever it "
|
|
1746
|
+
"started, and nothing was answered") from None
|
|
1747
|
+
return proc.returncode, stdout
|
|
1748
|
+
|
|
1749
|
+
|
|
1750
|
+
def safe_slug(text, fallback="note", limit=40):
|
|
1751
|
+
"""A file-name part holding only [A-Za-z0-9._-] (#19).
|
|
1752
|
+
|
|
1753
|
+
`polyctl mail send note "polyctl: kiro/* dal …"` kept the "/" and wrote into a directory
|
|
1754
|
+
that did not exist. Separators, wildcards, quotes, whitespace and letters
|
|
1755
|
+
outside ASCII all become a dash; what is left is cut to `limit`.
|
|
1756
|
+
"""
|
|
1757
|
+
slug = re.sub(r"[^A-Za-z0-9._-]+", "-", str(text or "")).strip("-.")[:limit].strip("-.")
|
|
1758
|
+
return slug or fallback
|
|
1759
|
+
|
|
1760
|
+
|
|
1761
|
+
def note(root, cfg, to, title, body, urgent=False):
|
|
1762
|
+
"""Write an architect message into the mailbox through the tool.
|
|
1763
|
+
|
|
1764
|
+
So that a woken architect needs no raw Write or Edit to do its job. That
|
|
1765
|
+
matters because the one time it had them, it used them on the orchestrator's
|
|
1766
|
+
own source and built a runaway. The mailbox is the only thing an unattended
|
|
1767
|
+
architect should be able to write, and this is the only door to it.
|
|
1768
|
+
"""
|
|
1769
|
+
box = os.path.join(root, cfg.get("mailbox", "agent-mail"))
|
|
1770
|
+
os.makedirs(box, exist_ok=True)
|
|
1771
|
+
slug = safe_slug(title.lower(), language.text(cfg, "mail.untitled-note"))
|
|
1772
|
+
# An urgent note is marked in the project's language; every reader knows both (LANGUAGE-FILES).
|
|
1773
|
+
kind = language.marker(cfg, "urgent-kind") if urgent else "DECISION"
|
|
1774
|
+
impl, arch = mail_names(cfg)
|
|
1775
|
+
to = safe_slug(to or impl, impl) # the implementer's role by default, whoever holds it (#31)
|
|
1776
|
+
name = f"{time.strftime('%Y%m%d-%H%M')}-{arch}-to-{to}-{kind}-{slug}.md"
|
|
1777
|
+
text = f"# {title}\n\n" + (language.marker(cfg, "urgent") + "\n\n" if urgent else "") + body.rstrip() + "\n"
|
|
1778
|
+
return write_mail(root, cfg, name, text, {"kind": kind.lower(), "from": arch, "to": to})
|
|
1779
|
+
|
|
1780
|
+
|
|
1781
|
+
def running_slice(root):
|
|
1782
|
+
"""The one board item review accounting treats as current.
|
|
1783
|
+
|
|
1784
|
+
A malformed board may contain several running entries. Preserve the board's
|
|
1785
|
+
established first-entry behavior, but resolve it once so review attribution,
|
|
1786
|
+
start time and re-specification cannot silently select different slices.
|
|
1787
|
+
"""
|
|
1788
|
+
items = board(root)["running"]
|
|
1789
|
+
return items[0] if items else None
|
|
1790
|
+
|
|
1791
|
+
|
|
1792
|
+
def slice_boundary(item):
|
|
1793
|
+
"""The declared review boundary for a parsed board item."""
|
|
1794
|
+
if not item:
|
|
1795
|
+
return ""
|
|
1796
|
+
notes = item.get("notes") or {}
|
|
1797
|
+
return notes.get("acceptance") or notes.get("scope") or item.get("title") or ""
|
|
1798
|
+
|
|
1799
|
+
|
|
1800
|
+
# ---- a slice's boundary: a sentence, or a file the row points at (#73, #35) ------------
|
|
1801
|
+
|
|
1802
|
+
BOUNDARY_SECTIONS = ("invariant", "scenarios", "paths", "out of scope", "why one slice")
|
|
1803
|
+
_BOUNDARY_COMMIT = re.compile(r"[0-9a-fA-F]{7,40}")
|
|
1804
|
+
# A declared path the slice creates, marked `(new)` in any language a project may choose (LANGUAGE-FILES).
|
|
1805
|
+
_NEW_PATH = re.compile(r"\s*\((?:" + "|".join(re.escape(word) for word in language.words("new-path")) + r")\)$",
|
|
1806
|
+
re.I)
|
|
1807
|
+
_NAMED_FILE = re.compile(r"(?<![\w/.-])((?:[\w.-]+/)+[\w.-]+\.\w+|[\w-]+\.(?:py|ts|tsx|js|jsx|mjs|cjs|go|rs|java|kt|"
|
|
1808
|
+
r"rb|cs|swift|c|h|cpp|hpp|sql|sh|ps1|json|ya?ml|toml|md))(?![\w/-])")
|
|
1809
|
+
_NAMED_SYMBOL = re.compile(r"`([A-Za-z_][A-Za-z0-9_]{2,})(?:\(\))?`")
|
|
1810
|
+
|
|
1811
|
+
|
|
1812
|
+
def boundary_pointer(item):
|
|
1813
|
+
"""(path, commit or None) when a board item's boundary is a file: `boundary: docs/slices/B8a.md@1a2b3c4`."""
|
|
1814
|
+
value = (((item or {}).get("notes") or {}).get("boundary") or "").strip()
|
|
1815
|
+
if not value:
|
|
1816
|
+
return None
|
|
1817
|
+
path, _, commit = value.partition("@")
|
|
1818
|
+
return path.strip(), (commit.strip() or None)
|
|
1819
|
+
|
|
1820
|
+
|
|
1821
|
+
def boundary_sections(text):
|
|
1822
|
+
"""{section: body} for the headed parts of a boundary file (#73)."""
|
|
1823
|
+
sections, current = {}, None
|
|
1824
|
+
for line in text.splitlines():
|
|
1825
|
+
heading = re.match(r"^#{1,4}\s+(.+?)\s*$", line)
|
|
1826
|
+
if heading:
|
|
1827
|
+
name = heading.group(1).strip().lower()
|
|
1828
|
+
current = next((section for section in BOUNDARY_SECTIONS if section in name), None)
|
|
1829
|
+
if current:
|
|
1830
|
+
sections.setdefault(current, "")
|
|
1831
|
+
continue
|
|
1832
|
+
if current:
|
|
1833
|
+
sections[current] += line + "\n"
|
|
1834
|
+
return sections
|
|
1835
|
+
|
|
1836
|
+
|
|
1837
|
+
def read_boundary(root, item):
|
|
1838
|
+
"""The boundary file a slice points at, read at the commit the pointer names (#73).
|
|
1839
|
+
|
|
1840
|
+
The boundary lived as prose in a table cell, and twice on 2026-09-07/08 the
|
|
1841
|
+
boundary itself was what was wrong, found mid-slice by an implementer that had
|
|
1842
|
+
started, with no diff to show what changed. A file read at a named commit is
|
|
1843
|
+
one source of truth, and when it has changed since, the change travels to the
|
|
1844
|
+
reviewer as a diff. None when the item's boundary is a sentence.
|
|
1845
|
+
|
|
1846
|
+
Returns {"file", "commit", "changed", "sha256", "label", "text", "sections", "problem"}.
|
|
1847
|
+
"""
|
|
1848
|
+
pointer = boundary_pointer(item)
|
|
1849
|
+
if not pointer:
|
|
1850
|
+
return None
|
|
1851
|
+
path, commit = pointer
|
|
1852
|
+
out = {"file": path, "commit": commit, "changed": None, "sha256": None, "sections": {}, "problem": None}
|
|
1853
|
+
real_root = os.path.realpath(root)
|
|
1854
|
+
target = os.path.realpath(os.path.join(root, path))
|
|
1855
|
+
if os.path.isabs(path) or not target.startswith(real_root + os.sep):
|
|
1856
|
+
out["problem"] = "it is not a path inside the repository"
|
|
1857
|
+
elif commit and not _BOUNDARY_COMMIT.fullmatch(commit):
|
|
1858
|
+
out["problem"] = f"{commit!r} is not a commit id"
|
|
1859
|
+
else:
|
|
1860
|
+
try:
|
|
1861
|
+
if commit:
|
|
1862
|
+
data = _git_output(root, "show", f"{commit}:{path}")
|
|
1863
|
+
else:
|
|
1864
|
+
with open(target, "rb") as fh:
|
|
1865
|
+
data = fh.read()
|
|
1866
|
+
except (OSError, RuntimeError) as exc:
|
|
1867
|
+
out["problem"] = str(exc)
|
|
1868
|
+
if out["problem"]:
|
|
1869
|
+
out.update(label=f"{path} (unreadable)",
|
|
1870
|
+
text=f"the boundary file {path} cannot be read: {out['problem']} - say so as a finding")
|
|
1871
|
+
return out
|
|
1872
|
+
# Keep readable contents for a deferred review, even when no commit makes them authoritative.
|
|
1873
|
+
if not commit:
|
|
1874
|
+
out["problem"] = "it names no commit; use boundary: path@commit"
|
|
1875
|
+
content = data.decode(UTF8, "replace")
|
|
1876
|
+
diff = ""
|
|
1877
|
+
if commit:
|
|
1878
|
+
try:
|
|
1879
|
+
diff = _git_output(root, "diff", commit, "--", path).decode(UTF8, "replace")
|
|
1880
|
+
except RuntimeError as exc:
|
|
1881
|
+
out["problem"] = f"cannot compare the boundary with its commit: {exc}"
|
|
1882
|
+
label = f"{path} at {commit[:12]}" if commit else f"{path} as it stands in the worktree"
|
|
1883
|
+
text = f"the boundary file {label}:\n\n{content}"
|
|
1884
|
+
if diff.strip():
|
|
1885
|
+
label += ", changed since"
|
|
1886
|
+
text += (f"\n\nThe file has changed since {commit[:12]}. The change is part of what you judge: "
|
|
1887
|
+
f"a boundary that moved mid-slice is a finding unless a decision records it.\n{diff}")
|
|
1888
|
+
if out["problem"]:
|
|
1889
|
+
text += f"\n\n{out['problem']} - say so as a finding"
|
|
1890
|
+
out.update(changed=None if out["problem"] else bool(diff.strip()),
|
|
1891
|
+
sha256="sha256:" + hashlib.sha256(data).hexdigest() if commit else None, label=label,
|
|
1892
|
+
text=text, sections=boundary_sections(content))
|
|
1893
|
+
return out
|
|
1894
|
+
|
|
1895
|
+
|
|
1896
|
+
# A boundary's numbered line: each is one of its scenarios to review_sections, and one of its criteria here.
|
|
1897
|
+
BOUNDARY_NUMBERED_LINE = re.compile(r"^\s*S?(\d{1,2})[.)]\s+(\S.{6,})$", re.M)
|
|
1898
|
+
# A number inside one line - `1.`, `1)` or `(1)` - standing between spaces.
|
|
1899
|
+
_INLINE_CRITERION = re.compile(r"(?<!\S)\(?(\d{1,2})[.)](?=\s)")
|
|
1900
|
+
# A semicolon outside backticks: the one in `polyctl review --boundary 'a; b'` separates nothing.
|
|
1901
|
+
_CRITERIA_SEMICOLON = re.compile(r";(?=(?:[^`]*`[^`]*`)*[^`]*$)")
|
|
1902
|
+
# The parts of a boundary file that say what must hold; its paths, out of scope and why-one-slice do not.
|
|
1903
|
+
CRITERIA_SECTIONS = ("invariant", "scenarios")
|
|
1904
|
+
|
|
1905
|
+
|
|
1906
|
+
def boundary_criteria(boundary, source=None):
|
|
1907
|
+
"""[{"id", "text"}] for each criterion an acceptance boundary lists; [] when it lists fewer than two.
|
|
1908
|
+
|
|
1909
|
+
One verdict for a whole boundary let a boundary of five conditions pass with one of
|
|
1910
|
+
them never examined, and nothing on the record said which (CRITERIA-VERDICTS). The
|
|
1911
|
+
criteria are the boundary's numbered lines. A one-line boundary, which a board row's
|
|
1912
|
+
acceptance always is, lists them with inline numbers counting from 1 - `1) … 2) …` -
|
|
1913
|
+
or separated by semicolons. A boundary file (`source`, from read_boundary) lists them
|
|
1914
|
+
in its Invariant and Scenarios sections as polyctl read the file, at the commit its row
|
|
1915
|
+
names; never in the diff of a later change, nor in what the file puts out of scope. A
|
|
1916
|
+
single sentence lists none, and is reviewed as it always was. The criteria are numbered
|
|
1917
|
+
by position, whatever numbers the boundary wrote, so no two share a name.
|
|
1918
|
+
"""
|
|
1919
|
+
if source:
|
|
1920
|
+
sections = source.get("sections") or {}
|
|
1921
|
+
text = "\n".join(body for name, body in sections.items() if name in CRITERIA_SECTIONS)
|
|
1922
|
+
else:
|
|
1923
|
+
text = boundary
|
|
1924
|
+
text = str(text or "").strip()
|
|
1925
|
+
items = [body for _, body in BOUNDARY_NUMBERED_LINE.findall(text)]
|
|
1926
|
+
if len(items) < 2 and "\n" not in text:
|
|
1927
|
+
marks = list(_INLINE_CRITERION.finditer(text))
|
|
1928
|
+
if len(marks) >= 2 and [int(mark.group(1)) for mark in marks] == list(range(1, len(marks) + 1)):
|
|
1929
|
+
ends = [mark.start() for mark in marks[1:]] + [len(text)]
|
|
1930
|
+
items = [text[mark.end():end] for mark, end in zip(marks, ends)]
|
|
1931
|
+
else:
|
|
1932
|
+
items = _CRITERIA_SEMICOLON.split(text)
|
|
1933
|
+
items = [item.strip().rstrip(";,").strip() for item in items]
|
|
1934
|
+
items = [item for item in items if item]
|
|
1935
|
+
return [{"id": number, "text": item} for number, item in enumerate(items, 1)] if len(items) >= 2 else []
|
|
1936
|
+
|
|
1937
|
+
|
|
1938
|
+
def declared_paths(item, boundary=None):
|
|
1939
|
+
"""[(path, new)] a slice declares: its boundary file's Paths section, or its row's `paths:` note."""
|
|
1940
|
+
if boundary and boundary.get("sections", {}).get("paths"):
|
|
1941
|
+
raw = [line.strip().lstrip("-*").strip() for line in boundary["sections"]["paths"].splitlines()]
|
|
1942
|
+
else:
|
|
1943
|
+
raw = (((item or {}).get("notes") or {}).get("paths") or "").split(",")
|
|
1944
|
+
out = []
|
|
1945
|
+
for entry in raw:
|
|
1946
|
+
entry = entry.strip().strip("`").strip()
|
|
1947
|
+
new = bool(_NEW_PATH.search(entry))
|
|
1948
|
+
path = _NEW_PATH.sub("", entry).strip().strip("`").strip()
|
|
1949
|
+
if path:
|
|
1950
|
+
out.append((path, new))
|
|
1951
|
+
return out
|
|
1952
|
+
|
|
1953
|
+
|
|
1954
|
+
def _symbol_owners(root, symbol):
|
|
1955
|
+
"""Tracked files that define a symbol, by a definition keyword in front of its name."""
|
|
1956
|
+
pattern = rf"(def|class|function|interface|type|struct|enum|trait|fn|func|const|let|var)[[:space:]]+{symbol}"
|
|
1957
|
+
try:
|
|
1958
|
+
listed = _git_output(root, "grep", "-l", "-z", "-w", "-E", pattern, timeout=20)
|
|
1959
|
+
except RuntimeError:
|
|
1960
|
+
return []
|
|
1961
|
+
return [os.fsdecode(path) for path in listed.split(b"\0") if path]
|
|
1962
|
+
|
|
1963
|
+
|
|
1964
|
+
def boundary_conflicts(root, item):
|
|
1965
|
+
"""What a slice's declared paths cannot hold, found when it is registered (#35).
|
|
1966
|
+
|
|
1967
|
+
2026-09-07: B8a named eight paths while its own acceptance could only be met
|
|
1968
|
+
by touching a ninth; the implementer found out mid-slice and lost three hours
|
|
1969
|
+
waiting on a decision. Advisory prose matching, so it names and never blocks:
|
|
1970
|
+
a declared path that neither exists nor is marked `(new)`, a file the
|
|
1971
|
+
acceptance names outside the declared paths, and the file defining a symbol
|
|
1972
|
+
the acceptance names in backticks when that file is outside them. An item
|
|
1973
|
+
that declares no paths has nothing to check.
|
|
1974
|
+
"""
|
|
1975
|
+
source = read_boundary(root, item)
|
|
1976
|
+
if source and source["problem"]:
|
|
1977
|
+
return [f"its boundary file {source['file']} cannot be read: {source['problem']}"]
|
|
1978
|
+
declared = declared_paths(item, source)
|
|
1979
|
+
if not declared:
|
|
1980
|
+
return []
|
|
1981
|
+
|
|
1982
|
+
def inside(path):
|
|
1983
|
+
return any(path == d or path.startswith(d.rstrip("/") + "/") for d, _ in declared)
|
|
1984
|
+
|
|
1985
|
+
out = [f"{path} is declared but does not exist; write `(new)` after it if the slice creates it"
|
|
1986
|
+
for path, new in declared if not new and not os.path.exists(os.path.join(root, path))]
|
|
1987
|
+
if source:
|
|
1988
|
+
text = "\n".join(body for name, body in source["sections"].items()
|
|
1989
|
+
if name not in ("paths", "out of scope", "why one slice")) \
|
|
1990
|
+
or source["text"]
|
|
1991
|
+
else:
|
|
1992
|
+
text = slice_boundary(item)
|
|
1993
|
+
tracked = None
|
|
1994
|
+
for named in sorted(set(_NAMED_FILE.findall(text))):
|
|
1995
|
+
if inside(named) or any(d.endswith("/" + named) for d, _ in declared):
|
|
1996
|
+
continue
|
|
1997
|
+
if "/" in named:
|
|
1998
|
+
if os.path.exists(os.path.join(root, named)):
|
|
1999
|
+
out.append(f"the acceptance names {named}, outside the declared paths")
|
|
2000
|
+
continue
|
|
2001
|
+
if tracked is None:
|
|
2002
|
+
try:
|
|
2003
|
+
tracked = [os.fsdecode(p) for p in _git_output(root, "ls-files", "-z").split(b"\0") if p]
|
|
2004
|
+
except RuntimeError:
|
|
2005
|
+
tracked = []
|
|
2006
|
+
owners = [p for p in tracked if os.path.basename(p) == named]
|
|
2007
|
+
if owners and not any(inside(p) for p in owners):
|
|
2008
|
+
out.append(f"the acceptance names {named} ({', '.join(owners[:3])}), outside the declared paths")
|
|
2009
|
+
for symbol in sorted(set(_NAMED_SYMBOL.findall(text)))[:10]:
|
|
2010
|
+
owners = _symbol_owners(root, symbol)
|
|
2011
|
+
if owners and not any(inside(p) for p in owners):
|
|
2012
|
+
out.append(f"the acceptance names `{symbol}`, defined in {', '.join(owners[:3])}, "
|
|
2013
|
+
"outside the declared paths")
|
|
2014
|
+
return out
|
|
2015
|
+
|
|
2016
|
+
|
|
2017
|
+
def boundary_advice(root, cfg, item):
|
|
2018
|
+
"""Everything worth saying about one item's boundary before it is worked (#35, #73)."""
|
|
2019
|
+
notes = (item or {}).get("notes") or {}
|
|
2020
|
+
out = boundary_conflicts(root, item)
|
|
2021
|
+
if notes.get("boundary") and notes.get("acceptance"):
|
|
2022
|
+
out.append("it has both a boundary file and an acceptance sentence; one source of truth, so keep the file")
|
|
2023
|
+
limit = settings.get(cfg, "boundary.inline_max_chars")
|
|
2024
|
+
sentence = notes.get("acceptance") or ""
|
|
2025
|
+
if not notes.get("boundary") and len(sentence) > limit:
|
|
2026
|
+
out.append(f"its acceptance is {len(sentence)} characters; above boundary.inline_max_chars ({limit}) it "
|
|
2027
|
+
"belongs in a file the row points at with `boundary: path@commit`")
|
|
2028
|
+
return out
|
|
2029
|
+
|
|
2030
|
+
|
|
2031
|
+
DECISION_CHAIN = "polyctl-decision-row-v1"
|
|
2032
|
+
|
|
2033
|
+
|
|
2034
|
+
def decisions_path(root):
|
|
2035
|
+
return os.path.join(root, ".polyctl", "ledger", "decisions.jsonl")
|
|
2036
|
+
|
|
2037
|
+
|
|
2038
|
+
def decision_rows(root):
|
|
2039
|
+
"""Architect decisions in the order `polyctl decide` recorded them (#65).
|
|
2040
|
+
|
|
2041
|
+
Rows written before the ledger was chained are read as its legacy prefix. A
|
|
2042
|
+
row added by hand after a chained one breaks the chain and the read raises:
|
|
2043
|
+
a re-specification nobody recorded must not reset a round budget.
|
|
2044
|
+
"""
|
|
2045
|
+
from .storage import read_chained_jsonl
|
|
2046
|
+
return [row for row in read_chained_jsonl(decisions_path(root), DECISION_CHAIN, legacy_prefix=True)
|
|
2047
|
+
if isinstance(row, dict)]
|
|
2048
|
+
|
|
2049
|
+
|
|
2050
|
+
def respecified_at(root, item=None):
|
|
2051
|
+
"""When the architect last re-specified the current slice (`polyctl decide --scope <id>`).
|
|
2052
|
+
|
|
2053
|
+
None when the decision ledger cannot be trusted: the budget then stands.
|
|
2054
|
+
"""
|
|
2055
|
+
item = item or running_slice(root)
|
|
2056
|
+
item_id = ((item or {}).get("id") or (item or {}).get("key") or "").strip()
|
|
2057
|
+
if not item_id:
|
|
2058
|
+
return None
|
|
2059
|
+
try:
|
|
2060
|
+
rows = decision_rows(root)
|
|
2061
|
+
except Exception:
|
|
2062
|
+
return None
|
|
2063
|
+
last = 0
|
|
2064
|
+
for r in rows:
|
|
2065
|
+
if r.get("by") == "architect" and (r.get("scope") or "").strip() == item_id:
|
|
2066
|
+
try:
|
|
2067
|
+
last = max(last, int(r.get("at") or 0))
|
|
2068
|
+
except (TypeError, ValueError):
|
|
2069
|
+
continue
|
|
2070
|
+
return last or None
|
|
2071
|
+
|
|
2072
|
+
|
|
2073
|
+
def _recorded_review_slice(root, reviews_dir, row):
|
|
2074
|
+
"""The slice of a review recorded before ledger rows carried it, from its unchanged file."""
|
|
2075
|
+
try:
|
|
2076
|
+
with open(os.path.join(root, reviews_dir, str(row.get("artefact"))), "rb") as fh:
|
|
2077
|
+
data = fh.read()
|
|
2078
|
+
except OSError:
|
|
2079
|
+
return None
|
|
2080
|
+
if "sha256:" + hashlib.sha256(data).hexdigest() != row.get("sha256"):
|
|
2081
|
+
return None
|
|
2082
|
+
evidence = review_evidence(data.decode(UTF8, "replace"))
|
|
2083
|
+
return (evidence or {}).get("slice") if isinstance(evidence, dict) else None
|
|
2084
|
+
|
|
2085
|
+
|
|
2086
|
+
def rounds(root, reviews_dir, item=None):
|
|
2087
|
+
"""Completed prospective review rounds spent on the current slice (#65), or on board item `item`.
|
|
2088
|
+
|
|
2089
|
+
A round is a review that produced a verdict, read from the chained review
|
|
2090
|
+
ledger in the order polyctl recorded it. Deleting, emptying or back-dating a review
|
|
2091
|
+
file changes nothing, and neither does editing the board's `since:`, which an
|
|
2092
|
+
implementer can reach. Only index-candidate reviews of the running board
|
|
2093
|
+
item's exact ID count - HEAD and candidate bytes are not slice identity. The
|
|
2094
|
+
count runs back to the newest approval that could authorise, or to the
|
|
2095
|
+
architect's newest recorded re-specification of the item. A fallback's
|
|
2096
|
+
approval after a rejection of the same candidate authorises nothing, so it
|
|
2097
|
+
ends nothing: the rejection still counts. UNAVAILABLE and INVALID are not
|
|
2098
|
+
rounds; nobody completed a review.
|
|
2099
|
+
|
|
2100
|
+
Unrecorded files count only rejections older than the ledger's first row;
|
|
2101
|
+
an unrecorded approval ends no round count (TRIAGE-1-MEDIUM). `polyctl slice status`
|
|
2102
|
+
counts the same way for any slice it is asked about (SLICE-LIFECYCLE).
|
|
2103
|
+
"""
|
|
2104
|
+
current = item if item is not None else running_slice(root)
|
|
2105
|
+
slice_id = ((current or {}).get("id") or (current or {}).get("key") or "").strip()
|
|
2106
|
+
if not slice_id:
|
|
2107
|
+
return 0
|
|
2108
|
+
respec = respecified_at(root, current) or 0
|
|
2109
|
+
|
|
2110
|
+
from .storage import read_chained_jsonl
|
|
2111
|
+
try:
|
|
2112
|
+
rows = read_chained_jsonl(review_ledger_path(root), REVIEW_CHAIN)
|
|
2113
|
+
except Exception:
|
|
2114
|
+
rows = [] # an unreadable ledger is reported elsewhere; files still count
|
|
2115
|
+
events, recorded, rejected = [], set(), set()
|
|
2116
|
+
for order, row in enumerate(rows):
|
|
2117
|
+
if not isinstance(row, dict) or row.get("kind") != "index-candidate":
|
|
2118
|
+
continue
|
|
2119
|
+
recorded.add(row.get("artefact"))
|
|
2120
|
+
verdict, candidate = row.get("verdict"), row.get("candidate")
|
|
2121
|
+
if verdict not in ("APPROVED", "NEEDS_CHANGES"):
|
|
2122
|
+
continue
|
|
2123
|
+
row_slice = row["slice"] if "slice" in row else _recorded_review_slice(root, reviews_dir, row)
|
|
2124
|
+
mine = (row_slice or "").strip() == slice_id
|
|
2125
|
+
try:
|
|
2126
|
+
at = int(row.get("at") or 0)
|
|
2127
|
+
except (TypeError, ValueError):
|
|
2128
|
+
at = 0
|
|
2129
|
+
if verdict == "NEEDS_CHANGES":
|
|
2130
|
+
if candidate:
|
|
2131
|
+
rejected.add(candidate)
|
|
2132
|
+
if mine:
|
|
2133
|
+
events.append((1, order, at, "round"))
|
|
2134
|
+
elif mine and row.get("authorizable") is True \
|
|
2135
|
+
and not (row.get("fallback") and candidate in rejected):
|
|
2136
|
+
events.append((1, order, at, "approved"))
|
|
2137
|
+
|
|
2138
|
+
for order, (f, v) in enumerate(reviews(root, reviews_dir, limit=None)):
|
|
2139
|
+
if f in recorded or v != "NEEDS_CHANGES":
|
|
2140
|
+
continue
|
|
2141
|
+
path = os.path.join(root, reviews_dir, f)
|
|
2142
|
+
try:
|
|
2143
|
+
at = int(os.path.getmtime(path))
|
|
2144
|
+
body = open(path, errors="replace", encoding=UTF8).read(100_000)
|
|
2145
|
+
except OSError:
|
|
2146
|
+
continue
|
|
2147
|
+
evidence = review_evidence(body)
|
|
2148
|
+
if not evidence or evidence.get("kind") != "index-candidate" \
|
|
2149
|
+
or (evidence.get("slice") or "").strip() != slice_id \
|
|
2150
|
+
or evidence.get("review_status") in ("unavailable", "invalid"):
|
|
2151
|
+
continue
|
|
2152
|
+
if rows and at >= int(rows[0].get("at") or 0):
|
|
2153
|
+
continue
|
|
2154
|
+
events.append((0, -order, at, "round"))
|
|
2155
|
+
|
|
2156
|
+
n = 0
|
|
2157
|
+
for _, _, at, what in sorted(events, reverse=True):
|
|
2158
|
+
if what == "approved":
|
|
2159
|
+
break
|
|
2160
|
+
if at < respec:
|
|
2161
|
+
continue
|
|
2162
|
+
n += 1
|
|
2163
|
+
return n
|
|
2164
|
+
|
|
2165
|
+
|
|
2166
|
+
def latest_slice_review(root, reviews_dir, slice_id):
|
|
2167
|
+
"""The newest review of a slice that reached a verdict, and what its file still says (SLICE-LIFECYCLE).
|
|
2168
|
+
|
|
2169
|
+
None when no review that reached a verdict names the slice. Otherwise {"artefact", "at", "kind", "verdict",
|
|
2170
|
+
"counts", "findings", "problem", "recorded"}: the verdict is the ledger's, and the counts and the findings are
|
|
2171
|
+
read from the artefact only while it holds the bytes its row recorded, since a review file is in the tree an
|
|
2172
|
+
implementer writes; `problem` says why they could not be read. A retrospective review (`kind` commit-range)
|
|
2173
|
+
counts too: a catch-up that asked for changes after the slice landed left findings open all the same.
|
|
2174
|
+
|
|
2175
|
+
A review written before the review ledger has no row, and `rounds` reads its file as it always was: a slice
|
|
2176
|
+
that began before the ledger has reviews only there, and an upgraded project said it had none. Where no row
|
|
2177
|
+
names the slice, the newest such file that does is its newest review, read as it stands - nothing records
|
|
2178
|
+
its bytes - and `recorded` is False (SLICE-LIFECYCLE-2).
|
|
2179
|
+
"""
|
|
2180
|
+
from .storage import read_chained_jsonl
|
|
2181
|
+
newest, recorded = None, set()
|
|
2182
|
+
for row in read_chained_jsonl(review_ledger_path(root), REVIEW_CHAIN):
|
|
2183
|
+
if not isinstance(row, dict):
|
|
2184
|
+
continue
|
|
2185
|
+
recorded.add(row.get("artefact"))
|
|
2186
|
+
if row.get("verdict") not in ("APPROVED", "NEEDS_CHANGES"):
|
|
2187
|
+
continue
|
|
2188
|
+
row_slice = row["slice"] if "slice" in row else _recorded_review_slice(root, reviews_dir, row)
|
|
2189
|
+
if (row_slice or "").strip() == slice_id:
|
|
2190
|
+
newest = row
|
|
2191
|
+
if newest is None:
|
|
2192
|
+
return _unrecorded_slice_review(root, reviews_dir, slice_id, recorded)
|
|
2193
|
+
out = {"artefact": newest.get("artefact"), "at": newest.get("at"), "kind": newest.get("kind"),
|
|
2194
|
+
"verdict": newest.get("verdict"), "counts": None, "findings": [], "problem": None, "recorded": True}
|
|
2195
|
+
try:
|
|
2196
|
+
with open(os.path.join(root, reviews_dir, str(newest.get("artefact"))), "rb") as fh:
|
|
2197
|
+
data = fh.read()
|
|
2198
|
+
except OSError:
|
|
2199
|
+
out["problem"] = "its file is not in the review directory: archived or removed"
|
|
2200
|
+
return out
|
|
2201
|
+
if "sha256:" + hashlib.sha256(data).hexdigest() != newest.get("sha256"):
|
|
2202
|
+
out["problem"] = "its file is not the bytes the review ledger recorded"
|
|
2203
|
+
return out
|
|
2204
|
+
return _review_findings(out, data.decode(UTF8, "replace"))
|
|
2205
|
+
|
|
2206
|
+
|
|
2207
|
+
def _review_findings(out, body):
|
|
2208
|
+
"""`out` with the counts and the finding lines a review's text holds."""
|
|
2209
|
+
counts = (review_evidence(body) or {}).get("counts")
|
|
2210
|
+
out["counts"] = counts if isinstance(counts, dict) else None
|
|
2211
|
+
out["findings"] = [f"[{found.group(1)}] {found.group(2).strip()}"
|
|
2212
|
+
for found in (_FINDING_LINE.match(line) for line in body.splitlines()) if found]
|
|
2213
|
+
return out
|
|
2214
|
+
|
|
2215
|
+
|
|
2216
|
+
def _unrecorded_slice_review(root, reviews_dir, slice_id, recorded):
|
|
2217
|
+
"""The newest review file of a slice the review ledger has no row for, read as `rounds` reads one; or None.
|
|
2218
|
+
|
|
2219
|
+
Newest by when the file was written, as `rounds` orders them, and only a review that reached a verdict and is
|
|
2220
|
+
not an attempt that never took place (SLICE-LIFECYCLE-2).
|
|
2221
|
+
"""
|
|
2222
|
+
for name, verdict in reviews(root, reviews_dir, limit=None):
|
|
2223
|
+
if name in recorded or verdict not in ("APPROVED", "NEEDS_CHANGES"):
|
|
2224
|
+
continue
|
|
2225
|
+
path = os.path.join(root, reviews_dir, name)
|
|
2226
|
+
try:
|
|
2227
|
+
at = int(os.path.getmtime(path))
|
|
2228
|
+
body = open(path, errors="replace", encoding=UTF8).read(100_000)
|
|
2229
|
+
except OSError:
|
|
2230
|
+
continue
|
|
2231
|
+
evidence = review_evidence(body)
|
|
2232
|
+
if not evidence or (evidence.get("slice") or "").strip() != slice_id \
|
|
2233
|
+
or evidence.get("review_status") in ("unavailable", "invalid"):
|
|
2234
|
+
continue
|
|
2235
|
+
return _review_findings({"artefact": name, "at": at, "kind": evidence.get("kind"), "verdict": verdict,
|
|
2236
|
+
"counts": None, "findings": [], "problem": None, "recorded": False}, body)
|
|
2237
|
+
return None
|
|
2238
|
+
|
|
2239
|
+
|
|
2240
|
+
def mailbox(root, mail_dir):
|
|
2241
|
+
d = os.path.join(root, mail_dir)
|
|
2242
|
+
files = [f for f in sorted(os.listdir(d)) if f != "README.md" and f.endswith(".md")] if os.path.isdir(d) else []
|
|
2243
|
+
if mail_store_mode(root) != "append-only":
|
|
2244
|
+
return files
|
|
2245
|
+
# The queue is derived from the store: a view file removed unhandled still counts (#80).
|
|
2246
|
+
try:
|
|
2247
|
+
rows = mail_store_rows(root)
|
|
2248
|
+
except Exception:
|
|
2249
|
+
return files
|
|
2250
|
+
stored = {row.get("id") for row in rows if row.get("event") == "message"}
|
|
2251
|
+
handled = {row.get("id") for row in rows if row.get("event") == "handled"}
|
|
2252
|
+
return sorted((stored - handled) | {name for name in files if name not in stored})
|
|
2253
|
+
|
|
2254
|
+
|
|
2255
|
+
REMOTE_PREFIX = "refs/remotes/origin/"
|
|
2256
|
+
|
|
2257
|
+
|
|
2258
|
+
def _default_remote_branch(root):
|
|
2259
|
+
"""The remote default branch to measure a checkout against, as a full ref, or None.
|
|
2260
|
+
|
|
2261
|
+
origin/HEAD can still name a branch the remote has since deleted, so a candidate
|
|
2262
|
+
is used only when it resolves to a commit.
|
|
2263
|
+
"""
|
|
2264
|
+
candidates = []
|
|
2265
|
+
try:
|
|
2266
|
+
ref = _git_output(root, "symbolic-ref", "--quiet",
|
|
2267
|
+
"refs/remotes/origin/HEAD").decode(UTF8, "replace").strip()
|
|
2268
|
+
if ref.startswith(REMOTE_PREFIX):
|
|
2269
|
+
candidates.append(ref)
|
|
2270
|
+
except RuntimeError:
|
|
2271
|
+
pass
|
|
2272
|
+
for candidate in (*candidates, REMOTE_PREFIX + "main", REMOTE_PREFIX + "master"):
|
|
2273
|
+
try:
|
|
2274
|
+
_git_output(root, "rev-parse", "--verify", "--quiet", candidate + "^{commit}")
|
|
2275
|
+
return candidate
|
|
2276
|
+
except RuntimeError:
|
|
2277
|
+
continue
|
|
2278
|
+
return None
|
|
2279
|
+
|
|
2280
|
+
|
|
2281
|
+
def git_state(root):
|
|
2282
|
+
"""Where this checkout stands, not only what it holds.
|
|
2283
|
+
|
|
2284
|
+
An ahead count alone reads as current on a checkout that is weeks behind. On
|
|
2285
|
+
2026-09-14 `polyctl status` printed "0 commits unpushed" for a branch 77 commits
|
|
2286
|
+
behind main and already merged into it, and two backlog rows were written from
|
|
2287
|
+
measurements taken in that checkout. So measure against the remote default
|
|
2288
|
+
branch, report behind as well as ahead, say when the branch is already merged,
|
|
2289
|
+
and say "?" when there is nothing to compare against instead of "0".
|
|
2290
|
+
|
|
2291
|
+
"ahead" stays a string for existing readers; "behind" is an int or None.
|
|
2292
|
+
"""
|
|
2293
|
+
ref = _default_remote_branch(root)
|
|
2294
|
+
ahead = behind = None
|
|
2295
|
+
if ref:
|
|
2296
|
+
try:
|
|
2297
|
+
counts = _git_output(root, "rev-list", "--left-right", "--count",
|
|
2298
|
+
f"{ref}...HEAD").decode(UTF8, "replace").split()
|
|
2299
|
+
if len(counts) == 2 and all(c.isdigit() for c in counts):
|
|
2300
|
+
behind, ahead = int(counts[0]), int(counts[1])
|
|
2301
|
+
except RuntimeError:
|
|
2302
|
+
pass
|
|
2303
|
+
merged = False
|
|
2304
|
+
if ahead == 0 and behind:
|
|
2305
|
+
# The default branch itself, checked out behind its remote, needs a pull; it
|
|
2306
|
+
# is not a merged branch. A detached head inside the base counts as merged.
|
|
2307
|
+
try:
|
|
2308
|
+
head = _git_output(root, "symbolic-ref", "--quiet", "HEAD").decode(UTF8, "replace").strip()
|
|
2309
|
+
except RuntimeError:
|
|
2310
|
+
head = ""
|
|
2311
|
+
merged = head != "refs/heads/" + ref[len(REMOTE_PREFIX):]
|
|
2312
|
+
known = ahead is not None
|
|
2313
|
+
return {
|
|
2314
|
+
"log": _git_text(root, "log", "--oneline", "-4").split("\n"),
|
|
2315
|
+
"dirty": [l for l in _git_text(root, "status", "--short").split("\n") if l.strip()],
|
|
2316
|
+
"ahead": str(ahead) if known else "?",
|
|
2317
|
+
"behind": behind,
|
|
2318
|
+
"base": ref[len("refs/remotes/"):] if known else None,
|
|
2319
|
+
"merged": merged,
|
|
2320
|
+
}
|
|
2321
|
+
|
|
2322
|
+
|
|
2323
|
+
# ---- OTLP/JSON trace export of the project's ledger (EXPORT-OTLP) -----------------------
|
|
2324
|
+
|
|
2325
|
+
_OTLP_COMMIT_CHAIN = "polyctl-commit-row-v1"
|
|
2326
|
+
_OTLP_STEP_RANK = {"verification": 0, "review": 1, "grant": 2, "commit": 3}
|
|
2327
|
+
_OTLP_TRAILER_ACTOR = re.compile(r"^([A-Za-z0-9_.-]+)(?:\s*\(([A-Za-z0-9_.-]+)\))?$")
|
|
2328
|
+
_OTLP_BOOL_KEYS = frozenset({
|
|
2329
|
+
"polyctl.authorizable",
|
|
2330
|
+
"polyctl.granted",
|
|
2331
|
+
"polyctl.export.attested_only",
|
|
2332
|
+
"polyctl.export.partial",
|
|
2333
|
+
})
|
|
2334
|
+
_OTLP_INT_KEYS = frozenset({
|
|
2335
|
+
"polyctl.gates.count",
|
|
2336
|
+
"polyctl.gates.passed_count",
|
|
2337
|
+
"polyctl.gates.failed_count",
|
|
2338
|
+
"polyctl.tests.pass_count",
|
|
2339
|
+
"polyctl.tests.fail_count",
|
|
2340
|
+
"polyctl.reviewers.count",
|
|
2341
|
+
"polyctl.candidate.changed_count",
|
|
2342
|
+
"polyctl.reasons.count",
|
|
2343
|
+
})
|
|
2344
|
+
_OTLP_LEDGER_NAMES = frozenset({
|
|
2345
|
+
"verifications.jsonl",
|
|
2346
|
+
"reviews.jsonl",
|
|
2347
|
+
"authority.jsonl",
|
|
2348
|
+
"waivers.jsonl",
|
|
2349
|
+
"slices.jsonl",
|
|
2350
|
+
})
|
|
2351
|
+
_OTLP_VOCAB_ALLOWLIST = {
|
|
2352
|
+
"service.name": frozenset({"polyctl"}),
|
|
2353
|
+
"polyctl.step": frozenset({"verification", "review", "grant", "commit"}),
|
|
2354
|
+
"polyctl.verification.profile": frozenset({"quick", "fast", "full", "ci", "default"}),
|
|
2355
|
+
"polyctl.review.kind": frozenset({"index-candidate", "commit-range"}),
|
|
2356
|
+
"polyctl.review.tier": frozenset({
|
|
2357
|
+
"independent", "same-family", "person", "full", "single", "strict", "lite",
|
|
2358
|
+
}),
|
|
2359
|
+
"polyctl.outcome": frozenset({
|
|
2360
|
+
"PASS", "FAIL", "APPROVED", "NEEDS_CHANGES", "UNAVAILABLE", "INVALID",
|
|
2361
|
+
"GRANTED", "REFUSED", "LANDED",
|
|
2362
|
+
}),
|
|
2363
|
+
}
|
|
2364
|
+
_OTLP_WAIVER_ID = re.compile(r"^W-\d+(?:-\d+)?$")
|
|
2365
|
+
_OTLP_PATTERN_ALLOWLIST = {
|
|
2366
|
+
"service.version": re.compile(r"^\d+\.\d+\.\d+(?:[-+._][0-9A-Za-z._-]+)?$"),
|
|
2367
|
+
"polyctl.candidate": re.compile(r"^sha256:[0-9a-f]{64}$"),
|
|
2368
|
+
"polyctl.commit.sha": re.compile(r"^[0-9a-f]{40}(?:[0-9a-f]{24})?$"),
|
|
2369
|
+
"polyctl.commit.tree": re.compile(r"^[0-9a-f]{40}(?:[0-9a-f]{24})?$"),
|
|
2370
|
+
"polyctl.verification.id": re.compile(r"^V-(?:\d+|\d{8}-\d{6}-[0-9a-f]{8})$"),
|
|
2371
|
+
"polyctl.waiver.id": re.compile(r"^W-\d+(?:-\d+)?(?:,\s*W-\d+(?:-\d+)?)*$"),
|
|
2372
|
+
"polyctl.review.id": re.compile(r"^\d{4}-?\d{2}-?\d{2}-\d{6}-(?:[0-9a-f]{7,40}|R-\d+)(?:-\d+)?\.md$"),
|
|
2373
|
+
"polyctl.implementer.actor": re.compile(r"^sha256:[0-9a-f]{16}$"),
|
|
2374
|
+
"polyctl.reviewer.actor": re.compile(r"^sha256:[0-9a-f]{16}$"),
|
|
2375
|
+
}
|
|
2376
|
+
_OTLP_SLICE_ID = re.compile(r"^[A-Z][A-Z0-9]*(?:-[A-Z0-9]+)*$")
|
|
2377
|
+
_OTLP_PURE_HEX = re.compile(r"^[0-9A-F]{16,}$")
|
|
2378
|
+
_OTLP_PROFILE_NAME = re.compile(r"^[a-z][a-z0-9_-]{0,31}$")
|
|
2379
|
+
_OTLP_HEX_LOWER = re.compile(r"^[0-9a-f]{16,}$")
|
|
2380
|
+
_OTLP_SHORT_SHA = re.compile(r"^[0-9a-f]{7,64}$")
|
|
2381
|
+
|
|
2382
|
+
|
|
2383
|
+
def _otlp_hex_id(seed, chars):
|
|
2384
|
+
"""Lowercase hex id of `chars` nybbles derived from `seed`, never all zeros."""
|
|
2385
|
+
digest = hashlib.sha256(seed.encode(UTF8)).hexdigest()[:chars]
|
|
2386
|
+
return digest if any(ch != "0" for ch in digest) else ("0" * (chars - 1) + "1")
|
|
2387
|
+
|
|
2388
|
+
|
|
2389
|
+
def _otlp_unix_nano(value):
|
|
2390
|
+
"""A ledger timestamp as integer unix nanoseconds, or 0 when absent."""
|
|
2391
|
+
if isinstance(value, int) and not isinstance(value, bool):
|
|
2392
|
+
return max(0, value) * 1_000_000_000
|
|
2393
|
+
seconds = _epoch(value)
|
|
2394
|
+
if seconds is None or seconds < 0:
|
|
2395
|
+
return 0
|
|
2396
|
+
return int(round(seconds * 1_000_000_000))
|
|
2397
|
+
|
|
2398
|
+
|
|
2399
|
+
def _otlp_declared_profiles(root):
|
|
2400
|
+
"""Profile names declared in `.polyctl/gates.json` plus the standard built-in profile names."""
|
|
2401
|
+
allowed = set(_OTLP_VOCAB_ALLOWLIST["polyctl.verification.profile"])
|
|
2402
|
+
if not root:
|
|
2403
|
+
return frozenset(allowed)
|
|
2404
|
+
try:
|
|
2405
|
+
with open(os.path.join(root, ".polyctl", "gates.json"), encoding=UTF8) as fh:
|
|
2406
|
+
spec = json.load(fh)
|
|
2407
|
+
except (OSError, ValueError):
|
|
2408
|
+
return frozenset(allowed)
|
|
2409
|
+
profiles = spec.get("profiles") if isinstance(spec, dict) else None
|
|
2410
|
+
if isinstance(profiles, dict):
|
|
2411
|
+
for name in profiles:
|
|
2412
|
+
if isinstance(name, str) and _OTLP_PROFILE_NAME.fullmatch(name) and not _OTLP_HEX_LOWER.fullmatch(name):
|
|
2413
|
+
allowed.add(name)
|
|
2414
|
+
return frozenset(allowed)
|
|
2415
|
+
|
|
2416
|
+
|
|
2417
|
+
def _otlp_changed_count(cand):
|
|
2418
|
+
"""Count of changed paths on a candidate record (`changed_paths`, `changed_count`, or `changed`)."""
|
|
2419
|
+
if not isinstance(cand, dict):
|
|
2420
|
+
return 0
|
|
2421
|
+
if isinstance(cand.get("changed_paths"), list):
|
|
2422
|
+
return len(cand["changed_paths"])
|
|
2423
|
+
if isinstance(cand.get("changed_count"), int) and not isinstance(cand.get("changed_count"), bool):
|
|
2424
|
+
return max(0, int(cand["changed_count"]))
|
|
2425
|
+
if isinstance(cand.get("changed"), list):
|
|
2426
|
+
return len(cand["changed"])
|
|
2427
|
+
return 0
|
|
2428
|
+
|
|
2429
|
+
|
|
2430
|
+
def _otlp_adapter_catalog(cfg=None):
|
|
2431
|
+
"""`(declared_families, adapter_name_to_family)` from shipped adapter declarations."""
|
|
2432
|
+
adapters = package_adapters()
|
|
2433
|
+
declared = set()
|
|
2434
|
+
by_name = {}
|
|
2435
|
+
for aid, spec in adapters.items():
|
|
2436
|
+
if not isinstance(spec, dict):
|
|
2437
|
+
continue
|
|
2438
|
+
fam = str(spec.get("family") or "").strip().lower()
|
|
2439
|
+
if not fam:
|
|
2440
|
+
continue
|
|
2441
|
+
declared.add(fam)
|
|
2442
|
+
by_name[str(aid).strip().lower()] = fam
|
|
2443
|
+
actor_name = str(spec.get("actor_name") or "").strip().lower()
|
|
2444
|
+
if actor_name:
|
|
2445
|
+
by_name[actor_name] = fam
|
|
2446
|
+
for binary in adapter_binaries(spec):
|
|
2447
|
+
if isinstance(binary, str) and binary.strip():
|
|
2448
|
+
by_name[binary.strip().lower()] = fam
|
|
2449
|
+
if isinstance(cfg, dict):
|
|
2450
|
+
blocks = [cfg.get("implementer"), cfg.get("reviewer")]
|
|
2451
|
+
reviewers = cfg.get("reviewers")
|
|
2452
|
+
if isinstance(reviewers, list):
|
|
2453
|
+
blocks.extend(reviewers)
|
|
2454
|
+
elif isinstance(reviewers, dict):
|
|
2455
|
+
blocks.extend(reviewers.values())
|
|
2456
|
+
actors = cfg.get("actors")
|
|
2457
|
+
if isinstance(actors, dict):
|
|
2458
|
+
for actor_key, actor_block in actors.items():
|
|
2459
|
+
if isinstance(actor_block, dict):
|
|
2460
|
+
blocks.append(dict(actor_block, actor=actor_block.get("actor") or actor_key))
|
|
2461
|
+
for block in blocks:
|
|
2462
|
+
if not isinstance(block, dict):
|
|
2463
|
+
continue
|
|
2464
|
+
ident = block_adapter(block)
|
|
2465
|
+
spec = adapters.get(ident) if ident else None
|
|
2466
|
+
fam = str((spec or {}).get("family") or "").strip().lower()
|
|
2467
|
+
if not fam:
|
|
2468
|
+
continue
|
|
2469
|
+
for key in ("id", "name", "actor", "binding"):
|
|
2470
|
+
val = block.get(key)
|
|
2471
|
+
if isinstance(val, str) and val.strip():
|
|
2472
|
+
by_name[val.strip().lower()] = fam
|
|
2473
|
+
return frozenset(declared), by_name
|
|
2474
|
+
|
|
2475
|
+
|
|
2476
|
+
def _otlp_waiver_ids(*values):
|
|
2477
|
+
"""Deduplicated allowlisted waiver ids from one or more raw strings or comma-joined trailers (LOWS-0.7)."""
|
|
2478
|
+
out = []
|
|
2479
|
+
for value in values:
|
|
2480
|
+
if not isinstance(value, str) or not value.strip():
|
|
2481
|
+
continue
|
|
2482
|
+
for part in value.split(","):
|
|
2483
|
+
token = part.strip()
|
|
2484
|
+
if token and _OTLP_WAIVER_ID.fullmatch(token) and token not in out:
|
|
2485
|
+
out.append(token)
|
|
2486
|
+
return out
|
|
2487
|
+
|
|
2488
|
+
|
|
2489
|
+
def _otlp_allow_string(key, value, declared_families=None, declared_profiles=None):
|
|
2490
|
+
"""Return `value` if and only if `(key, value)` belongs to the OTLP attribute allowlist."""
|
|
2491
|
+
if value is None or isinstance(value, bool):
|
|
2492
|
+
return None
|
|
2493
|
+
text = str(value).strip()
|
|
2494
|
+
if not text:
|
|
2495
|
+
return None
|
|
2496
|
+
if key == "polyctl.waiver.id":
|
|
2497
|
+
ids = _otlp_waiver_ids(text)
|
|
2498
|
+
return ", ".join(ids) if ids and _OTLP_PATTERN_ALLOWLIST[key].fullmatch(text) else None
|
|
2499
|
+
if key == "polyctl.verification.profile":
|
|
2500
|
+
profiles = declared_profiles if declared_profiles is not None else _OTLP_VOCAB_ALLOWLIST[key]
|
|
2501
|
+
return text if text in profiles else None
|
|
2502
|
+
if key == "polyctl.export.unattested_ledgers":
|
|
2503
|
+
parts = [part.strip() for part in text.split(",") if part.strip()]
|
|
2504
|
+
if parts and all(part in _OTLP_LEDGER_NAMES for part in parts):
|
|
2505
|
+
return ",".join(parts)
|
|
2506
|
+
return None
|
|
2507
|
+
if key in _OTLP_VOCAB_ALLOWLIST:
|
|
2508
|
+
return text if text in _OTLP_VOCAB_ALLOWLIST[key] else None
|
|
2509
|
+
if key in ("polyctl.implementer.family", "polyctl.reviewer.family"):
|
|
2510
|
+
families = declared_families if declared_families is not None else _otlp_adapter_catalog()[0]
|
|
2511
|
+
low = text.lower()
|
|
2512
|
+
return low if (low in families or low == "unknown") else None
|
|
2513
|
+
if key == "polyctl.slice":
|
|
2514
|
+
if len(text) <= 32 and _OTLP_SLICE_ID.fullmatch(text) and not _OTLP_PURE_HEX.fullmatch(text):
|
|
2515
|
+
return text
|
|
2516
|
+
return None
|
|
2517
|
+
pattern = _OTLP_PATTERN_ALLOWLIST.get(key)
|
|
2518
|
+
if pattern is not None:
|
|
2519
|
+
return text if pattern.fullmatch(text) else None
|
|
2520
|
+
return None
|
|
2521
|
+
|
|
2522
|
+
|
|
2523
|
+
def _otlp_attributes(pairs, declared_families=None, declared_profiles=None):
|
|
2524
|
+
"""OTLP/JSON KeyValue list from allowlisted `(key, value)` pairs."""
|
|
2525
|
+
out, seen = [], set()
|
|
2526
|
+
for key, value in pairs:
|
|
2527
|
+
if not key or key in seen or value is None:
|
|
2528
|
+
continue
|
|
2529
|
+
if key in _OTLP_BOOL_KEYS:
|
|
2530
|
+
if not isinstance(value, bool):
|
|
2531
|
+
continue
|
|
2532
|
+
encoded = {"boolValue": value}
|
|
2533
|
+
elif key in _OTLP_INT_KEYS:
|
|
2534
|
+
if not isinstance(value, int) or isinstance(value, bool) or value < 0:
|
|
2535
|
+
continue
|
|
2536
|
+
encoded = {"intValue": str(value)}
|
|
2537
|
+
else:
|
|
2538
|
+
allowed = _otlp_allow_string(
|
|
2539
|
+
key, value, declared_families=declared_families, declared_profiles=declared_profiles
|
|
2540
|
+
)
|
|
2541
|
+
if allowed is None:
|
|
2542
|
+
continue
|
|
2543
|
+
encoded = {"stringValue": allowed}
|
|
2544
|
+
seen.add(key)
|
|
2545
|
+
out.append({"key": key, "value": encoded})
|
|
2546
|
+
return out
|
|
2547
|
+
|
|
2548
|
+
|
|
2549
|
+
def _otlp_actor_pairs(prefix, value, identity=None, catalog=None):
|
|
2550
|
+
"""`[(prefix.actor, sha256_id), (prefix.family, family)]` from an actor dict, route id or trailer."""
|
|
2551
|
+
declared_families, by_name = catalog if catalog is not None else _otlp_adapter_catalog()
|
|
2552
|
+
raw_actor = None
|
|
2553
|
+
stated_family = None
|
|
2554
|
+
candidates = []
|
|
2555
|
+
|
|
2556
|
+
if isinstance(value, dict):
|
|
2557
|
+
for field in ("actor", "binding", "id", "name"):
|
|
2558
|
+
cand = value.get(field)
|
|
2559
|
+
if isinstance(cand, str) and cand.strip():
|
|
2560
|
+
raw_actor = cand.strip()
|
|
2561
|
+
break
|
|
2562
|
+
if isinstance(value.get("family"), str) and value["family"].strip():
|
|
2563
|
+
stated_family = value["family"].strip()
|
|
2564
|
+
for field in ("adapter", "tool", "actor", "binding", "id", "name"):
|
|
2565
|
+
cand = value.get(field)
|
|
2566
|
+
if isinstance(cand, str) and cand.strip():
|
|
2567
|
+
candidates.append(cand.strip())
|
|
2568
|
+
elif isinstance(value, str) and value.strip():
|
|
2569
|
+
text = value.strip()
|
|
2570
|
+
match = _OTLP_TRAILER_ACTOR.match(text)
|
|
2571
|
+
if match:
|
|
2572
|
+
raw_actor = match.group(1)
|
|
2573
|
+
stated_family = match.group(2)
|
|
2574
|
+
else:
|
|
2575
|
+
raw_actor = text
|
|
2576
|
+
if raw_actor:
|
|
2577
|
+
candidates.append(raw_actor)
|
|
2578
|
+
|
|
2579
|
+
if isinstance(identity, dict):
|
|
2580
|
+
if not raw_actor:
|
|
2581
|
+
for field in ("actor", "binding", "id", "name"):
|
|
2582
|
+
cand = identity.get(field)
|
|
2583
|
+
if isinstance(cand, str) and cand.strip():
|
|
2584
|
+
raw_actor = cand.strip()
|
|
2585
|
+
break
|
|
2586
|
+
if not stated_family and isinstance(identity.get("family"), str) and identity["family"].strip():
|
|
2587
|
+
stated_family = identity["family"].strip()
|
|
2588
|
+
for field in ("adapter", "tool", "actor", "binding", "id", "name"):
|
|
2589
|
+
cand = identity.get(field)
|
|
2590
|
+
if isinstance(cand, str) and cand.strip():
|
|
2591
|
+
candidates.append(cand.strip())
|
|
2592
|
+
|
|
2593
|
+
if not raw_actor and not stated_family and not candidates:
|
|
2594
|
+
return []
|
|
2595
|
+
|
|
2596
|
+
resolved_family = None
|
|
2597
|
+
for cand in candidates:
|
|
2598
|
+
if "/" in cand or "\\" in cand:
|
|
2599
|
+
continue
|
|
2600
|
+
low = cand.lower()
|
|
2601
|
+
if low in by_name:
|
|
2602
|
+
resolved_family = by_name[low]
|
|
2603
|
+
break
|
|
2604
|
+
stem = low.split("-reviewer", 1)[0]
|
|
2605
|
+
if stem and stem in by_name:
|
|
2606
|
+
resolved_family = by_name[stem]
|
|
2607
|
+
break
|
|
2608
|
+
if resolved_family is None and isinstance(stated_family, str):
|
|
2609
|
+
low_fam = stated_family.strip().lower()
|
|
2610
|
+
if low_fam in declared_families:
|
|
2611
|
+
resolved_family = low_fam
|
|
2612
|
+
family = resolved_family if resolved_family in declared_families else "unknown"
|
|
2613
|
+
|
|
2614
|
+
pairs = []
|
|
2615
|
+
if raw_actor:
|
|
2616
|
+
actor_hash = "sha256:" + _otlp_hex_id(f"polyctl-otlp-actor-v1:{raw_actor}", 16)
|
|
2617
|
+
pairs.append((f"{prefix}.actor", actor_hash))
|
|
2618
|
+
pairs.append((f"{prefix}.family", family))
|
|
2619
|
+
return pairs
|
|
2620
|
+
|
|
2621
|
+
|
|
2622
|
+
def _otlp_commit_trailers(commit):
|
|
2623
|
+
"""`{short_name: first_value}` of a landed commit's `Polyctl-` trailers."""
|
|
2624
|
+
raw = commit.get("trailers")
|
|
2625
|
+
if isinstance(raw, list):
|
|
2626
|
+
carried = _polyctl_trailers(raw)
|
|
2627
|
+
return {key[len(TRAILER_PREFIX):].lower(): values[0] for key, values in carried.items() if values}
|
|
2628
|
+
if isinstance(raw, dict):
|
|
2629
|
+
return {str(key).lower(): str(val) for key, val in raw.items() if val is not None}
|
|
2630
|
+
return {}
|
|
2631
|
+
|
|
2632
|
+
|
|
2633
|
+
def _otlp_resolve_grant(row, maps):
|
|
2634
|
+
"""Resolve a grant row's candidate, slice, review, verification, waiver and tree links once."""
|
|
2635
|
+
cand = row.get("candidate") if isinstance(row.get("candidate"), dict) else {}
|
|
2636
|
+
itree = _otlp_allow_string("polyctl.commit.tree", cand.get("index_tree") or row.get("tree"))
|
|
2637
|
+
vid = _otlp_allow_string("polyctl.verification.id", row.get("verification"))
|
|
2638
|
+
artefact = _otlp_allow_string("polyctl.review.id", os.path.basename(str(row.get("review") or "")))
|
|
2639
|
+
wids = _otlp_waiver_ids(row.get("waiver"), row.get("gates_waiver"))
|
|
2640
|
+
wid = ", ".join(wids) if wids else None
|
|
2641
|
+
cdigest = (
|
|
2642
|
+
_otlp_allow_string("polyctl.candidate", cand.get("digest"))
|
|
2643
|
+
or maps["verification_to_candidate"].get(vid)
|
|
2644
|
+
or maps["review_to_candidate"].get(artefact)
|
|
2645
|
+
or next((maps["waiver_to_candidate"][w] for w in wids if w in maps["waiver_to_candidate"]), None)
|
|
2646
|
+
or maps["tree_to_candidate"].get(itree)
|
|
2647
|
+
)
|
|
2648
|
+
review_row = maps["review_by_artefact"].get(artefact) or (
|
|
2649
|
+
maps["review_by_candidate"].get(cdigest) if not wid else None
|
|
2650
|
+
)
|
|
2651
|
+
if artefact is None and review_row is not None:
|
|
2652
|
+
artefact = _otlp_allow_string("polyctl.review.id", os.path.basename(str(review_row.get("artefact") or "")))
|
|
2653
|
+
move_only = row.get("move_only") if isinstance(row.get("move_only"), dict) else {}
|
|
2654
|
+
gslice = _otlp_allow_string("polyctl.slice", row.get("slice") or move_only.get("slice"))
|
|
2655
|
+
slice_id = (
|
|
2656
|
+
gslice
|
|
2657
|
+
or maps["review_to_slice"].get(artefact)
|
|
2658
|
+
or next((maps["waiver_to_slice"][w] for w in wids if w in maps["waiver_to_slice"]), None)
|
|
2659
|
+
or maps["verification_to_slice"].get(vid)
|
|
2660
|
+
or maps["candidate_to_slice"].get(cdigest)
|
|
2661
|
+
or maps["tree_to_slice"].get(itree)
|
|
2662
|
+
)
|
|
2663
|
+
return {
|
|
2664
|
+
"cand": cand,
|
|
2665
|
+
"itree": itree,
|
|
2666
|
+
"vid": vid,
|
|
2667
|
+
"artefact": artefact,
|
|
2668
|
+
"wid": wid,
|
|
2669
|
+
"wids": wids,
|
|
2670
|
+
"cdigest": cdigest,
|
|
2671
|
+
"slice_id": slice_id,
|
|
2672
|
+
"review_row": review_row,
|
|
2673
|
+
}
|
|
2674
|
+
|
|
2675
|
+
|
|
2676
|
+
def _otlp_resolve_commit(commit, maps):
|
|
2677
|
+
"""Resolve a landed commit's grant, candidate, slice, review, verification and waiver links once."""
|
|
2678
|
+
csha = _otlp_allow_string("polyctl.commit.sha", commit.get("sha"))
|
|
2679
|
+
ctree = _otlp_allow_string("polyctl.commit.tree", commit.get("tree"))
|
|
2680
|
+
trailers = _otlp_commit_trailers(commit)
|
|
2681
|
+
has_polyctl_trailer = any(trailers.get(k) for k in ("grant", "tree", "verification", "review", "waiver"))
|
|
2682
|
+
grant_token = str(trailers.get("grant") or "").strip()
|
|
2683
|
+
grant_row = maps["token_to_grant"].get(grant_token) or maps["tree_to_grant"].get(ctree)
|
|
2684
|
+
gcand = (
|
|
2685
|
+
grant_row.get("candidate")
|
|
2686
|
+
if isinstance(grant_row, dict) and isinstance(grant_row.get("candidate"), dict)
|
|
2687
|
+
else {}
|
|
2688
|
+
)
|
|
2689
|
+
vid = _otlp_allow_string(
|
|
2690
|
+
"polyctl.verification.id", trailers.get("verification") or (grant_row or {}).get("verification")
|
|
2691
|
+
)
|
|
2692
|
+
artefact = _otlp_allow_string(
|
|
2693
|
+
"polyctl.review.id",
|
|
2694
|
+
os.path.basename(str(trailers.get("review") or (grant_row or {}).get("review") or "")),
|
|
2695
|
+
)
|
|
2696
|
+
wids = _otlp_waiver_ids(
|
|
2697
|
+
trailers.get("waiver"),
|
|
2698
|
+
(grant_row or {}).get("waiver"),
|
|
2699
|
+
(grant_row or {}).get("gates_waiver"),
|
|
2700
|
+
)
|
|
2701
|
+
wid = ", ".join(wids) if wids else None
|
|
2702
|
+
cdigest = (
|
|
2703
|
+
_otlp_allow_string("polyctl.candidate", gcand.get("digest"))
|
|
2704
|
+
or maps["verification_to_candidate"].get(vid)
|
|
2705
|
+
or maps["review_to_candidate"].get(artefact)
|
|
2706
|
+
or next((maps["waiver_to_candidate"][w] for w in wids if w in maps["waiver_to_candidate"]), None)
|
|
2707
|
+
or maps["tree_to_candidate"].get(ctree)
|
|
2708
|
+
)
|
|
2709
|
+
review_row = (
|
|
2710
|
+
maps["review_by_artefact"].get(artefact)
|
|
2711
|
+
or (maps["review_by_candidate"].get(cdigest) if not wid else None)
|
|
2712
|
+
or maps["review_by_commit"].get(csha)
|
|
2713
|
+
)
|
|
2714
|
+
if artefact is None and review_row is not None:
|
|
2715
|
+
artefact = _otlp_allow_string("polyctl.review.id", os.path.basename(str(review_row.get("artefact") or "")))
|
|
2716
|
+
gmove = (grant_row or {}).get("move_only") if isinstance((grant_row or {}).get("move_only"), dict) else {}
|
|
2717
|
+
slice_id = (
|
|
2718
|
+
maps["commit_to_slice"].get(csha)
|
|
2719
|
+
or _otlp_allow_string("polyctl.slice", (grant_row or {}).get("slice") or gmove.get("slice"))
|
|
2720
|
+
or maps["review_to_slice"].get(artefact)
|
|
2721
|
+
or next((maps["waiver_to_slice"][w] for w in wids if w in maps["waiver_to_slice"]), None)
|
|
2722
|
+
or maps["verification_to_slice"].get(vid)
|
|
2723
|
+
or maps["candidate_to_slice"].get(cdigest)
|
|
2724
|
+
or maps["tree_to_slice"].get(ctree)
|
|
2725
|
+
)
|
|
2726
|
+
relevant = bool(
|
|
2727
|
+
has_polyctl_trailer
|
|
2728
|
+
or grant_row is not None
|
|
2729
|
+
or csha in maps["commit_to_slice"]
|
|
2730
|
+
or ctree in maps["tree_to_candidate"]
|
|
2731
|
+
or ctree in maps["tree_to_slice"]
|
|
2732
|
+
)
|
|
2733
|
+
return {
|
|
2734
|
+
"csha": csha,
|
|
2735
|
+
"ctree": ctree,
|
|
2736
|
+
"trailers": trailers,
|
|
2737
|
+
"grant_token": grant_token or None,
|
|
2738
|
+
"grant_row": grant_row,
|
|
2739
|
+
"vid": vid,
|
|
2740
|
+
"artefact": artefact,
|
|
2741
|
+
"wid": wid,
|
|
2742
|
+
"wids": wids,
|
|
2743
|
+
"cdigest": cdigest,
|
|
2744
|
+
"slice_id": slice_id,
|
|
2745
|
+
"review_row": review_row,
|
|
2746
|
+
"relevant": relevant,
|
|
2747
|
+
}
|
|
2748
|
+
|
|
2749
|
+
|
|
2750
|
+
def export_otlp(root, cfg=None, *, attested=False):
|
|
2751
|
+
"""The project's ledger as one OTLP/JSON ExportTraceServiceRequest (EXPORT-OTLP).
|
|
2752
|
+
|
|
2753
|
+
Strictly read-only: reads snapshot history without creating `.lock` files or settling
|
|
2754
|
+
pending seals, and never writes to the project or `~/.polyctl`. By default (`attested=False`),
|
|
2755
|
+
refuses with `LedgerCorruption` when any ledger has rows or a seal not covered by this
|
|
2756
|
+
machine's checkpoint store; with `attested=True`, exports only the checkpoint-attested
|
|
2757
|
+
prefix and states so in `resource.attributes`.
|
|
2758
|
+
"""
|
|
2759
|
+
from . import __version__
|
|
2760
|
+
from .storage import LedgerCorruption, _read_chained_history_snapshot, chained_row_digest
|
|
2761
|
+
|
|
2762
|
+
ledger_specs = (
|
|
2763
|
+
("verifications", os.path.join(root, ".polyctl", "ledger", "verifications.jsonl"), VERIFICATION_CHAIN, True),
|
|
2764
|
+
("reviews", review_ledger_path(root), REVIEW_CHAIN, False),
|
|
2765
|
+
("grants", os.path.join(root, ".polyctl", "ledger", "authority.jsonl"), AUTHORITY_CHAIN, False),
|
|
2766
|
+
("waivers", waivers_path(root), WAIVER_CHAIN, True),
|
|
2767
|
+
("slice_rows", slice_ledger_path(root), SLICE_CHAIN, False),
|
|
2768
|
+
)
|
|
2769
|
+
loaded = {}
|
|
2770
|
+
unattested_list = []
|
|
2771
|
+
for name, path, chain, legacy in ledger_specs:
|
|
2772
|
+
raw_rows, unattested = _read_chained_history_snapshot(
|
|
2773
|
+
path, chain, legacy_prefix=legacy, attested=True
|
|
2774
|
+
)
|
|
2775
|
+
loaded[name] = [row for row in raw_rows if isinstance(row, dict)]
|
|
2776
|
+
if unattested is not None:
|
|
2777
|
+
unattested_list.append(unattested)
|
|
2778
|
+
if unattested_list and not attested:
|
|
2779
|
+
raise LedgerCorruption("; ".join(item["message"] for item in unattested_list))
|
|
2780
|
+
|
|
2781
|
+
verifications = loaded["verifications"]
|
|
2782
|
+
reviews = loaded["reviews"]
|
|
2783
|
+
grants = loaded["grants"]
|
|
2784
|
+
waivers = loaded["waivers"]
|
|
2785
|
+
slice_rows = loaded["slice_rows"]
|
|
2786
|
+
voided_ordinals = {row["voids"]: row.get("slice") for row in slice_rows if isinstance(row.get("voids"), int)}
|
|
2787
|
+
slice_rows = [
|
|
2788
|
+
row for row in slice_rows
|
|
2789
|
+
if "voids" not in row and not (
|
|
2790
|
+
row.get("ordinal") in voided_ordinals and voided_ordinals[row.get("ordinal")] == row.get("slice")
|
|
2791
|
+
)
|
|
2792
|
+
]
|
|
2793
|
+
commits = [row for row in landed_commits(root, "HEAD") if isinstance(row, dict)]
|
|
2794
|
+
|
|
2795
|
+
if cfg is None:
|
|
2796
|
+
cfg = project_config_document(root).get("config")
|
|
2797
|
+
catalog = _otlp_adapter_catalog(cfg)
|
|
2798
|
+
declared_families = catalog[0]
|
|
2799
|
+
declared_profiles = _otlp_declared_profiles(root)
|
|
2800
|
+
|
|
2801
|
+
maps = {
|
|
2802
|
+
"candidate_to_slice": {},
|
|
2803
|
+
"tree_to_candidate": {},
|
|
2804
|
+
"tree_to_slice": {},
|
|
2805
|
+
"verification_to_slice": {},
|
|
2806
|
+
"verification_to_candidate": {},
|
|
2807
|
+
"review_to_slice": {},
|
|
2808
|
+
"review_to_candidate": {},
|
|
2809
|
+
"review_by_artefact": {},
|
|
2810
|
+
"review_by_candidate": {},
|
|
2811
|
+
"review_by_commit": {},
|
|
2812
|
+
"grant_by_review": {},
|
|
2813
|
+
"grant_by_candidate": {},
|
|
2814
|
+
"waiver_to_slice": {},
|
|
2815
|
+
"waiver_to_candidate": {},
|
|
2816
|
+
"token_to_grant": {},
|
|
2817
|
+
"tree_to_grant": {},
|
|
2818
|
+
"commit_to_slice": {},
|
|
2819
|
+
}
|
|
2820
|
+
|
|
2821
|
+
commit_shas = [
|
|
2822
|
+
csha for csha in (_otlp_allow_string("polyctl.commit.sha", c.get("sha")) for c in commits) if csha
|
|
2823
|
+
]
|
|
2824
|
+
for row in slice_rows:
|
|
2825
|
+
notes = row.get("notes") if isinstance(row.get("notes"), dict) else {}
|
|
2826
|
+
raw_commit = str(row.get("commit") or notes.get("commit") or "").strip().lower()
|
|
2827
|
+
sslice = _otlp_allow_string("polyctl.slice", row.get("slice"))
|
|
2828
|
+
if not (raw_commit and sslice and _OTLP_SHORT_SHA.fullmatch(raw_commit)):
|
|
2829
|
+
continue
|
|
2830
|
+
csha = _otlp_allow_string("polyctl.commit.sha", raw_commit)
|
|
2831
|
+
if csha:
|
|
2832
|
+
maps["commit_to_slice"][csha] = sslice
|
|
2833
|
+
else:
|
|
2834
|
+
matches = [full for full in commit_shas if full.startswith(raw_commit)]
|
|
2835
|
+
if len(matches) == 1:
|
|
2836
|
+
maps["commit_to_slice"][matches[0]] = sslice
|
|
2837
|
+
|
|
2838
|
+
for row in verifications:
|
|
2839
|
+
vid = _otlp_allow_string("polyctl.verification.id", row.get("id"))
|
|
2840
|
+
cand = row.get("candidate") if isinstance(row.get("candidate"), dict) else {}
|
|
2841
|
+
cdigest = _otlp_allow_string("polyctl.candidate", cand.get("digest"))
|
|
2842
|
+
itree = _otlp_allow_string("polyctl.commit.tree", cand.get("index_tree"))
|
|
2843
|
+
vslice = _otlp_allow_string("polyctl.slice", row.get("slice"))
|
|
2844
|
+
if vid and cdigest:
|
|
2845
|
+
maps["verification_to_candidate"][vid] = cdigest
|
|
2846
|
+
if itree and cdigest:
|
|
2847
|
+
maps["tree_to_candidate"][itree] = cdigest
|
|
2848
|
+
if vslice and cdigest:
|
|
2849
|
+
maps["candidate_to_slice"][cdigest] = vslice
|
|
2850
|
+
if vslice and vid:
|
|
2851
|
+
maps["verification_to_slice"][vid] = vslice
|
|
2852
|
+
if vslice and itree:
|
|
2853
|
+
maps["tree_to_slice"][itree] = vslice
|
|
2854
|
+
|
|
2855
|
+
for row in reviews:
|
|
2856
|
+
artefact = _otlp_allow_string("polyctl.review.id", os.path.basename(str(row.get("artefact") or "")))
|
|
2857
|
+
rslice = _otlp_allow_string("polyctl.slice", row.get("slice"))
|
|
2858
|
+
cdigest = _otlp_allow_string("polyctl.candidate", row.get("candidate"))
|
|
2859
|
+
if artefact:
|
|
2860
|
+
maps["review_by_artefact"][artefact] = row
|
|
2861
|
+
if cdigest:
|
|
2862
|
+
maps["review_by_candidate"][cdigest] = row
|
|
2863
|
+
if artefact and cdigest:
|
|
2864
|
+
maps["review_to_candidate"][artefact] = cdigest
|
|
2865
|
+
if artefact and rslice:
|
|
2866
|
+
maps["review_to_slice"][artefact] = rslice
|
|
2867
|
+
if cdigest and rslice:
|
|
2868
|
+
maps["candidate_to_slice"][cdigest] = rslice
|
|
2869
|
+
for rcommit in (row.get("commits") if isinstance(row.get("commits"), list) else []):
|
|
2870
|
+
csha = _otlp_allow_string("polyctl.commit.sha", rcommit)
|
|
2871
|
+
if csha:
|
|
2872
|
+
maps["review_by_commit"][csha] = row
|
|
2873
|
+
if rslice:
|
|
2874
|
+
maps["commit_to_slice"].setdefault(csha, rslice)
|
|
2875
|
+
|
|
2876
|
+
for row in waivers:
|
|
2877
|
+
if row.get("event") != "waived":
|
|
2878
|
+
continue
|
|
2879
|
+
wid = _otlp_allow_string("polyctl.waiver.id", row.get("id"))
|
|
2880
|
+
wslice = _otlp_allow_string("polyctl.slice", row.get("slice"))
|
|
2881
|
+
cdigest = _otlp_allow_string("polyctl.candidate", row.get("candidate"))
|
|
2882
|
+
if wid and cdigest:
|
|
2883
|
+
maps["waiver_to_candidate"][wid] = cdigest
|
|
2884
|
+
if wid and wslice:
|
|
2885
|
+
maps["waiver_to_slice"][wid] = wslice
|
|
2886
|
+
if cdigest and wslice:
|
|
2887
|
+
maps["candidate_to_slice"][cdigest] = wslice
|
|
2888
|
+
|
|
2889
|
+
def _propagate_grants():
|
|
2890
|
+
for row in grants:
|
|
2891
|
+
token = str(row.get("token") or "").strip()
|
|
2892
|
+
if token:
|
|
2893
|
+
maps["token_to_grant"][token] = row
|
|
2894
|
+
info = _otlp_resolve_grant(row, maps)
|
|
2895
|
+
itree = info["itree"]
|
|
2896
|
+
vid = info["vid"]
|
|
2897
|
+
artefact = info["artefact"]
|
|
2898
|
+
wid = info["wid"]
|
|
2899
|
+
resolved_cand = info["cdigest"]
|
|
2900
|
+
resolved_slice = info["slice_id"]
|
|
2901
|
+
if itree and row.get("granted") is True:
|
|
2902
|
+
maps["tree_to_grant"][itree] = row
|
|
2903
|
+
if artefact:
|
|
2904
|
+
maps["grant_by_review"][artefact] = row
|
|
2905
|
+
if resolved_cand:
|
|
2906
|
+
maps["grant_by_candidate"][resolved_cand] = row
|
|
2907
|
+
if resolved_cand and resolved_slice:
|
|
2908
|
+
maps["candidate_to_slice"][resolved_cand] = resolved_slice
|
|
2909
|
+
if itree and resolved_cand:
|
|
2910
|
+
maps["tree_to_candidate"][itree] = resolved_cand
|
|
2911
|
+
if itree and resolved_slice:
|
|
2912
|
+
maps["tree_to_slice"][itree] = resolved_slice
|
|
2913
|
+
if vid and resolved_slice:
|
|
2914
|
+
maps["verification_to_slice"][vid] = resolved_slice
|
|
2915
|
+
if vid and resolved_cand:
|
|
2916
|
+
maps["verification_to_candidate"][vid] = resolved_cand
|
|
2917
|
+
if artefact and resolved_slice:
|
|
2918
|
+
maps["review_to_slice"][artefact] = resolved_slice
|
|
2919
|
+
if artefact and resolved_cand:
|
|
2920
|
+
maps["review_to_candidate"][artefact] = resolved_cand
|
|
2921
|
+
for w in info["wids"]:
|
|
2922
|
+
if resolved_slice:
|
|
2923
|
+
maps["waiver_to_slice"][w] = resolved_slice
|
|
2924
|
+
if resolved_cand:
|
|
2925
|
+
maps["waiver_to_candidate"][w] = resolved_cand
|
|
2926
|
+
|
|
2927
|
+
_propagate_grants()
|
|
2928
|
+
|
|
2929
|
+
for commit in commits:
|
|
2930
|
+
info = _otlp_resolve_commit(commit, maps)
|
|
2931
|
+
ctree = info["ctree"]
|
|
2932
|
+
vid = info["vid"]
|
|
2933
|
+
artefact = info["artefact"]
|
|
2934
|
+
cdigest = info["cdigest"]
|
|
2935
|
+
cslice = info["slice_id"]
|
|
2936
|
+
if cdigest and cslice:
|
|
2937
|
+
maps["candidate_to_slice"][cdigest] = cslice
|
|
2938
|
+
if ctree and cslice:
|
|
2939
|
+
maps["tree_to_slice"][ctree] = cslice
|
|
2940
|
+
if ctree and cdigest:
|
|
2941
|
+
maps["tree_to_candidate"][ctree] = cdigest
|
|
2942
|
+
if vid and cslice:
|
|
2943
|
+
maps["verification_to_slice"][vid] = cslice
|
|
2944
|
+
if artefact and cslice:
|
|
2945
|
+
maps["review_to_slice"][artefact] = cslice
|
|
2946
|
+
for w in info["wids"]:
|
|
2947
|
+
if cslice:
|
|
2948
|
+
maps["waiver_to_slice"][w] = cslice
|
|
2949
|
+
|
|
2950
|
+
_propagate_grants()
|
|
2951
|
+
|
|
2952
|
+
steps = []
|
|
2953
|
+
for row in verifications:
|
|
2954
|
+
record_digest = chained_row_digest(row, VERIFICATION_CHAIN)
|
|
2955
|
+
vid = _otlp_allow_string("polyctl.verification.id", row.get("id"))
|
|
2956
|
+
cand = row.get("candidate") if isinstance(row.get("candidate"), dict) else {}
|
|
2957
|
+
cdigest = _otlp_allow_string("polyctl.candidate", cand.get("digest")) or maps["verification_to_candidate"].get(vid)
|
|
2958
|
+
itree = _otlp_allow_string("polyctl.commit.tree", cand.get("index_tree"))
|
|
2959
|
+
slice_id = (
|
|
2960
|
+
_otlp_allow_string("polyctl.slice", row.get("slice"))
|
|
2961
|
+
or maps["verification_to_slice"].get(vid)
|
|
2962
|
+
or maps["candidate_to_slice"].get(cdigest)
|
|
2963
|
+
or maps["tree_to_slice"].get(itree)
|
|
2964
|
+
)
|
|
2965
|
+
gates = [g for g in (row.get("gates") or []) if isinstance(g, dict)]
|
|
2966
|
+
passed_gates = sum(
|
|
2967
|
+
1 for g in gates
|
|
2968
|
+
if (g.get("passed") is True if isinstance(g.get("passed"), bool) else g.get("outcome") == "PASS")
|
|
2969
|
+
)
|
|
2970
|
+
failed_gates = len(gates) - passed_gates
|
|
2971
|
+
pass_tests = 0
|
|
2972
|
+
fail_tests = 0
|
|
2973
|
+
for g in gates:
|
|
2974
|
+
counts = g.get("counts") if isinstance(g.get("counts"), dict) else {}
|
|
2975
|
+
p_val = counts.get("pass") if "pass" in counts else g.get("pass_count")
|
|
2976
|
+
f_val = counts.get("fail") if "fail" in counts else g.get("fail_count")
|
|
2977
|
+
if isinstance(p_val, int) and not isinstance(p_val, bool):
|
|
2978
|
+
pass_tests += int(p_val)
|
|
2979
|
+
if isinstance(f_val, int) and not isinstance(f_val, bool):
|
|
2980
|
+
fail_tests += int(f_val)
|
|
2981
|
+
duration_ns = int(round(sum(
|
|
2982
|
+
float(g["seconds"]) for g in gates
|
|
2983
|
+
if isinstance(g.get("seconds"), (int, float)) and not isinstance(g.get("seconds"), bool)
|
|
2984
|
+
and float(g["seconds"]) > 0
|
|
2985
|
+
) * 1_000_000_000))
|
|
2986
|
+
start_ns = _otlp_unix_nano(row.get("at"))
|
|
2987
|
+
end_ns = start_ns + duration_ns if duration_ns > 0 else start_ns
|
|
2988
|
+
changed_count = _otlp_changed_count(cand)
|
|
2989
|
+
if isinstance(row.get("passed"), bool):
|
|
2990
|
+
outcome = "PASS" if row["passed"] else "FAIL"
|
|
2991
|
+
else:
|
|
2992
|
+
outcome = _otlp_allow_string("polyctl.outcome", row.get("outcome")) or "FAIL"
|
|
2993
|
+
steps.append({
|
|
2994
|
+
"step": "verification",
|
|
2995
|
+
"record_digest": record_digest,
|
|
2996
|
+
"span_id": _otlp_hex_id(f"polyctl-otlp-span-v1:verification:{record_digest}", 16),
|
|
2997
|
+
"slice": slice_id,
|
|
2998
|
+
"candidate": cdigest,
|
|
2999
|
+
"start_ns": start_ns,
|
|
3000
|
+
"end_ns": end_ns,
|
|
3001
|
+
"status_code": 1 if outcome == "PASS" else 2,
|
|
3002
|
+
"ref_id": vid,
|
|
3003
|
+
"links": {},
|
|
3004
|
+
"pairs": [
|
|
3005
|
+
("polyctl.step", "verification"),
|
|
3006
|
+
("polyctl.slice", slice_id),
|
|
3007
|
+
("polyctl.candidate", cdigest),
|
|
3008
|
+
("polyctl.verification.id", vid),
|
|
3009
|
+
("polyctl.verification.profile", row.get("profile")),
|
|
3010
|
+
("polyctl.outcome", outcome),
|
|
3011
|
+
("polyctl.commit.tree", itree),
|
|
3012
|
+
("polyctl.gates.count", len(gates)),
|
|
3013
|
+
("polyctl.gates.passed_count", passed_gates),
|
|
3014
|
+
("polyctl.gates.failed_count", failed_gates),
|
|
3015
|
+
("polyctl.tests.pass_count", pass_tests),
|
|
3016
|
+
("polyctl.tests.fail_count", fail_tests),
|
|
3017
|
+
("polyctl.candidate.changed_count", changed_count),
|
|
3018
|
+
],
|
|
3019
|
+
})
|
|
3020
|
+
|
|
3021
|
+
for row in reviews:
|
|
3022
|
+
record_digest = chained_row_digest(row, REVIEW_CHAIN)
|
|
3023
|
+
artefact = _otlp_allow_string("polyctl.review.id", os.path.basename(str(row.get("artefact") or "")))
|
|
3024
|
+
cdigest = _otlp_allow_string("polyctl.candidate", row.get("candidate")) or maps["review_to_candidate"].get(artefact)
|
|
3025
|
+
slice_id = (
|
|
3026
|
+
_otlp_allow_string("polyctl.slice", row.get("slice"))
|
|
3027
|
+
or maps["review_to_slice"].get(artefact)
|
|
3028
|
+
or maps["candidate_to_slice"].get(cdigest)
|
|
3029
|
+
)
|
|
3030
|
+
linked_grant = maps["grant_by_review"].get(artefact) or maps["grant_by_candidate"].get(cdigest) or {}
|
|
3031
|
+
verdict = _otlp_allow_string("polyctl.outcome", row.get("verdict")) or "INVALID"
|
|
3032
|
+
start_ns = _otlp_unix_nano(row.get("at"))
|
|
3033
|
+
reviewers_list = row.get("reviewers") if isinstance(row.get("reviewers"), list) else None
|
|
3034
|
+
pairs = [
|
|
3035
|
+
("polyctl.step", "review"),
|
|
3036
|
+
("polyctl.slice", slice_id),
|
|
3037
|
+
("polyctl.candidate", cdigest),
|
|
3038
|
+
("polyctl.review.id", artefact),
|
|
3039
|
+
("polyctl.review.kind", row.get("kind")),
|
|
3040
|
+
("polyctl.review.tier", row.get("tier")),
|
|
3041
|
+
("polyctl.outcome", verdict),
|
|
3042
|
+
("polyctl.authorizable", row.get("authorizable") if isinstance(row.get("authorizable"), bool) else None),
|
|
3043
|
+
("polyctl.reviewers.count", len(reviewers_list) if reviewers_list is not None else None),
|
|
3044
|
+
]
|
|
3045
|
+
pairs.extend(_otlp_actor_pairs(
|
|
3046
|
+
"polyctl.implementer",
|
|
3047
|
+
row.get("implementer")
|
|
3048
|
+
or linked_grant.get("implementer")
|
|
3049
|
+
or (linked_grant.get("role_bindings") or {}).get("implementer"),
|
|
3050
|
+
identity=(
|
|
3051
|
+
row.get("implementer_identity")
|
|
3052
|
+
or row.get("author")
|
|
3053
|
+
or linked_grant.get("implementer_identity")
|
|
3054
|
+
or linked_grant.get("author")
|
|
3055
|
+
),
|
|
3056
|
+
catalog=catalog,
|
|
3057
|
+
))
|
|
3058
|
+
pairs.extend(_otlp_actor_pairs(
|
|
3059
|
+
"polyctl.reviewer",
|
|
3060
|
+
row.get("reviewer") or linked_grant.get("reviewer"),
|
|
3061
|
+
identity=row.get("reviewer_identity") or linked_grant.get("reviewer_identity"),
|
|
3062
|
+
catalog=catalog,
|
|
3063
|
+
))
|
|
3064
|
+
steps.append({
|
|
3065
|
+
"step": "review",
|
|
3066
|
+
"record_digest": record_digest,
|
|
3067
|
+
"span_id": _otlp_hex_id(f"polyctl-otlp-span-v1:review:{record_digest}", 16),
|
|
3068
|
+
"slice": slice_id,
|
|
3069
|
+
"candidate": cdigest,
|
|
3070
|
+
"start_ns": start_ns,
|
|
3071
|
+
"end_ns": start_ns,
|
|
3072
|
+
"status_code": 1 if verdict == "APPROVED" else 2,
|
|
3073
|
+
"ref_id": artefact,
|
|
3074
|
+
"links": {"candidate": cdigest},
|
|
3075
|
+
"pairs": pairs,
|
|
3076
|
+
})
|
|
3077
|
+
|
|
3078
|
+
for row in grants:
|
|
3079
|
+
record_digest = chained_row_digest(row, AUTHORITY_CHAIN)
|
|
3080
|
+
info = _otlp_resolve_grant(row, maps)
|
|
3081
|
+
cand = info["cand"]
|
|
3082
|
+
itree = info["itree"]
|
|
3083
|
+
vid = info["vid"]
|
|
3084
|
+
artefact = info["artefact"]
|
|
3085
|
+
wid = info["wid"]
|
|
3086
|
+
cdigest = info["cdigest"]
|
|
3087
|
+
slice_id = info["slice_id"]
|
|
3088
|
+
review_row = info["review_row"] or {}
|
|
3089
|
+
granted = bool(row.get("granted") is True)
|
|
3090
|
+
start_ns = _otlp_unix_nano(row.get("at"))
|
|
3091
|
+
changed_count = _otlp_changed_count(cand)
|
|
3092
|
+
reasons = row.get("reasons") if isinstance(row.get("reasons"), list) else []
|
|
3093
|
+
pairs = [
|
|
3094
|
+
("polyctl.step", "grant"),
|
|
3095
|
+
("polyctl.slice", slice_id),
|
|
3096
|
+
("polyctl.candidate", cdigest),
|
|
3097
|
+
("polyctl.verification.id", vid),
|
|
3098
|
+
("polyctl.review.id", artefact),
|
|
3099
|
+
("polyctl.waiver.id", wid),
|
|
3100
|
+
("polyctl.review.tier", row.get("tier") or review_row.get("tier")),
|
|
3101
|
+
("polyctl.outcome", "GRANTED" if granted else "REFUSED"),
|
|
3102
|
+
("polyctl.granted", granted),
|
|
3103
|
+
("polyctl.commit.tree", itree),
|
|
3104
|
+
("polyctl.candidate.changed_count", changed_count),
|
|
3105
|
+
("polyctl.reasons.count", len(reasons)),
|
|
3106
|
+
]
|
|
3107
|
+
pairs.extend(_otlp_actor_pairs(
|
|
3108
|
+
"polyctl.implementer",
|
|
3109
|
+
row.get("implementer") or (row.get("role_bindings") or {}).get("implementer"),
|
|
3110
|
+
identity=(
|
|
3111
|
+
row.get("implementer_identity")
|
|
3112
|
+
or row.get("author")
|
|
3113
|
+
or review_row.get("implementer_identity")
|
|
3114
|
+
or review_row.get("author")
|
|
3115
|
+
),
|
|
3116
|
+
catalog=catalog,
|
|
3117
|
+
))
|
|
3118
|
+
pairs.extend(_otlp_actor_pairs(
|
|
3119
|
+
"polyctl.reviewer",
|
|
3120
|
+
row.get("reviewer") or review_row.get("reviewer"),
|
|
3121
|
+
identity=row.get("reviewer_identity") or review_row.get("reviewer_identity"),
|
|
3122
|
+
catalog=catalog,
|
|
3123
|
+
))
|
|
3124
|
+
steps.append({
|
|
3125
|
+
"step": "grant",
|
|
3126
|
+
"record_digest": record_digest,
|
|
3127
|
+
"span_id": _otlp_hex_id(f"polyctl-otlp-span-v1:grant:{record_digest}", 16),
|
|
3128
|
+
"slice": slice_id,
|
|
3129
|
+
"candidate": cdigest,
|
|
3130
|
+
"start_ns": start_ns,
|
|
3131
|
+
"end_ns": start_ns,
|
|
3132
|
+
"status_code": 1 if granted else 2,
|
|
3133
|
+
"ref_id": record_digest,
|
|
3134
|
+
"grant_token": str(row.get("token") or "").strip() or None,
|
|
3135
|
+
"grant_tree": itree if granted else None,
|
|
3136
|
+
"links": {"review": artefact, "verification": vid, "candidate": cdigest},
|
|
3137
|
+
"pairs": pairs,
|
|
3138
|
+
})
|
|
3139
|
+
|
|
3140
|
+
for commit in commits:
|
|
3141
|
+
info = _otlp_resolve_commit(commit, maps)
|
|
3142
|
+
if not info["relevant"]:
|
|
3143
|
+
continue
|
|
3144
|
+
csha = info["csha"]
|
|
3145
|
+
ctree = info["ctree"]
|
|
3146
|
+
trailers = info["trailers"]
|
|
3147
|
+
grant_row = info["grant_row"] or {}
|
|
3148
|
+
review_row = info["review_row"] or {}
|
|
3149
|
+
vid = info["vid"]
|
|
3150
|
+
artefact = info["artefact"]
|
|
3151
|
+
wid = info["wid"]
|
|
3152
|
+
cdigest = info["cdigest"]
|
|
3153
|
+
slice_id = info["slice_id"]
|
|
3154
|
+
record_digest = chained_row_digest(
|
|
3155
|
+
{"sha": commit.get("sha"), "tree": commit.get("tree"), "at": commit.get("at")},
|
|
3156
|
+
_OTLP_COMMIT_CHAIN,
|
|
3157
|
+
)
|
|
3158
|
+
start_ns = _otlp_unix_nano(commit.get("at"))
|
|
3159
|
+
pairs = [
|
|
3160
|
+
("polyctl.step", "commit"),
|
|
3161
|
+
("polyctl.slice", slice_id),
|
|
3162
|
+
("polyctl.candidate", cdigest),
|
|
3163
|
+
("polyctl.commit.sha", csha),
|
|
3164
|
+
("polyctl.commit.tree", ctree),
|
|
3165
|
+
("polyctl.verification.id", vid),
|
|
3166
|
+
("polyctl.review.id", artefact),
|
|
3167
|
+
("polyctl.waiver.id", wid),
|
|
3168
|
+
("polyctl.outcome", "LANDED"),
|
|
3169
|
+
]
|
|
3170
|
+
pairs.extend(_otlp_actor_pairs(
|
|
3171
|
+
"polyctl.implementer",
|
|
3172
|
+
trailers.get("author")
|
|
3173
|
+
or grant_row.get("implementer")
|
|
3174
|
+
or (grant_row.get("role_bindings") or {}).get("implementer"),
|
|
3175
|
+
identity=(
|
|
3176
|
+
grant_row.get("implementer_identity")
|
|
3177
|
+
or grant_row.get("author")
|
|
3178
|
+
or review_row.get("implementer_identity")
|
|
3179
|
+
or review_row.get("author")
|
|
3180
|
+
),
|
|
3181
|
+
catalog=catalog,
|
|
3182
|
+
))
|
|
3183
|
+
pairs.extend(_otlp_actor_pairs(
|
|
3184
|
+
"polyctl.reviewer",
|
|
3185
|
+
trailers.get("reviewer") or grant_row.get("reviewer") or review_row.get("reviewer"),
|
|
3186
|
+
identity=grant_row.get("reviewer_identity") or review_row.get("reviewer_identity"),
|
|
3187
|
+
catalog=catalog,
|
|
3188
|
+
))
|
|
3189
|
+
steps.append({
|
|
3190
|
+
"step": "commit",
|
|
3191
|
+
"record_digest": record_digest,
|
|
3192
|
+
"span_id": _otlp_hex_id(f"polyctl-otlp-span-v1:commit:{record_digest}", 16),
|
|
3193
|
+
"slice": slice_id,
|
|
3194
|
+
"candidate": cdigest,
|
|
3195
|
+
"start_ns": start_ns,
|
|
3196
|
+
"end_ns": start_ns,
|
|
3197
|
+
"status_code": 1,
|
|
3198
|
+
"ref_id": csha,
|
|
3199
|
+
"links": {
|
|
3200
|
+
"grant_token": info["grant_token"],
|
|
3201
|
+
"grant_tree": ctree,
|
|
3202
|
+
"review": artefact,
|
|
3203
|
+
"verification": vid,
|
|
3204
|
+
},
|
|
3205
|
+
"pairs": pairs,
|
|
3206
|
+
})
|
|
3207
|
+
|
|
3208
|
+
grouped, group_order = {}, []
|
|
3209
|
+
for item in steps:
|
|
3210
|
+
if item["slice"]:
|
|
3211
|
+
key = f"slice:{item['slice']}"
|
|
3212
|
+
elif item["candidate"]:
|
|
3213
|
+
key = f"candidate:{item['candidate']}"
|
|
3214
|
+
else:
|
|
3215
|
+
key = f"step:{item['record_digest']}"
|
|
3216
|
+
if key not in grouped:
|
|
3217
|
+
grouped[key] = []
|
|
3218
|
+
group_order.append(key)
|
|
3219
|
+
grouped[key].append(item)
|
|
3220
|
+
|
|
3221
|
+
spans = []
|
|
3222
|
+
for key in group_order:
|
|
3223
|
+
# Same-second rows of one step keep the ledger's own append order; breaking ties by
|
|
3224
|
+
# record_digest put an earlier grant row after a later one whenever its hash sorted higher (CI-LINUX-0.7).
|
|
3225
|
+
group = sorted(
|
|
3226
|
+
grouped[key],
|
|
3227
|
+
key=lambda item: (item["start_ns"], _OTLP_STEP_RANK.get(item["step"], 9)),
|
|
3228
|
+
)
|
|
3229
|
+
trace_slice = next((item["slice"] for item in reversed(group) if item["slice"]), None)
|
|
3230
|
+
trace_candidate = next((item["candidate"] for item in reversed(group) if item["candidate"]), None)
|
|
3231
|
+
trace_seed = "|".join(
|
|
3232
|
+
[f"polyctl-otlp-trace-v1:{key}:{trace_candidate or ''}"]
|
|
3233
|
+
+ sorted(item["record_digest"] for item in group)
|
|
3234
|
+
)
|
|
3235
|
+
trace_id = _otlp_hex_id(trace_seed, 32)
|
|
3236
|
+
|
|
3237
|
+
verification_spans = {}
|
|
3238
|
+
review_spans = {}
|
|
3239
|
+
grant_by_token = {}
|
|
3240
|
+
grant_by_tree = {}
|
|
3241
|
+
root_span_id = group[0]["span_id"]
|
|
3242
|
+
|
|
3243
|
+
for index, item in enumerate(group):
|
|
3244
|
+
parent_span_id = None
|
|
3245
|
+
if index > 0:
|
|
3246
|
+
links = item.get("links") or {}
|
|
3247
|
+
if item["step"] == "commit":
|
|
3248
|
+
parent_span_id = (
|
|
3249
|
+
grant_by_token.get(links.get("grant_token"))
|
|
3250
|
+
or grant_by_tree.get(links.get("grant_tree"))
|
|
3251
|
+
or review_spans.get(links.get("review"))
|
|
3252
|
+
or verification_spans.get(links.get("verification"))
|
|
3253
|
+
or group[index - 1]["span_id"]
|
|
3254
|
+
)
|
|
3255
|
+
elif item["step"] == "grant":
|
|
3256
|
+
parent_span_id = (
|
|
3257
|
+
review_spans.get(links.get("review"))
|
|
3258
|
+
or verification_spans.get(links.get("verification"))
|
|
3259
|
+
or group[index - 1]["span_id"]
|
|
3260
|
+
)
|
|
3261
|
+
elif item["step"] == "review":
|
|
3262
|
+
parent_span_id = verification_spans.get(links.get("candidate")) or root_span_id
|
|
3263
|
+
else:
|
|
3264
|
+
parent_span_id = group[index - 1]["span_id"]
|
|
3265
|
+
if parent_span_id == item["span_id"]:
|
|
3266
|
+
parent_span_id = root_span_id if root_span_id != item["span_id"] else None
|
|
3267
|
+
|
|
3268
|
+
if item["step"] == "verification":
|
|
3269
|
+
if item.get("ref_id"):
|
|
3270
|
+
verification_spans[item["ref_id"]] = item["span_id"]
|
|
3271
|
+
if item.get("candidate"):
|
|
3272
|
+
verification_spans[item["candidate"]] = item["span_id"]
|
|
3273
|
+
elif item["step"] == "review" and item.get("ref_id"):
|
|
3274
|
+
review_spans[item["ref_id"]] = item["span_id"]
|
|
3275
|
+
elif item["step"] == "grant":
|
|
3276
|
+
if item.get("grant_token"):
|
|
3277
|
+
grant_by_token[item["grant_token"]] = item["span_id"]
|
|
3278
|
+
if item.get("grant_tree"):
|
|
3279
|
+
grant_by_tree[item["grant_tree"]] = item["span_id"]
|
|
3280
|
+
|
|
3281
|
+
enriched_pairs = []
|
|
3282
|
+
for attr_key, attr_val in item["pairs"]:
|
|
3283
|
+
if attr_key == "polyctl.slice" and attr_val is None:
|
|
3284
|
+
attr_val = trace_slice
|
|
3285
|
+
elif attr_key == "polyctl.candidate" and attr_val is None:
|
|
3286
|
+
attr_val = trace_candidate
|
|
3287
|
+
enriched_pairs.append((attr_key, attr_val))
|
|
3288
|
+
|
|
3289
|
+
span = {
|
|
3290
|
+
"traceId": trace_id,
|
|
3291
|
+
"spanId": item["span_id"],
|
|
3292
|
+
"name": item["step"],
|
|
3293
|
+
"kind": 1,
|
|
3294
|
+
"startTimeUnixNano": str(item["start_ns"]),
|
|
3295
|
+
"endTimeUnixNano": str(item["end_ns"]),
|
|
3296
|
+
"attributes": _otlp_attributes(
|
|
3297
|
+
enriched_pairs, declared_families=declared_families, declared_profiles=declared_profiles
|
|
3298
|
+
),
|
|
3299
|
+
"status": {"code": item["status_code"]},
|
|
3300
|
+
}
|
|
3301
|
+
if parent_span_id:
|
|
3302
|
+
span["parentSpanId"] = parent_span_id
|
|
3303
|
+
spans.append(span)
|
|
3304
|
+
|
|
3305
|
+
resource_pairs = [
|
|
3306
|
+
("service.name", "polyctl"),
|
|
3307
|
+
("service.version", __version__),
|
|
3308
|
+
]
|
|
3309
|
+
if attested:
|
|
3310
|
+
resource_pairs.append(("polyctl.export.attested_only", True))
|
|
3311
|
+
resource_pairs.append(("polyctl.export.partial", bool(unattested_list)))
|
|
3312
|
+
if unattested_list:
|
|
3313
|
+
resource_pairs.append((
|
|
3314
|
+
"polyctl.export.unattested_ledgers",
|
|
3315
|
+
",".join(item["ledger"] for item in unattested_list),
|
|
3316
|
+
))
|
|
3317
|
+
|
|
3318
|
+
return {
|
|
3319
|
+
"resourceSpans": [
|
|
3320
|
+
{
|
|
3321
|
+
"resource": {
|
|
3322
|
+
"attributes": _otlp_attributes(
|
|
3323
|
+
resource_pairs, declared_families=declared_families, declared_profiles=declared_profiles
|
|
3324
|
+
),
|
|
3325
|
+
},
|
|
3326
|
+
"scopeSpans": [
|
|
3327
|
+
{
|
|
3328
|
+
"scope": {
|
|
3329
|
+
"name": "polyctl",
|
|
3330
|
+
"version": __version__,
|
|
3331
|
+
},
|
|
3332
|
+
"spans": spans,
|
|
3333
|
+
}
|
|
3334
|
+
],
|
|
3335
|
+
}
|
|
3336
|
+
]
|
|
3337
|
+
}
|