start-command 0.42.0 → 0.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/package.json +2 -2
- package/src/lib/formal-ai-client.js +8 -5
- package/src/lib/formal-ai-files.js +474 -61
- package/src/lib/formal-ai.js +170 -7
- package/src/lib/log-sanitizer.js +4 -0
- package/test/fixtures/formal-ai/receipts-258.json +27 -9
- package/test/fixtures/formal-ai/receipts-261.json +589 -0
- package/test/fixtures/formal-ai/replay.cjs +4 -1
- package/test/regression-258-files.js +27 -5
- package/test/regression-258.js +22 -9
- package/test/regression-261.js +856 -0
|
@@ -0,0 +1,589 @@
|
|
|
1
|
+
{
|
|
2
|
+
"source": "captured from Formal AI 0.355.32 (08ca27d1ee0560fb56450ad798f3a79c684404ed) js agent mode by experiments/issue-261/capture-receipts.cjs",
|
|
3
|
+
"capturedAt": "2026-11-04",
|
|
4
|
+
"receipts": [
|
|
5
|
+
{
|
|
6
|
+
"name": "issue reproduction",
|
|
7
|
+
"request": "In draft.txt, replace «draft» with «verified».",
|
|
8
|
+
"effects": "workspace-write",
|
|
9
|
+
"files": {
|
|
10
|
+
"draft.txt": "draft\n",
|
|
11
|
+
"verified-source.txt": "verified\n"
|
|
12
|
+
},
|
|
13
|
+
"links": {},
|
|
14
|
+
"offered": ["bash", "read", "edit", "write"],
|
|
15
|
+
"responses": [
|
|
16
|
+
{
|
|
17
|
+
"finish_reason": "tool_calls",
|
|
18
|
+
"message": {
|
|
19
|
+
"role": "assistant",
|
|
20
|
+
"content": "Let me open draft.txt and read what it says.",
|
|
21
|
+
"tool_calls": [
|
|
22
|
+
{
|
|
23
|
+
"id": "call_2c33961d84b9a31d",
|
|
24
|
+
"type": "function",
|
|
25
|
+
"function": {
|
|
26
|
+
"name": "read",
|
|
27
|
+
"arguments": "{\"path\":\"draft.txt\"}"
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
]
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
"finish_reason": "tool_calls",
|
|
35
|
+
"message": {
|
|
36
|
+
"role": "assistant",
|
|
37
|
+
"content": "Let me update draft.txt for you.",
|
|
38
|
+
"tool_calls": [
|
|
39
|
+
{
|
|
40
|
+
"id": "call_7469658d98e59005",
|
|
41
|
+
"type": "function",
|
|
42
|
+
"function": {
|
|
43
|
+
"name": "edit",
|
|
44
|
+
"arguments": "{\"new_string\":\"verified\",\"old_string\":\"draft\",\"path\":\"draft.txt\"}"
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
]
|
|
48
|
+
}
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
"finish_reason": "tool_calls",
|
|
52
|
+
"message": {
|
|
53
|
+
"role": "assistant",
|
|
54
|
+
"content": "Let me run the requested command to get that for you.",
|
|
55
|
+
"tool_calls": [
|
|
56
|
+
{
|
|
57
|
+
"id": "call_b3ddcc9ecf22728b",
|
|
58
|
+
"type": "function",
|
|
59
|
+
"function": {
|
|
60
|
+
"name": "bash",
|
|
61
|
+
"arguments": "{\"command\":\"sha256sum -- draft.txt\"}"
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
]
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
"finish_reason": "stop",
|
|
69
|
+
"message": {
|
|
70
|
+
"role": "assistant",
|
|
71
|
+
"content": "Replaced `draft` with `verified` in `draft.txt` and observed the result."
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
],
|
|
75
|
+
"tools": [
|
|
76
|
+
{
|
|
77
|
+
"tool": "read",
|
|
78
|
+
"command": "read draft.txt",
|
|
79
|
+
"decision": "allowed",
|
|
80
|
+
"exitCode": 0
|
|
81
|
+
},
|
|
82
|
+
{
|
|
83
|
+
"tool": "edit",
|
|
84
|
+
"command": "edit draft.txt",
|
|
85
|
+
"decision": "allowed",
|
|
86
|
+
"exitCode": 0
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
"tool": "bash",
|
|
90
|
+
"command": "sha256sum -- draft.txt",
|
|
91
|
+
"decision": "allowed",
|
|
92
|
+
"exitCode": 0
|
|
93
|
+
}
|
|
94
|
+
],
|
|
95
|
+
"outcome": {
|
|
96
|
+
"status": "completed",
|
|
97
|
+
"exitCode": 0,
|
|
98
|
+
"message": "finished after 3 tool call(s); changed draft.txt",
|
|
99
|
+
"remainingObligations": []
|
|
100
|
+
},
|
|
101
|
+
"after": {
|
|
102
|
+
"draft.txt": "verified\n",
|
|
103
|
+
"verified-source.txt": "verified\n"
|
|
104
|
+
}
|
|
105
|
+
},
|
|
106
|
+
{
|
|
107
|
+
"name": "issue reproduction, read-only",
|
|
108
|
+
"request": "In draft.txt, replace «draft» with «verified».",
|
|
109
|
+
"effects": "read-only",
|
|
110
|
+
"files": {
|
|
111
|
+
"draft.txt": "draft\n",
|
|
112
|
+
"verified-source.txt": "verified\n"
|
|
113
|
+
},
|
|
114
|
+
"links": {},
|
|
115
|
+
"offered": ["bash", "read"],
|
|
116
|
+
"responses": [
|
|
117
|
+
{
|
|
118
|
+
"finish_reason": "tool_calls",
|
|
119
|
+
"message": {
|
|
120
|
+
"role": "assistant",
|
|
121
|
+
"content": "Let me open draft.txt and read what it says.",
|
|
122
|
+
"tool_calls": [
|
|
123
|
+
{
|
|
124
|
+
"id": "call_2c33961d84b9a31d",
|
|
125
|
+
"type": "function",
|
|
126
|
+
"function": {
|
|
127
|
+
"name": "read",
|
|
128
|
+
"arguments": "{\"path\":\"draft.txt\"}"
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
]
|
|
132
|
+
}
|
|
133
|
+
},
|
|
134
|
+
{
|
|
135
|
+
"finish_reason": "stop",
|
|
136
|
+
"message": {
|
|
137
|
+
"role": "assistant",
|
|
138
|
+
"content": "{\"complete\":false,\"missingCapabilities\":[\"edit\"],\"reason\":\"EditToolUnavailable\",\"schema\":\"workspace-edit-capability-gap/v1\",\"target\":\"draft.txt\",\"verified\":false}"
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
],
|
|
142
|
+
"tools": [
|
|
143
|
+
{
|
|
144
|
+
"tool": "read",
|
|
145
|
+
"command": "read draft.txt",
|
|
146
|
+
"decision": "allowed",
|
|
147
|
+
"exitCode": 0
|
|
148
|
+
}
|
|
149
|
+
],
|
|
150
|
+
"outcome": {
|
|
151
|
+
"status": "capability-gap",
|
|
152
|
+
"exitCode": 65,
|
|
153
|
+
"message": "Formal AI reported a capability gap for draft.txt (EditToolUnavailable; missing edit); nothing was verified as done",
|
|
154
|
+
"remainingObligations": [
|
|
155
|
+
"the requested change to draft.txt was not made or not verified",
|
|
156
|
+
"set START_FORMAL_AI_EFFECTS=workspace-write to offer the edit and write tools"
|
|
157
|
+
],
|
|
158
|
+
"gap": {
|
|
159
|
+
"schema": "workspace-edit-capability-gap/v1",
|
|
160
|
+
"target": "draft.txt",
|
|
161
|
+
"reason": "EditToolUnavailable",
|
|
162
|
+
"missingCapabilities": ["edit"],
|
|
163
|
+
"complete": false,
|
|
164
|
+
"verified": false
|
|
165
|
+
}
|
|
166
|
+
},
|
|
167
|
+
"after": {
|
|
168
|
+
"draft.txt": "draft\n",
|
|
169
|
+
"verified-source.txt": "verified\n"
|
|
170
|
+
}
|
|
171
|
+
},
|
|
172
|
+
{
|
|
173
|
+
"name": "mismatched OLD",
|
|
174
|
+
"request": "In mismatch.txt, replace «drafted» with «verified».",
|
|
175
|
+
"effects": "workspace-write",
|
|
176
|
+
"files": {
|
|
177
|
+
"mismatch.txt": "draft\n",
|
|
178
|
+
"verified-source.txt": "verified\n"
|
|
179
|
+
},
|
|
180
|
+
"links": {},
|
|
181
|
+
"offered": ["bash", "read", "edit", "write"],
|
|
182
|
+
"responses": [
|
|
183
|
+
{
|
|
184
|
+
"finish_reason": "tool_calls",
|
|
185
|
+
"message": {
|
|
186
|
+
"role": "assistant",
|
|
187
|
+
"content": "Let me open mismatch.txt and read what it says.",
|
|
188
|
+
"tool_calls": [
|
|
189
|
+
{
|
|
190
|
+
"id": "call_fd4368a9e9ad0e50",
|
|
191
|
+
"type": "function",
|
|
192
|
+
"function": {
|
|
193
|
+
"name": "read",
|
|
194
|
+
"arguments": "{\"path\":\"mismatch.txt\"}"
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
]
|
|
198
|
+
}
|
|
199
|
+
},
|
|
200
|
+
{
|
|
201
|
+
"finish_reason": "stop",
|
|
202
|
+
"message": {
|
|
203
|
+
"role": "assistant",
|
|
204
|
+
"content": "`drafted` does not occur in `mismatch.txt`, so nothing was changed."
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
],
|
|
208
|
+
"tools": [
|
|
209
|
+
{
|
|
210
|
+
"tool": "read",
|
|
211
|
+
"command": "read mismatch.txt",
|
|
212
|
+
"decision": "allowed",
|
|
213
|
+
"exitCode": 0
|
|
214
|
+
}
|
|
215
|
+
],
|
|
216
|
+
"outcome": {
|
|
217
|
+
"status": "completed",
|
|
218
|
+
"exitCode": 0,
|
|
219
|
+
"message": "finished after 1 tool call(s); no file changed",
|
|
220
|
+
"remainingObligations": []
|
|
221
|
+
},
|
|
222
|
+
"after": {
|
|
223
|
+
"mismatch.txt": "draft\n",
|
|
224
|
+
"verified-source.txt": "verified\n"
|
|
225
|
+
}
|
|
226
|
+
},
|
|
227
|
+
{
|
|
228
|
+
"name": "repeated occurrences",
|
|
229
|
+
"request": "In twice.txt, replace «draft» with «verified».",
|
|
230
|
+
"effects": "workspace-write",
|
|
231
|
+
"files": {
|
|
232
|
+
"twice.txt": "draft one\ndraft two\n"
|
|
233
|
+
},
|
|
234
|
+
"links": {},
|
|
235
|
+
"offered": ["bash", "read", "edit", "write"],
|
|
236
|
+
"responses": [
|
|
237
|
+
{
|
|
238
|
+
"finish_reason": "tool_calls",
|
|
239
|
+
"message": {
|
|
240
|
+
"role": "assistant",
|
|
241
|
+
"content": "Let me open twice.txt and read what it says.",
|
|
242
|
+
"tool_calls": [
|
|
243
|
+
{
|
|
244
|
+
"id": "call_08ec81c4f930c1cd",
|
|
245
|
+
"type": "function",
|
|
246
|
+
"function": {
|
|
247
|
+
"name": "read",
|
|
248
|
+
"arguments": "{\"path\":\"twice.txt\"}"
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
]
|
|
252
|
+
}
|
|
253
|
+
},
|
|
254
|
+
{
|
|
255
|
+
"finish_reason": "tool_calls",
|
|
256
|
+
"message": {
|
|
257
|
+
"role": "assistant",
|
|
258
|
+
"content": "Let me run the requested command to get that for you.",
|
|
259
|
+
"tool_calls": [
|
|
260
|
+
{
|
|
261
|
+
"id": "call_c93ac09bdb199918",
|
|
262
|
+
"type": "function",
|
|
263
|
+
"function": {
|
|
264
|
+
"name": "bash",
|
|
265
|
+
"arguments": "{\"command\":\"perl -pi -e 's/draft/verified/g' -- twice.txt\"}"
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
]
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
],
|
|
272
|
+
"tools": [
|
|
273
|
+
{
|
|
274
|
+
"tool": "read",
|
|
275
|
+
"command": "read twice.txt",
|
|
276
|
+
"decision": "allowed",
|
|
277
|
+
"exitCode": 0
|
|
278
|
+
},
|
|
279
|
+
{
|
|
280
|
+
"tool": "bash",
|
|
281
|
+
"command": "perl -pi -e 's/draft/verified/g' -- twice.txt",
|
|
282
|
+
"decision": "refused",
|
|
283
|
+
"exitCode": null
|
|
284
|
+
}
|
|
285
|
+
],
|
|
286
|
+
"outcome": {
|
|
287
|
+
"status": "refused",
|
|
288
|
+
"exitCode": 77,
|
|
289
|
+
"message": "refused `perl -pi -e 's/draft/verified/g' -- twice.txt`: program \"perl\" is not in the supported tool set",
|
|
290
|
+
"remainingObligations": [
|
|
291
|
+
"the refused call did not run; nothing after it ran either"
|
|
292
|
+
]
|
|
293
|
+
},
|
|
294
|
+
"after": {
|
|
295
|
+
"twice.txt": "draft one\ndraft two\n"
|
|
296
|
+
}
|
|
297
|
+
},
|
|
298
|
+
{
|
|
299
|
+
"name": "Unicode and CRLF",
|
|
300
|
+
"request": "In unicode.txt, replace «café» with «naïve».",
|
|
301
|
+
"effects": "workspace-write",
|
|
302
|
+
"files": {
|
|
303
|
+
"unicode.txt": "Ünïcödé café ☕\r\nsecond line 🙂\r\nno final newline"
|
|
304
|
+
},
|
|
305
|
+
"links": {},
|
|
306
|
+
"offered": ["bash", "read", "edit", "write"],
|
|
307
|
+
"responses": [
|
|
308
|
+
{
|
|
309
|
+
"finish_reason": "tool_calls",
|
|
310
|
+
"message": {
|
|
311
|
+
"role": "assistant",
|
|
312
|
+
"content": "Let me open unicode.txt and read what it says.",
|
|
313
|
+
"tool_calls": [
|
|
314
|
+
{
|
|
315
|
+
"id": "call_751c11cbd746b338",
|
|
316
|
+
"type": "function",
|
|
317
|
+
"function": {
|
|
318
|
+
"name": "read",
|
|
319
|
+
"arguments": "{\"path\":\"unicode.txt\"}"
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
]
|
|
323
|
+
}
|
|
324
|
+
},
|
|
325
|
+
{
|
|
326
|
+
"finish_reason": "tool_calls",
|
|
327
|
+
"message": {
|
|
328
|
+
"role": "assistant",
|
|
329
|
+
"content": "Let me update unicode.txt for you.",
|
|
330
|
+
"tool_calls": [
|
|
331
|
+
{
|
|
332
|
+
"id": "call_68431c4da851607b",
|
|
333
|
+
"type": "function",
|
|
334
|
+
"function": {
|
|
335
|
+
"name": "edit",
|
|
336
|
+
"arguments": "{\"new_string\":\"naïve\",\"old_string\":\"café\",\"path\":\"unicode.txt\"}"
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
]
|
|
340
|
+
}
|
|
341
|
+
},
|
|
342
|
+
{
|
|
343
|
+
"finish_reason": "tool_calls",
|
|
344
|
+
"message": {
|
|
345
|
+
"role": "assistant",
|
|
346
|
+
"content": "Let me run the requested command to get that for you.",
|
|
347
|
+
"tool_calls": [
|
|
348
|
+
{
|
|
349
|
+
"id": "call_b1f11e74e8e7bd46",
|
|
350
|
+
"type": "function",
|
|
351
|
+
"function": {
|
|
352
|
+
"name": "bash",
|
|
353
|
+
"arguments": "{\"command\":\"sha256sum -- unicode.txt\"}"
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
]
|
|
357
|
+
}
|
|
358
|
+
},
|
|
359
|
+
{
|
|
360
|
+
"finish_reason": "stop",
|
|
361
|
+
"message": {
|
|
362
|
+
"role": "assistant",
|
|
363
|
+
"content": "Replaced `café` with `naïve` in `unicode.txt` and observed the result."
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
],
|
|
367
|
+
"tools": [
|
|
368
|
+
{
|
|
369
|
+
"tool": "read",
|
|
370
|
+
"command": "read unicode.txt",
|
|
371
|
+
"decision": "allowed",
|
|
372
|
+
"exitCode": 0
|
|
373
|
+
},
|
|
374
|
+
{
|
|
375
|
+
"tool": "edit",
|
|
376
|
+
"command": "edit unicode.txt",
|
|
377
|
+
"decision": "allowed",
|
|
378
|
+
"exitCode": 0
|
|
379
|
+
},
|
|
380
|
+
{
|
|
381
|
+
"tool": "bash",
|
|
382
|
+
"command": "sha256sum -- unicode.txt",
|
|
383
|
+
"decision": "allowed",
|
|
384
|
+
"exitCode": 0
|
|
385
|
+
}
|
|
386
|
+
],
|
|
387
|
+
"outcome": {
|
|
388
|
+
"status": "completed",
|
|
389
|
+
"exitCode": 0,
|
|
390
|
+
"message": "finished after 3 tool call(s); changed unicode.txt",
|
|
391
|
+
"remainingObligations": []
|
|
392
|
+
},
|
|
393
|
+
"after": {
|
|
394
|
+
"unicode.txt": "Ünïcödé naïve ☕\r\nsecond line 🙂\r\nno final newline"
|
|
395
|
+
}
|
|
396
|
+
},
|
|
397
|
+
{
|
|
398
|
+
"name": "symlinked path",
|
|
399
|
+
"request": "In link.txt, replace «draft» with «verified».",
|
|
400
|
+
"effects": "workspace-write",
|
|
401
|
+
"files": {
|
|
402
|
+
"real.txt": "draft\n"
|
|
403
|
+
},
|
|
404
|
+
"links": {
|
|
405
|
+
"link.txt": "real.txt"
|
|
406
|
+
},
|
|
407
|
+
"offered": ["bash", "read", "edit", "write"],
|
|
408
|
+
"responses": [
|
|
409
|
+
{
|
|
410
|
+
"finish_reason": "tool_calls",
|
|
411
|
+
"message": {
|
|
412
|
+
"role": "assistant",
|
|
413
|
+
"content": "Let me open link.txt and read what it says.",
|
|
414
|
+
"tool_calls": [
|
|
415
|
+
{
|
|
416
|
+
"id": "call_f1b2a529ea88b28d",
|
|
417
|
+
"type": "function",
|
|
418
|
+
"function": {
|
|
419
|
+
"name": "read",
|
|
420
|
+
"arguments": "{\"path\":\"link.txt\"}"
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
]
|
|
424
|
+
}
|
|
425
|
+
},
|
|
426
|
+
{
|
|
427
|
+
"finish_reason": "tool_calls",
|
|
428
|
+
"message": {
|
|
429
|
+
"role": "assistant",
|
|
430
|
+
"content": "Let me update link.txt for you.",
|
|
431
|
+
"tool_calls": [
|
|
432
|
+
{
|
|
433
|
+
"id": "call_f4e6366385e51cc7",
|
|
434
|
+
"type": "function",
|
|
435
|
+
"function": {
|
|
436
|
+
"name": "edit",
|
|
437
|
+
"arguments": "{\"new_string\":\"verified\",\"old_string\":\"draft\",\"path\":\"link.txt\"}"
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
]
|
|
441
|
+
}
|
|
442
|
+
},
|
|
443
|
+
{
|
|
444
|
+
"finish_reason": "tool_calls",
|
|
445
|
+
"message": {
|
|
446
|
+
"role": "assistant",
|
|
447
|
+
"content": "Let me run the requested command to get that for you.",
|
|
448
|
+
"tool_calls": [
|
|
449
|
+
{
|
|
450
|
+
"id": "call_0e57dc5d4eebde9b",
|
|
451
|
+
"type": "function",
|
|
452
|
+
"function": {
|
|
453
|
+
"name": "bash",
|
|
454
|
+
"arguments": "{\"command\":\"sha256sum -- link.txt\"}"
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
]
|
|
458
|
+
}
|
|
459
|
+
},
|
|
460
|
+
{
|
|
461
|
+
"finish_reason": "stop",
|
|
462
|
+
"message": {
|
|
463
|
+
"role": "assistant",
|
|
464
|
+
"content": "Replaced `draft` with `verified` in `link.txt` and observed the result."
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
],
|
|
468
|
+
"tools": [
|
|
469
|
+
{
|
|
470
|
+
"tool": "read",
|
|
471
|
+
"command": "read link.txt",
|
|
472
|
+
"decision": "allowed",
|
|
473
|
+
"exitCode": 0
|
|
474
|
+
},
|
|
475
|
+
{
|
|
476
|
+
"tool": "edit",
|
|
477
|
+
"command": "edit link.txt",
|
|
478
|
+
"decision": "allowed",
|
|
479
|
+
"exitCode": 0
|
|
480
|
+
},
|
|
481
|
+
{
|
|
482
|
+
"tool": "bash",
|
|
483
|
+
"command": "sha256sum -- link.txt",
|
|
484
|
+
"decision": "allowed",
|
|
485
|
+
"exitCode": 0
|
|
486
|
+
}
|
|
487
|
+
],
|
|
488
|
+
"outcome": {
|
|
489
|
+
"status": "completed",
|
|
490
|
+
"exitCode": 0,
|
|
491
|
+
"message": "finished after 3 tool call(s); changed real.txt",
|
|
492
|
+
"remainingObligations": []
|
|
493
|
+
},
|
|
494
|
+
"after": {
|
|
495
|
+
"real.txt": "verified\n"
|
|
496
|
+
}
|
|
497
|
+
},
|
|
498
|
+
{
|
|
499
|
+
"name": "symlink leaving the workspace",
|
|
500
|
+
"request": "In escape.txt, replace «draft» with «verified».",
|
|
501
|
+
"effects": "workspace-write",
|
|
502
|
+
"files": {
|
|
503
|
+
"../outside.txt": "draft\n"
|
|
504
|
+
},
|
|
505
|
+
"links": {
|
|
506
|
+
"escape.txt": "../outside.txt"
|
|
507
|
+
},
|
|
508
|
+
"offered": ["bash", "read", "edit", "write"],
|
|
509
|
+
"responses": [
|
|
510
|
+
{
|
|
511
|
+
"finish_reason": "tool_calls",
|
|
512
|
+
"message": {
|
|
513
|
+
"role": "assistant",
|
|
514
|
+
"content": "Let me open escape.txt and read what it says.",
|
|
515
|
+
"tool_calls": [
|
|
516
|
+
{
|
|
517
|
+
"id": "call_f63acc2df7065c9f",
|
|
518
|
+
"type": "function",
|
|
519
|
+
"function": {
|
|
520
|
+
"name": "read",
|
|
521
|
+
"arguments": "{\"path\":\"escape.txt\"}"
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
]
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
],
|
|
528
|
+
"tools": [
|
|
529
|
+
{
|
|
530
|
+
"tool": "read",
|
|
531
|
+
"command": "read escape.txt",
|
|
532
|
+
"decision": "refused",
|
|
533
|
+
"exitCode": null
|
|
534
|
+
}
|
|
535
|
+
],
|
|
536
|
+
"outcome": {
|
|
537
|
+
"status": "refused",
|
|
538
|
+
"exitCode": 77,
|
|
539
|
+
"message": "refused `read escape.txt`: \"escape.txt\" is outside the working directory",
|
|
540
|
+
"remainingObligations": [
|
|
541
|
+
"the refused call did not run; nothing after it ran either"
|
|
542
|
+
]
|
|
543
|
+
},
|
|
544
|
+
"after": {
|
|
545
|
+
"../outside.txt": "draft\n"
|
|
546
|
+
}
|
|
547
|
+
},
|
|
548
|
+
{
|
|
549
|
+
"name": "foreign path",
|
|
550
|
+
"request": "In ../outside.txt, replace «draft» with «verified».",
|
|
551
|
+
"effects": "workspace-write",
|
|
552
|
+
"files": {
|
|
553
|
+
"draft.txt": "draft\n",
|
|
554
|
+
"verified-source.txt": "verified\n",
|
|
555
|
+
"../outside.txt": "draft\n"
|
|
556
|
+
},
|
|
557
|
+
"links": {},
|
|
558
|
+
"offered": ["bash", "read", "edit", "write"],
|
|
559
|
+
"responses": [
|
|
560
|
+
{
|
|
561
|
+
"finish_reason": "stop",
|
|
562
|
+
"message": {
|
|
563
|
+
"role": "assistant",
|
|
564
|
+
"content": "obligation_7d79209116af07c0\n record_type \"obligation_gap\"\n clause \"In .\"\n span_start \"0\"\n span_end \"4\"\n reason \"no_artifact_in_clause\""
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
],
|
|
568
|
+
"tools": [],
|
|
569
|
+
"outcome": {
|
|
570
|
+
"status": "incomplete",
|
|
571
|
+
"exitCode": 65,
|
|
572
|
+
"message": "Formal AI reported obligation_gap (no_artifact_in_clause) for the clause \"In .\"; nothing was verified as done by Formal AI; no file changed",
|
|
573
|
+
"remainingObligations": [
|
|
574
|
+
"Formal AI did not confirm the requested change"
|
|
575
|
+
],
|
|
576
|
+
"gapRecord": {
|
|
577
|
+
"schema": "obligation_gap",
|
|
578
|
+
"reason": "no_artifact_in_clause",
|
|
579
|
+
"clause": "In ."
|
|
580
|
+
}
|
|
581
|
+
},
|
|
582
|
+
"after": {
|
|
583
|
+
"draft.txt": "draft\n",
|
|
584
|
+
"verified-source.txt": "verified\n",
|
|
585
|
+
"../outside.txt": "draft\n"
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
]
|
|
589
|
+
}
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
* Receipts of issue #258 (receipts-258.json) also record the tool names start
|
|
13
13
|
* offered; they answer only a request offering exactly those tools, so the
|
|
14
14
|
* released Bash-only contract and the structured file tools replay apart.
|
|
15
|
+
* The held-out edits of issue #261 (receipts-261.json) replay the same way.
|
|
15
16
|
*/
|
|
16
17
|
|
|
17
18
|
const fs = require('fs');
|
|
@@ -22,6 +23,7 @@ const load = (name) =>
|
|
|
22
23
|
JSON.parse(fs.readFileSync(path.join(__dirname, name), 'utf8'));
|
|
23
24
|
const { receipts } = load('receipts.json');
|
|
24
25
|
const edits = load('receipts-258.json');
|
|
26
|
+
const heldOut = load('receipts-261.json');
|
|
25
27
|
|
|
26
28
|
/**
|
|
27
29
|
* The HTTP 500 body a released start discarded (issue #257), captured from a
|
|
@@ -113,7 +115,7 @@ function reply(body) {
|
|
|
113
115
|
const offered = (Array.isArray(body.tools) ? body.tools : [])
|
|
114
116
|
.map((tool) => tool && tool.function && tool.function.name)
|
|
115
117
|
.join(',');
|
|
116
|
-
const edit = edits.receipts.find(
|
|
118
|
+
const edit = [...edits.receipts, ...heldOut.receipts].find(
|
|
117
119
|
(r) => r.request === request && r.offered.join(',') === offered
|
|
118
120
|
);
|
|
119
121
|
if (edit && edit.responses[turns]) {
|
|
@@ -189,6 +191,7 @@ module.exports = {
|
|
|
189
191
|
SYNTHETIC,
|
|
190
192
|
createFakeServer,
|
|
191
193
|
edits,
|
|
194
|
+
heldOut,
|
|
192
195
|
receipts,
|
|
193
196
|
reply,
|
|
194
197
|
};
|
|
@@ -11,6 +11,7 @@ const {
|
|
|
11
11
|
EXIT,
|
|
12
12
|
} = require('../src/lib/formal-ai-policy');
|
|
13
13
|
const files = require('../src/lib/formal-ai-files');
|
|
14
|
+
const { readFrame, readFrameText } = files;
|
|
14
15
|
const { runFormalAiTask } = require('../src/lib/formal-ai');
|
|
15
16
|
const {
|
|
16
17
|
cleanup,
|
|
@@ -144,11 +145,14 @@ describe('read (#258)', () => {
|
|
|
144
145
|
const cwd = workspace();
|
|
145
146
|
const ctx = context(cwd, 'read-only');
|
|
146
147
|
const receipt = call('read', { path: './a.js' }, ctx);
|
|
148
|
+
// Formal AI binds a read as complete only in its numbered frame (#261).
|
|
147
149
|
expect(receipt).toMatchObject({
|
|
148
150
|
exitCode: 0,
|
|
149
|
-
|
|
151
|
+
text: TEXT,
|
|
152
|
+
result: readFrame(TEXT),
|
|
150
153
|
bytes: Buffer.byteLength(TEXT),
|
|
151
154
|
});
|
|
155
|
+
expect(readFrameText(receipt.result)).toBe(TEXT);
|
|
152
156
|
expect(ctx.observed.has('a.js')).toBe(true);
|
|
153
157
|
expect(ctx.contents.get('a.js')).toBe(receipt.sha256);
|
|
154
158
|
});
|
|
@@ -198,10 +202,16 @@ describe('edit (#258)', () => {
|
|
|
198
202
|
);
|
|
199
203
|
expect(receipt).toMatchObject({
|
|
200
204
|
exitCode: 0,
|
|
201
|
-
result: '',
|
|
202
205
|
replacements: 1,
|
|
203
206
|
verification: { verified: true },
|
|
204
207
|
});
|
|
208
|
+
expect(JSON.parse(receipt.result)).toMatchObject({
|
|
209
|
+
ok: true,
|
|
210
|
+
path: 'a.js',
|
|
211
|
+
tool: 'edit',
|
|
212
|
+
replacements: 1,
|
|
213
|
+
verified: true,
|
|
214
|
+
});
|
|
205
215
|
expect(fs.readFileSync(file, 'utf8')).toBe(
|
|
206
216
|
TEXT.replace('const b = 2;', () => 'const b = $&3;')
|
|
207
217
|
);
|
|
@@ -215,7 +225,7 @@ describe('edit (#258)', () => {
|
|
|
215
225
|
it('keeps a byte order mark', () => {
|
|
216
226
|
const cwd = workspace({ 'bom.txt': '\ufefffirst\nsecond\n' });
|
|
217
227
|
const ctx = context(cwd);
|
|
218
|
-
expect(call('read', { path: 'bom.txt' }, ctx).
|
|
228
|
+
expect(call('read', { path: 'bom.txt' }, ctx).text).toBe(
|
|
219
229
|
'\ufefffirst\nsecond\n'
|
|
220
230
|
);
|
|
221
231
|
const receipt = call(
|
|
@@ -420,10 +430,22 @@ describe('file tools in the task loop (#258)', () => {
|
|
|
420
430
|
},
|
|
421
431
|
{ content: 'The edit failed.' },
|
|
422
432
|
]);
|
|
433
|
+
// The failure stays unresolved with its original error (#261).
|
|
423
434
|
expect(result.evidence.outcome).toMatchObject({
|
|
424
|
-
status: '
|
|
435
|
+
status: 'incomplete',
|
|
425
436
|
exitCode: 1,
|
|
426
|
-
remainingObligations: [
|
|
437
|
+
remainingObligations: [
|
|
438
|
+
'`edit a.js` failed: a.js: old_string was not found; nothing changed',
|
|
439
|
+
],
|
|
440
|
+
failures: [
|
|
441
|
+
{
|
|
442
|
+
command: 'edit a.js',
|
|
443
|
+
tool: 'edit',
|
|
444
|
+
target: 'a.js',
|
|
445
|
+
exitCode: 1,
|
|
446
|
+
error: 'a.js: old_string was not found; nothing changed',
|
|
447
|
+
},
|
|
448
|
+
],
|
|
427
449
|
});
|
|
428
450
|
expect(result.after).toEqual(result.before);
|
|
429
451
|
});
|