simplicio-prompt 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +173 -0
- package/YOOL_TUPLE_HAMT.md +1149 -0
- package/adopters.md +24 -0
- package/benchmarks/generate_prompt_benchmark_pdf.py +355 -0
- package/benchmarks/generate_v2_benchmark_pdf.py +302 -0
- package/benchmarks/prompt_vs_normal.py +431 -0
- package/benchmarks/prompt_vs_normal_benchmark.pdf +124 -0
- package/benchmarks/prompt_vs_normal_results.md +148 -0
- package/benchmarks/v2_safe_speed_benchmark.pdf +118 -0
- package/benchmarks/v2_safe_speed_benchmark.py +626 -0
- package/benchmarks/v2_safe_speed_results.json +446 -0
- package/benchmarks/v2_safe_speed_results.md +96 -0
- package/docs/assets/simplicio-prompt-hero.png +0 -0
- package/docs/assets/yool-v2-safe-speed-infographic-en.png +0 -0
- package/docs/assets/yool-v2-safe-speed-infographic-pt.png +0 -0
- package/examples/node/build-catalog.mjs +70 -0
- package/examples/python/minimal_bus.py +134 -0
- package/examples/python/receipts.py +152 -0
- package/guardrails/cpu_throttle.py +119 -0
- package/guardrails/disk_gc.py +212 -0
- package/kernel/README.md +82 -0
- package/kernel/yool_tuple_kernel.py +1109 -0
- package/kernel-implementation-request.md +38 -0
- package/package.json +40 -0
- package/prompts/agent-runtime-execution-prompt.md +119 -0
- package/prompts/legacy-tuple-space-engine-prompt.md +36 -0
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
"""Benchmark normal instruction flow vs the Yool runtime prompt flow.
|
|
2
|
+
|
|
3
|
+
This does not call a hosted LLM. It measures the operational behavior that the
|
|
4
|
+
prompt requires from an executor:
|
|
5
|
+
|
|
6
|
+
- normal instruction: flat planning + sequential execution;
|
|
7
|
+
- yool prompt: lazy batch_spawn + lane worker fan-out + tuple receipts.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import math
|
|
14
|
+
import hashlib
|
|
15
|
+
import json
|
|
16
|
+
import statistics
|
|
17
|
+
import sys
|
|
18
|
+
import time
|
|
19
|
+
import tracemalloc
|
|
20
|
+
from dataclasses import asdict, dataclass
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any, Callable
|
|
23
|
+
|
|
24
|
+
ROOT = Path(__file__).resolve().parents[1]
|
|
25
|
+
sys.path.insert(0, str(ROOT))
|
|
26
|
+
|
|
27
|
+
from kernel.yool_tuple_kernel import ( # noqa: E402
|
|
28
|
+
LaneWorkerPool,
|
|
29
|
+
RuntimePolicy,
|
|
30
|
+
TupleSpace,
|
|
31
|
+
YoolTuple,
|
|
32
|
+
build_default_space,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
YOOL_PROMPT = """Use o repo canonico https://github.com/wesleysimplicio/simplicio-prompt.
|
|
36
|
+
Leia antes de editar: YOOL_TUPLE_HAMT.md, kernel/yool_tuple_kernel.py,
|
|
37
|
+
guardrails/cpu_throttle.py, guardrails/disk_gc.py, examples/python/receipts.py
|
|
38
|
+
e scripts/build_hamt.py.
|
|
39
|
+
|
|
40
|
+
Ao receber "Implement X": decomponha em grafo Hilbert-indexed, crie tuple raiz,
|
|
41
|
+
use batch_spawn(depth, branching, compression_threshold) para 1.000.000+
|
|
42
|
+
subagents sem enumerar, execute work ativo com spawn_agent, roteie por out/in/rd,
|
|
43
|
+
route_packet e scan_index, aplique hookwall, compress_token e prune_idle, e use
|
|
44
|
+
LaneWorkerPool respeitando YOOL_TUPLE_* env vars.
|
|
45
|
+
|
|
46
|
+
Execute:
|
|
47
|
+
python kernel/yool_tuple_kernel.py
|
|
48
|
+
|
|
49
|
+
Responda SEMPRE exatamente neste formato (sem variações):
|
|
50
|
+
[Tuple Space Snapshot]
|
|
51
|
+
[Active Agents/Subagents]
|
|
52
|
+
[Total Agents/Subagents]
|
|
53
|
+
[Próximo Yool a executar]
|
|
54
|
+
[Resultado parcial]"""
|
|
55
|
+
|
|
56
|
+
NORMAL_INSTRUCTION = (
|
|
57
|
+
"Implement X quickly. Use agents if useful. Report the result when finished."
|
|
58
|
+
)
|
|
59
|
+
NORMAL_REPEATED_CONTEXT = (
|
|
60
|
+
NORMAL_INSTRUCTION
|
|
61
|
+
+ "\nSubagent {index}: carry the orchestration rules, task goal, route, status, "
|
|
62
|
+
"and result summary in chat context."
|
|
63
|
+
)
|
|
64
|
+
YOOL_TUPLE_ENVELOPE = (
|
|
65
|
+
'{"yool":"prompt_worker","lane":"benchmark","args":{"index":0},'
|
|
66
|
+
'"receipts":[],"route":"tuple-space"}'
|
|
67
|
+
)
|
|
68
|
+
YOOL_BATCH_ENVELOPE = (
|
|
69
|
+
"batch_spawn(depth=4, branching=32, compression_threshold=1024, "
|
|
70
|
+
"virtual_agents=1048576)"
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass
|
|
75
|
+
class BenchmarkResult:
|
|
76
|
+
profile: str
|
|
77
|
+
phase: str
|
|
78
|
+
tasks: int
|
|
79
|
+
wall_ms: float
|
|
80
|
+
peak_kb: float
|
|
81
|
+
active_agents: int
|
|
82
|
+
compressed_agents: int
|
|
83
|
+
virtual_agents: int
|
|
84
|
+
total_agents: int
|
|
85
|
+
receipts: int
|
|
86
|
+
notes: str = ""
|
|
87
|
+
|
|
88
|
+
@property
|
|
89
|
+
def throughput_tasks_s(self) -> float:
|
|
90
|
+
if self.wall_ms <= 0:
|
|
91
|
+
return 0.0
|
|
92
|
+
return self.tasks / (self.wall_ms / 1000)
|
|
93
|
+
|
|
94
|
+
def to_dict(self) -> dict[str, Any]:
|
|
95
|
+
data = asdict(self)
|
|
96
|
+
data["throughput_tasks_s"] = self.throughput_tasks_s
|
|
97
|
+
return data
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass
|
|
101
|
+
class TokenModelResult:
|
|
102
|
+
scenario: str
|
|
103
|
+
normal_tokens: int
|
|
104
|
+
yool_tokens: int
|
|
105
|
+
savings_tokens: int
|
|
106
|
+
savings_pct: float
|
|
107
|
+
notes: str
|
|
108
|
+
|
|
109
|
+
def to_dict(self) -> dict[str, Any]:
|
|
110
|
+
return asdict(self)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def measure(fn: Callable[[], BenchmarkResult]) -> BenchmarkResult:
|
|
114
|
+
tracemalloc.start()
|
|
115
|
+
try:
|
|
116
|
+
result = fn()
|
|
117
|
+
_current, peak = tracemalloc.get_traced_memory()
|
|
118
|
+
result.peak_kb = peak / 1024
|
|
119
|
+
return result
|
|
120
|
+
finally:
|
|
121
|
+
tracemalloc.stop()
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def simulated_yool_work(index: int, sleep_ms: float) -> str:
|
|
125
|
+
if sleep_ms > 0:
|
|
126
|
+
time.sleep(sleep_ms / 1000)
|
|
127
|
+
payload = f"task:{index}".encode("utf-8")
|
|
128
|
+
return hashlib.blake2b(payload, digest_size=8).hexdigest()
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def estimate_tokens(text: str) -> int:
|
|
132
|
+
"""Deterministic rough estimate: about four UTF-8 chars per token."""
|
|
133
|
+
return max(1, math.ceil(len(text.encode("utf-8")) / 4))
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def normal_flat_scale(total_agents: int) -> BenchmarkResult:
|
|
137
|
+
def run() -> BenchmarkResult:
|
|
138
|
+
t0 = time.perf_counter()
|
|
139
|
+
agents = [
|
|
140
|
+
{"id": index, "yool": "normal_worker", "lane": "flat"}
|
|
141
|
+
for index in range(total_agents)
|
|
142
|
+
]
|
|
143
|
+
wall_ms = (time.perf_counter() - t0) * 1000
|
|
144
|
+
return BenchmarkResult(
|
|
145
|
+
profile="normal instruction",
|
|
146
|
+
phase="flat materialization",
|
|
147
|
+
tasks=total_agents,
|
|
148
|
+
wall_ms=wall_ms,
|
|
149
|
+
peak_kb=0.0,
|
|
150
|
+
active_agents=len(agents),
|
|
151
|
+
compressed_agents=0,
|
|
152
|
+
virtual_agents=0,
|
|
153
|
+
total_agents=len(agents),
|
|
154
|
+
receipts=0,
|
|
155
|
+
notes="flat Python list of subagents",
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
return measure(run)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def yool_lazy_scale(
|
|
162
|
+
total_agents: int, branching: int, threshold: int
|
|
163
|
+
) -> BenchmarkResult:
|
|
164
|
+
depth = 1
|
|
165
|
+
virtual = branching
|
|
166
|
+
while virtual < total_agents:
|
|
167
|
+
depth += 1
|
|
168
|
+
virtual *= branching
|
|
169
|
+
|
|
170
|
+
def run() -> BenchmarkResult:
|
|
171
|
+
space, root = build_default_space()
|
|
172
|
+
t0 = time.perf_counter()
|
|
173
|
+
receipt = space.batch_spawn(
|
|
174
|
+
root,
|
|
175
|
+
"prompt_worker",
|
|
176
|
+
depth=depth,
|
|
177
|
+
branching=branching,
|
|
178
|
+
compression_threshold=threshold,
|
|
179
|
+
)
|
|
180
|
+
wall_ms = (time.perf_counter() - t0) * 1000
|
|
181
|
+
snapshot = space.snapshot()
|
|
182
|
+
return BenchmarkResult(
|
|
183
|
+
profile="yool prompt",
|
|
184
|
+
phase="lazy batch_spawn",
|
|
185
|
+
tasks=receipt.virtual_agents,
|
|
186
|
+
wall_ms=wall_ms,
|
|
187
|
+
peak_kb=0.0,
|
|
188
|
+
active_agents=snapshot["active_agents"],
|
|
189
|
+
compressed_agents=snapshot["compressed_agents"],
|
|
190
|
+
virtual_agents=snapshot["virtual_agents"],
|
|
191
|
+
total_agents=snapshot["total_agents"],
|
|
192
|
+
receipts=1,
|
|
193
|
+
notes=f"depth={depth}, branching={branching}",
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
return measure(run)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def normal_sequential_execution(tasks: int, sleep_ms: float) -> BenchmarkResult:
|
|
200
|
+
def run() -> BenchmarkResult:
|
|
201
|
+
receipts = []
|
|
202
|
+
t0 = time.perf_counter()
|
|
203
|
+
for index in range(tasks):
|
|
204
|
+
receipts.append(simulated_yool_work(index, sleep_ms))
|
|
205
|
+
wall_ms = (time.perf_counter() - t0) * 1000
|
|
206
|
+
return BenchmarkResult(
|
|
207
|
+
profile="normal instruction",
|
|
208
|
+
phase="sequential execution",
|
|
209
|
+
tasks=tasks,
|
|
210
|
+
wall_ms=wall_ms,
|
|
211
|
+
peak_kb=0.0,
|
|
212
|
+
active_agents=tasks,
|
|
213
|
+
compressed_agents=0,
|
|
214
|
+
virtual_agents=0,
|
|
215
|
+
total_agents=tasks,
|
|
216
|
+
receipts=len(receipts),
|
|
217
|
+
notes="single worker, no lane fan-out",
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
return measure(run)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def yool_lane_execution(
|
|
224
|
+
tasks: int, sleep_ms: float, concurrency: int, max_concurrency: int
|
|
225
|
+
) -> BenchmarkResult:
|
|
226
|
+
def run() -> BenchmarkResult:
|
|
227
|
+
t0 = time.perf_counter()
|
|
228
|
+
policy = RuntimePolicy(
|
|
229
|
+
lane_concurrency=concurrency,
|
|
230
|
+
max_lane_concurrency=max_concurrency,
|
|
231
|
+
cpu_quota_pct=95,
|
|
232
|
+
queue_maxsize=8192,
|
|
233
|
+
compression_threshold=1024,
|
|
234
|
+
)
|
|
235
|
+
space = TupleSpace(policy=policy)
|
|
236
|
+
root = YoolTuple("kernel_root", (0,), "root", "runtime", "benchmark")
|
|
237
|
+
space.out_tuple(root)
|
|
238
|
+
for index in range(tasks):
|
|
239
|
+
space.spawn_agent(
|
|
240
|
+
root,
|
|
241
|
+
"prompt_worker",
|
|
242
|
+
{"index": index, "lane": "benchmark"},
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
def executor(tup: YoolTuple) -> str:
|
|
246
|
+
result = simulated_yool_work(int(tup.data["index"]), sleep_ms)
|
|
247
|
+
tup.receipts.append(f"ok:{result}")
|
|
248
|
+
return result
|
|
249
|
+
|
|
250
|
+
pool = LaneWorkerPool(space, policy=policy)
|
|
251
|
+
receipts = [
|
|
252
|
+
item for item in pool.run_lane("benchmark", executor) if item is not None
|
|
253
|
+
]
|
|
254
|
+
wall_ms = (time.perf_counter() - t0) * 1000
|
|
255
|
+
snapshot = space.snapshot()
|
|
256
|
+
return BenchmarkResult(
|
|
257
|
+
profile="yool prompt",
|
|
258
|
+
phase="lane fan-out execution",
|
|
259
|
+
tasks=tasks,
|
|
260
|
+
wall_ms=wall_ms,
|
|
261
|
+
peak_kb=0.0,
|
|
262
|
+
active_agents=snapshot["active_agents"],
|
|
263
|
+
compressed_agents=snapshot["compressed_agents"],
|
|
264
|
+
virtual_agents=snapshot["virtual_agents"],
|
|
265
|
+
total_agents=snapshot["total_agents"],
|
|
266
|
+
receipts=len(receipts),
|
|
267
|
+
notes=f"lane_concurrency={concurrency}, max_lane_concurrency={max_concurrency}",
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
return measure(run)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def summarise(results: list[BenchmarkResult]) -> dict[str, Any]:
|
|
274
|
+
by_phase: dict[tuple[str, str], BenchmarkResult] = {}
|
|
275
|
+
for result in results:
|
|
276
|
+
by_phase[(result.profile, result.phase)] = result
|
|
277
|
+
|
|
278
|
+
def compare(normal_phase: str, yool_phase: str) -> dict[str, Any] | None:
|
|
279
|
+
normal = by_phase.get(("normal instruction", normal_phase))
|
|
280
|
+
yool = by_phase.get(("yool prompt", yool_phase))
|
|
281
|
+
if normal is None or yool is None:
|
|
282
|
+
return None
|
|
283
|
+
return {
|
|
284
|
+
"speedup_x": normal.wall_ms / yool.wall_ms if yool.wall_ms else None,
|
|
285
|
+
"peak_memory_reduction_x": normal.peak_kb / yool.peak_kb
|
|
286
|
+
if yool.peak_kb
|
|
287
|
+
else None,
|
|
288
|
+
"normal_phase": normal_phase,
|
|
289
|
+
"yool_phase": yool_phase,
|
|
290
|
+
"normal_wall_ms": normal.wall_ms,
|
|
291
|
+
"yool_wall_ms": yool.wall_ms,
|
|
292
|
+
"normal_peak_kb": normal.peak_kb,
|
|
293
|
+
"yool_peak_kb": yool.peak_kb,
|
|
294
|
+
"normal_tasks": normal.tasks,
|
|
295
|
+
"yool_tasks": yool.tasks,
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
comparisons: dict[str, Any] = {}
|
|
299
|
+
scale = compare("flat materialization", "lazy batch_spawn")
|
|
300
|
+
if scale is not None:
|
|
301
|
+
comparisons["scale_representation"] = scale
|
|
302
|
+
execution = compare("sequential execution", "lane fan-out execution")
|
|
303
|
+
if execution is not None:
|
|
304
|
+
comparisons["active_execution"] = execution
|
|
305
|
+
|
|
306
|
+
return {
|
|
307
|
+
"results": [item.to_dict() for item in results],
|
|
308
|
+
"comparisons": comparisons,
|
|
309
|
+
"token_model": [item.to_dict() for item in token_model()],
|
|
310
|
+
"token_assumptions": {
|
|
311
|
+
"method": "rough local estimate using ceil(utf8_bytes / 4)",
|
|
312
|
+
"normal_instruction_tokens": estimate_tokens(NORMAL_INSTRUCTION),
|
|
313
|
+
"yool_prompt_tokens": estimate_tokens(YOOL_PROMPT),
|
|
314
|
+
"normal_repeated_context_tokens": estimate_tokens(
|
|
315
|
+
NORMAL_REPEATED_CONTEXT.format(index=0)
|
|
316
|
+
),
|
|
317
|
+
"yool_tuple_envelope_tokens": estimate_tokens(YOOL_TUPLE_ENVELOPE),
|
|
318
|
+
},
|
|
319
|
+
"wall_ms_median": statistics.median(item.wall_ms for item in results),
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def token_model() -> list[TokenModelResult]:
|
|
324
|
+
prompt_tokens = estimate_tokens(YOOL_PROMPT)
|
|
325
|
+
normal_instruction_tokens = estimate_tokens(NORMAL_INSTRUCTION)
|
|
326
|
+
normal_repeated_tokens = estimate_tokens(NORMAL_REPEATED_CONTEXT.format(index=0))
|
|
327
|
+
tuple_tokens = estimate_tokens(YOOL_TUPLE_ENVELOPE)
|
|
328
|
+
batch_tokens = estimate_tokens(YOOL_BATCH_ENVELOPE)
|
|
329
|
+
|
|
330
|
+
scale_agents = 1_048_576
|
|
331
|
+
normal_scale = normal_instruction_tokens + normal_repeated_tokens * scale_agents
|
|
332
|
+
yool_scale = prompt_tokens + batch_tokens
|
|
333
|
+
|
|
334
|
+
active_tasks = 256
|
|
335
|
+
normal_active = normal_repeated_tokens * active_tasks
|
|
336
|
+
yool_active = prompt_tokens + tuple_tokens * active_tasks
|
|
337
|
+
|
|
338
|
+
active_large = 512
|
|
339
|
+
normal_active_large = normal_repeated_tokens * active_large
|
|
340
|
+
yool_active_large = prompt_tokens + tuple_tokens * active_large
|
|
341
|
+
|
|
342
|
+
one_off_normal = normal_instruction_tokens
|
|
343
|
+
one_off_yool = prompt_tokens
|
|
344
|
+
|
|
345
|
+
return [
|
|
346
|
+
_token_result(
|
|
347
|
+
"one_off_prompt_bootstrap",
|
|
348
|
+
one_off_normal,
|
|
349
|
+
one_off_yool,
|
|
350
|
+
"One-off call: normal instruction is cheaper, but it lacks execution protocol.",
|
|
351
|
+
),
|
|
352
|
+
_token_result(
|
|
353
|
+
"scale_1m_subagents",
|
|
354
|
+
normal_scale,
|
|
355
|
+
yool_scale,
|
|
356
|
+
"Normal path enumerates/repeats subagent context; Yool path sends one batch_spawn envelope.",
|
|
357
|
+
),
|
|
358
|
+
_token_result(
|
|
359
|
+
"active_256_tasks",
|
|
360
|
+
normal_active,
|
|
361
|
+
yool_active,
|
|
362
|
+
"Yool prompt is paid once; active work uses compact tuple envelopes.",
|
|
363
|
+
),
|
|
364
|
+
_token_result(
|
|
365
|
+
"active_512_tasks",
|
|
366
|
+
normal_active_large,
|
|
367
|
+
yool_active_large,
|
|
368
|
+
"Savings grow as repeated chat-context orchestration is replaced by tuple envelopes.",
|
|
369
|
+
),
|
|
370
|
+
]
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def _token_result(
|
|
374
|
+
scenario: str, normal_tokens: int, yool_tokens: int, notes: str
|
|
375
|
+
) -> TokenModelResult:
|
|
376
|
+
savings = normal_tokens - yool_tokens
|
|
377
|
+
savings_pct = (savings / normal_tokens * 100) if normal_tokens else 0.0
|
|
378
|
+
return TokenModelResult(
|
|
379
|
+
scenario=scenario,
|
|
380
|
+
normal_tokens=normal_tokens,
|
|
381
|
+
yool_tokens=yool_tokens,
|
|
382
|
+
savings_tokens=savings,
|
|
383
|
+
savings_pct=savings_pct,
|
|
384
|
+
notes=notes,
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def main() -> int:
|
|
389
|
+
parser = argparse.ArgumentParser()
|
|
390
|
+
parser.add_argument("--tasks", type=int, default=256)
|
|
391
|
+
parser.add_argument("--scale-agents", type=int, default=131_072)
|
|
392
|
+
parser.add_argument("--sleep-ms", type=float, default=2.0)
|
|
393
|
+
parser.add_argument("--lane-concurrency", type=int, default=32)
|
|
394
|
+
parser.add_argument("--max-lane-concurrency", type=int, default=64)
|
|
395
|
+
parser.add_argument("--compression-threshold", type=int, default=1024)
|
|
396
|
+
parser.add_argument("--json", action="store_true")
|
|
397
|
+
args = parser.parse_args()
|
|
398
|
+
|
|
399
|
+
results = [
|
|
400
|
+
normal_flat_scale(args.scale_agents),
|
|
401
|
+
yool_lazy_scale(
|
|
402
|
+
args.scale_agents, branching=32, threshold=args.compression_threshold
|
|
403
|
+
),
|
|
404
|
+
normal_sequential_execution(args.tasks, args.sleep_ms),
|
|
405
|
+
yool_lane_execution(
|
|
406
|
+
args.tasks,
|
|
407
|
+
args.sleep_ms,
|
|
408
|
+
concurrency=args.lane_concurrency,
|
|
409
|
+
max_concurrency=args.max_lane_concurrency,
|
|
410
|
+
),
|
|
411
|
+
]
|
|
412
|
+
payload = summarise(results)
|
|
413
|
+
|
|
414
|
+
if args.json:
|
|
415
|
+
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
|
416
|
+
else:
|
|
417
|
+
print("Prompt benchmark")
|
|
418
|
+
for item in payload["results"]:
|
|
419
|
+
print(
|
|
420
|
+
f"- {item['profile']} / {item['phase']}: "
|
|
421
|
+
f"{item['wall_ms']:.2f} ms, "
|
|
422
|
+
f"{item['throughput_tasks_s']:.1f} tasks/s, "
|
|
423
|
+
f"peak={item['peak_kb']:.1f} KiB, "
|
|
424
|
+
f"total_agents={item['total_agents']}"
|
|
425
|
+
)
|
|
426
|
+
print(json.dumps(payload["comparisons"], indent=2, ensure_ascii=False))
|
|
427
|
+
return 0
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
if __name__ == "__main__":
|
|
431
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
%PDF-1.4
|
|
2
|
+
%���� ReportLab Generated PDF document (opensource)
|
|
3
|
+
1 0 obj
|
|
4
|
+
<<
|
|
5
|
+
/F1 2 0 R /F2 3 0 R /F3 4 0 R /F4 6 0 R
|
|
6
|
+
>>
|
|
7
|
+
endobj
|
|
8
|
+
2 0 obj
|
|
9
|
+
<<
|
|
10
|
+
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
|
11
|
+
>>
|
|
12
|
+
endobj
|
|
13
|
+
3 0 obj
|
|
14
|
+
<<
|
|
15
|
+
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
|
16
|
+
>>
|
|
17
|
+
endobj
|
|
18
|
+
4 0 obj
|
|
19
|
+
<<
|
|
20
|
+
/BaseFont /Times-Roman /Encoding /WinAnsiEncoding /Name /F3 /Subtype /Type1 /Type /Font
|
|
21
|
+
>>
|
|
22
|
+
endobj
|
|
23
|
+
5 0 obj
|
|
24
|
+
<<
|
|
25
|
+
/Contents 12 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 11 0 R /Resources <<
|
|
26
|
+
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
|
27
|
+
>> /Rotate 0 /Trans <<
|
|
28
|
+
|
|
29
|
+
>>
|
|
30
|
+
/Type /Page
|
|
31
|
+
>>
|
|
32
|
+
endobj
|
|
33
|
+
6 0 obj
|
|
34
|
+
<<
|
|
35
|
+
/BaseFont /Courier /Encoding /WinAnsiEncoding /Name /F4 /Subtype /Type1 /Type /Font
|
|
36
|
+
>>
|
|
37
|
+
endobj
|
|
38
|
+
7 0 obj
|
|
39
|
+
<<
|
|
40
|
+
/Contents 13 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 11 0 R /Resources <<
|
|
41
|
+
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
|
42
|
+
>> /Rotate 0 /Trans <<
|
|
43
|
+
|
|
44
|
+
>>
|
|
45
|
+
/Type /Page
|
|
46
|
+
>>
|
|
47
|
+
endobj
|
|
48
|
+
8 0 obj
|
|
49
|
+
<<
|
|
50
|
+
/Contents 14 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 11 0 R /Resources <<
|
|
51
|
+
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
|
52
|
+
>> /Rotate 0 /Trans <<
|
|
53
|
+
|
|
54
|
+
>>
|
|
55
|
+
/Type /Page
|
|
56
|
+
>>
|
|
57
|
+
endobj
|
|
58
|
+
9 0 obj
|
|
59
|
+
<<
|
|
60
|
+
/PageMode /UseNone /Pages 11 0 R /Type /Catalog
|
|
61
|
+
>>
|
|
62
|
+
endobj
|
|
63
|
+
10 0 obj
|
|
64
|
+
<<
|
|
65
|
+
/Author (wesleysimplicio/simplicio-prompt) /CreationDate (D:20260521154938-03'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260521154938-03'00') /Producer (ReportLab PDF Library - \(opensource\))
|
|
66
|
+
/Subject (\(unspecified\)) /Title (Yool Prompt Benchmark) /Trapped /False
|
|
67
|
+
>>
|
|
68
|
+
endobj
|
|
69
|
+
11 0 obj
|
|
70
|
+
<<
|
|
71
|
+
/Count 3 /Kids [ 5 0 R 7 0 R 8 0 R ] /Type /Pages
|
|
72
|
+
>>
|
|
73
|
+
endobj
|
|
74
|
+
12 0 obj
|
|
75
|
+
<<
|
|
76
|
+
/Filter [ /ASCII85Decode /FlateDecode ] /Length 2430
|
|
77
|
+
>>
|
|
78
|
+
stream
|
|
79
|
+
Gb"/(>E@OI&q8_F_+2!(1;&gn>6CZigU'Zg"3&+@_1<eOf?27!,KZFN^V3(WUh!\5bK2eYd:0F$o.i\Y%/J/g0gu4oB_+/c,P5f'SpX0CAS?(9W5&Hd?D-J>X*7We2cCPEBqurK[D7_,%<=-rRc3i2$8g,&'ZAUrO<JA/6H5NV&RhC[FlSB+Rf0-41A%k'esdT6=6ShA>fR`Y5-O6lS()/7ULuJ\<pR9.q[+q3U;baJH=$jBmoT9%g7r&h4mm0H.^q@Zh2WD;M;+^_"3k0*9i7a0;1<nMhV0Nd+00LsajMFdSqKnkRqCR&^>-_?YI6Gp`jL5P(h)]uaD49DXrl:CQ^CZ<h):$9OD_pL?]-7U``Y3HrkTr93k`#dol^d.r-3NH8s,<qZ)CsQqlTPOCpC:EM.c,jPZHupY%h182`?N%PsoDHl>;qj-q:[([q2#$Zkr!aY6u'-aL9iX*mdk&8Wc/)=o?<1oHp`)ec2[Er'!V07HpqB<Cr_63U#=1p)8Z!Mol)XjL94&,0oRo7<rOhQYp_#EIJ?aF&^l3VOP/cUK(1H]oTK.,qGJ?5t@Fu@1Q(oCK,3C3aGs3]8b!F)5gVk_n+.Q2u\3![>h5(BpU4u?p?qrD3ItiV[4csM],UuQRpOB(I'qGmaGb-T2g92(+#O`*sq[>(c-JAUuGJa_V,!56$$+T(XH1?62u:FquFKMYQ"D`kZdJoqT65%a-Q*X@(>rEKm2oP2CO%(4++aN_:n,88C(U\XBf^&kkEbchnRldcTAlfNMTPlm;MWiGBbFs9I<oSOul'NQKar-,5Td)@Q5Qo+TgVY703n5O`V\8aoQ/;Le8.273E]e'?1bS9^_>R`l'8/2mp$Vk3H&0<CS^&?ooA#,-2siiB0IhgfnY1+4U;#NY5<#VPQV`.Z5]n6UU$!K#Pg+7thmu\59II(8f_/LBULUE)-%:_D4^[UP4D;X9m8`/C3B>,4#qN@1[f08A!#'8X!3mXBrrt?1h<.Tb03-Q$b8:%"^_QKS*4+3J</c%Hh<6G3:e:bXJ;D(`5M?HW_'FqU$O^Z'&?!hN*HqDa.u)JEXD/jWY9Cg+hY>EFJW?9o_%^YXFj#ZEo[`/#?r=4.:4n2rQlRetO,1c-fWNY@GV!7S/,Hb`La>/pchldXi+/9F>_aF)0qkNuHZF^`%)B\d&#eH5&kQ#u0@Y)">!r&.t[8HQf536I>3ge4IJe7C7&$>f1jV6D/^6-/aQ7#"HFjBg^G6"Mlp&laVo^]EW9%bkA]rRn'm+9`b[NKC==i`P<thC06W:M%trC:@^?aq&DgB(4]nR-/h"<QSs"<DdD]Al$_DV+1>(QSZ^dAVal7"kU)4'PbhB._Y0W*aNC#dG/E/kSP]h4(p-:j'FaC4bZr&$$dR.\l*FeAM)a25mW&:&&RRb2;G[E!#r@oOSM2N!^iMr82CV<)*+m7BS33?S-iV;Yb*Md@e8F==PYGK(HpDX]m'P5UJffMJIFp]U:)N/*`*<Y2a_Xrt+&`Ch5JZaB/$.J5Nt'Q)!YCa]5BmoN/ci%td'<#A1pf%1\euXhI^7R"\p:t,_eK0RH3;Ob"46<B^ct$`W6uW8<]`Q(CVB+9$B;8&K==8/=#V*;@@/Ro]$LnA`bp#X*se_^n3"o"!M;A+=?kbOI/`<g1V?oR</@OK%,o2mSq53c_i8;]M"VY%B6XTNL!S-_`b4fjptWNfcG\mZ:'gY3T.c5N*:Api'HQ<$%/.LpTmu)@):1--'L0N!5ka<BRX.nsW%!Zl?iPCkTi0TNFeG_.2/q6uC`9\VZ5!H*;D-`FhJ1$,q/D(3$?%s-L\8iU#TEP_M8IMDR@pOT>pqPY8d1h*RRu3d<%Dq/I2JDMquITu>bbrERrD]I?ruE7m_9n_Y8+P^#e6-$([]e"OGcR2OtYGMER>nN%Mq97kn76#.',Ghh=SZP?*'[V$#dQcQqBT2!P^+XHJ2G3Z+XF[[Nk^3K@LU[m6q=sA`%>*l-hURiKT@uatYO3>nX"ak_[M3RgUk5(bK9PW%CM')5D`=.pZ.BA'Uu.g+:OOs'"lEbsCj+c!b?4J:F+J\PkXmOhlF>d6Yui83E6A9R&dHR497@h;/ZkWiKJk9Qu+>p-i$Hm:qk/HT-23Fu"qZPUl0=\[(br1Kh!gRmWM.qo;i[2e,rabEDh>&PMg"7kE=(F=^SA2DP_7p5Q9hKbN!)UD`cb*f9$]k?AS4CNLg?hCaeS)\VNWVarX@36W+I7n!o[=Hj6S&*0"h'JVJK\<fVq'&lhf6pQrM.3_HXOs7d(W@=Rj4uGcNg].0c&MWA&'G1Ho*NLn:_'PX.b$;o13arH:K$VD/%W?.Y:m*N)Qm9gcRmX.j2T7![m-!t#nES,0Yq$)q*b"!Mq_k&OLnfLN,%'ON7_XNN!s&0W\/3^6`k=&gN-euX)#&3k/af=5566;d^ru~>endstream
|
|
80
|
+
endobj
|
|
81
|
+
13 0 obj
|
|
82
|
+
<<
|
|
83
|
+
/Filter [ /ASCII85Decode /FlateDecode ] /Length 3042
|
|
84
|
+
>>
|
|
85
|
+
stream
|
|
86
|
+
Gau0DCN%re(B'h3EA9_0Ztmj25Otp.dS:&32Uf1g*D?fmOdK5a8Q8,-M>,Wj+_<h'fQ!e;U5q_'^#XS]T$'cb9`20fs)Cg+Ako[V(0NQ?YK:eoQf_q&3^<IaA?@.@gMgFSE[_9!_`$<NHC-Ub^iom=N$rcgbJhfK[_.-jS"g%2[S:XM)G]>r1rVu(l1u-Z/>WMYh[DiJj92pS4fr706Qi+?[4*ppD61E/'uR_em:8u4T<9HCmW_&W/1k2Vr%FMRg(Li!hZt4Vl#e8pq/0'aF:?dpls;M(*:&0YRB1\A7jSQQf6@0FQ">9Jgo'/IGL*S3?DSA`(>%dk-ZR2--"k=j=_6hVr;5g$#hnbeTcLWadLtgeTAL_.N^rh)]I)`WO>FJ`5Ta_uhLG1Hk2s2,%msfm]]E=630d\I0hb)G)aZ^DIIu!3kd9;He,;d$m=\db)E$?#XJ^R"Ig3j&Ln>FX&HQYOf('MaG7-B_^pacGFeU4n.MW;7!4d\eW*jg3(Uq&M09Y.1@RWV]Z;,IA")o(C/hLu%-qeogo>BeMk'#8B%^_[NGA":0dH"BAP/M<7&e62LBp$8mQnf8[N;2*R8WABp'_=/n4T7*c2\#"dV8d0/?G870^>pfX8g.q!.rgn&JDF4oAPh\43R0XK9uV0[e5GU//eXaQ/FjFr'$%E@3X"u9e^IV721MPr<:LTe<3/h`5tmh>M:$pUSamC4ArUUCM<fVs\g]F1]X[!i.(M2.L=<U@`s30g?lVnYN(aAB%*SA"L^^nj(^k>*".V.Xr%TOq-eU$Z7nWQ$2Ockda\p@S>W.N<2%s65]O4_hp+VooB[cMWq1";P#6JXo'=imkS3RbbT_+!N:THH5p'FH+ACi'1O2P<[:f%ZDM:7BF_Bus#:Bq"KW*K`)=buD07>mkYPn:@?=Eu#S]tP4%Le9=7>NUKMoV,87Z4;Os,\rj?@J_^$.sS)^Y0.I'11+/Z68ho<)k=VE:A1,o0X3r_&ik`;m58_#52bf$idhseE4OSNa+e<rFkAG#Ec&KofUfjVRCk]DDQd\U'@5O!XPVK6g7S-F1YW;LPA$S-n)kg0)ns#Ik-H]nCZdu%cZ259'/qZ%6b[c](5*4ph@#eddlHK9-gWmKkIgTX)iA+fHsf\A\T&MO-8pY(Ol/c@cU+;nd1!b#$/Zh#i2`>dh=1W^2K(q<`98N^[)m$#6L0]K"rsp,+8W.hI15_-'soF:Q_!PSjJ-Fu;:e!T+5QkOgQ342L2XWrhV4-mDInrK[3]q)KW%XS:".1LKj:%eWJR/Eq%[D](54j+?"J]\*M\[=psu""?0[J?GMea7c[[NS$c_.#VMcA7FnN:-IW>1(j$N:<ZS?-SBXDLLf')P(S^7W_f,nuj=ZPm/m,)s_A4E8lmL4:"_C6`#e1YRe*<?AA4NB,X`F#4q3f]&T[g-9Bd"6CPkTd@u$\ht."29i&jrt^UfNj./mc*Mdr-7a])Cj?*h$Z=VG@1QN)+:7ZnADSA4,0kuf>+M-cDPMsrC8lQ+sUpYR@fS)IW<O>d8T"jT:$@s5D[7$LI1^P;\Bq]kBO-c:5RuV+.<i7FY0TkN0Nsm>F&if^?PUGPk^]=_bl!@C52HO^$GP?3o?J9e>6ILjem]+EP&+I>"tul'`D:1RQ-UE/+e:0Ps:LH(/tA(ds=SCH:$FU[(#T4?1F:>P#GO_bc3-]PGi$e"-4DW%RRB8Z,le@Y&=nSM.X&fq^%'q?<"CUbmI%:WDE.!SY@m@A8-XM12]0n0WmkX_/=@U`;8VZT&!?U1%Z%U^^^%i9R@?GHS5Zn<?k1Upe&?cMj!\mYW31gjKRBNjU9TkD>o9;Fp=oUP%>%RI9NgL-s*Oa.&OR4+]'/8;&McHkrN<3AZ;c_n^1Kbh'?^jX1*)52/6p]]iR>YBt9r-RE+U/$IXGqU7NHK(1UAF($_b@@k1O@CER3$hIqUe/Bo)`/<qF.Ns@%sYl^4`?F$TeRhZ*m#1/eq9.!VZ$.MuQ0'2g@NJ=G6A94n4+0GLU@9$36"*m4OcbE0;>JhCDd7C7HPh9h>9-J3A!VU]5,aGk2rb_O-_K,=n+c<iDg">V3;2rEqS>nNerh2^;#YB?3YpR:nDahMb-M/glE\?*0A6c)f&i.fa?`2S`T^#YQG9V$5Z%V/I5NC#2p4J!em!5p`-Rt9>_V)V8Wg%%G-1H>@3!c#t>DC6[I]PG`UuM5FC[3\?po,Qpf@=A-``,5B'\:2[YTs\SJ3:%NQj!%>O:,1O_sGkE8i-]ZUpgSi^mZ*J:t`CZ`4],gYtZfjNQ):<55u`?8n:)Th;biJNrXHk@0R3SXLdB>p,:QsFk5%!1U=$,\*qN*GoOmZni=YJToSfX)!VRS;4UluH>&DO,-4s*eZ#5Jqk<JWG:td;5(3]Ah(37)X1t6;)Re\DUe#Y.8E?8S.ZUN)?n9\N<=BUX\Qt+UckahQU$+fkl4?B&$X2,(.8OQ^)_b<6A`1eXm^^R:8u?m%V/[tsrGU.#;55@^``!)G-jV-7O5dS4K`:c=l"i*SEdi2Cq"==#\WHsuJ5X4ifa>N"BXBAF&YiT*L@oG1.[e7eZ`'4h9Qe@?&<5fdI4T8F4j4ubbi<_EY/%TeXJaBb26=t_2OJ3!>:`o`6X]DZ-"`jmpBo:Fjb$`94jVQ+IlW($MTeJnAAiQ#WS"VO?dfFHoe\6c4WH(`#JNAn0_9c=\(#-(@Wf=gaHG?qQcBCcl_QWJXGE43F@Tl@gu`W+ZK1R&.<[$<9&JjWIgp2ebn)9ca3eaE?En7N^(7T0!:EMKKNal;`XAmL;AV3GXX+GlNq=kke<5s)'aFI8<W8<9UdE+ZL%Z.t$7(/Y7s*T*^1dC17F4cF(MT%`QFT'U,IjE-r(#5Y\KWH#PoPl7&i%DM.n)jJa""WbT@;B!P@;9ck>-YEF+qrm&Q]*)(T/A$5G'Fpe4aEkAI.B-"[]oqecK]jMCNF5fqm^5Q]CNSH22Kfoq-;rkmN.Nrthb)`p0b&q:@2S^XbuF?$,<LMp+W7Z6psD.r=T8Uq(/!SZVM,WioVp6q@7^TPR!r~>endstream
|
|
87
|
+
endobj
|
|
88
|
+
14 0 obj
|
|
89
|
+
<<
|
|
90
|
+
/Filter [ /ASCII85Decode /FlateDecode ] /Length 420
|
|
91
|
+
>>
|
|
92
|
+
stream
|
|
93
|
+
Gau0=]5GJ>']&?qB;A'Q[c><OMH%0kJ?i0Tj9>^?]UFLSr-ban>4nSh/<qgA]!(_0!2h-_Z`&D"L=nmY87Yp1`>#U(DR87]&=6"#==hlGL";\aXu,fKRMcis8:qf)@q3;PSf4c9^'e&%f79];@ZlQ,M:+M[>K^-Fk]T7O0_%4?SB31AL]fboq(u4$?t(BF85_m8X?u3U,uIYrrja%"3)'L\.44e)f:U4%DAomj/"USQE.dN8F!P?T%fRnJI2;H=n1+P$d:WhYhM_p^:Lg\RD:`T!Kt@CS8>cF+2<DOI`Abp=E,F$pLOE+]dFH4MZJDdXOeeutns;PDS$'%CCX3:(6f6Cgb0fBf]*@"3H_N"Sh&!MAU#!8*kTLTgPk*5M\MgN;EWlaA]$:d<^S$"Nnoh>c`,FO`i!lI)\i9~>endstream
|
|
94
|
+
endobj
|
|
95
|
+
xref
|
|
96
|
+
0 15
|
|
97
|
+
0000000000 65535 f
|
|
98
|
+
0000000061 00000 n
|
|
99
|
+
0000000122 00000 n
|
|
100
|
+
0000000229 00000 n
|
|
101
|
+
0000000341 00000 n
|
|
102
|
+
0000000450 00000 n
|
|
103
|
+
0000000655 00000 n
|
|
104
|
+
0000000760 00000 n
|
|
105
|
+
0000000965 00000 n
|
|
106
|
+
0000001170 00000 n
|
|
107
|
+
0000001239 00000 n
|
|
108
|
+
0000001547 00000 n
|
|
109
|
+
0000001619 00000 n
|
|
110
|
+
0000004141 00000 n
|
|
111
|
+
0000007275 00000 n
|
|
112
|
+
trailer
|
|
113
|
+
<<
|
|
114
|
+
/ID
|
|
115
|
+
[<8ca17f21cc76142cb58a8925493355bc><8ca17f21cc76142cb58a8925493355bc>]
|
|
116
|
+
% ReportLab generated PDF document -- digest (opensource)
|
|
117
|
+
|
|
118
|
+
/Info 10 0 R
|
|
119
|
+
/Root 9 0 R
|
|
120
|
+
/Size 15
|
|
121
|
+
>>
|
|
122
|
+
startxref
|
|
123
|
+
7786
|
|
124
|
+
%%EOF
|