stratum-cli 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.stratumrc.json.example +11 -1
- package/CHANGELOG.md +30 -0
- package/dist/index.js +4798 -1744
- package/evals/scenarios/cache/cache-compression-prefix-not-worth.json +92 -0
- package/evals/scenarios/cache/cache-compression-reuses-prefix.json +101 -0
- package/evals/scenarios/replay/replay-rename-across-files.json +85 -0
- package/package.json +1 -1
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "cache-compression-prefix-not-worth",
|
|
3
|
+
"group": "cache",
|
|
4
|
+
"title": "Con agent.compressionReusePrefix, si reutilizar el prefijo no compensa se usa el resumen de siempre",
|
|
5
|
+
"description": "El turno de cache-compression-cache-impact con la opción activa. La compresión salta justo después de una lectura grande, que es lo único que no está en caché: mandar el historial tal cual dejaría fuera de caché más que el resumen recortado entero. Se usa el resumen de siempre y todo queda como en aquel escenario.",
|
|
6
|
+
"difficulty": "intermediate",
|
|
7
|
+
"requires": {
|
|
8
|
+
"commands": ["node"]
|
|
9
|
+
},
|
|
10
|
+
"setup": {
|
|
11
|
+
"files": {
|
|
12
|
+
"logs/README.md": "Logs de los tres servicios."
|
|
13
|
+
},
|
|
14
|
+
"commands": [
|
|
15
|
+
[
|
|
16
|
+
"node",
|
|
17
|
+
"-e",
|
|
18
|
+
"const fs = require('fs');['one', 'two', 'three'].forEach((name, k) => { const lines = []; for (let i = 0; i < 300; i++) lines.push('service-' + name + ' request ' + String(i).padStart(4, '0') + ' handled in ' + (20 + ((i * 7 + k) % 60)) + ' ms status 200 ok'); lines[150] = 'service-' + name + ' request FAILED with code E' + (41 + k) + ' upstream timeout'; const nl = String.fromCharCode(10); fs.writeFileSync('logs/' + name + '.log', lines.join(nl) + nl);});"
|
|
19
|
+
]
|
|
20
|
+
],
|
|
21
|
+
"config": {
|
|
22
|
+
"agent": {
|
|
23
|
+
"compressionThreshold": 0.62,
|
|
24
|
+
"compressionKeepRounds": 1,
|
|
25
|
+
"compressionReusePrefix": true
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
},
|
|
29
|
+
"input": "Lee logs/one.log, logs/two.log y logs/three.log, de uno en uno, y dime el código de error que aparece en cada uno.",
|
|
30
|
+
"script": [
|
|
31
|
+
{ "toolCalls": [{ "name": "read_file", "args": { "path": "logs/one.log" } }] },
|
|
32
|
+
{ "toolCalls": [{ "name": "read_file", "args": { "path": "logs/two.log" } }] },
|
|
33
|
+
{ "toolCalls": [{ "name": "read_file", "args": { "path": "logs/three.log" } }] },
|
|
34
|
+
{ "text": "Los códigos son E41, E42 y E43." }
|
|
35
|
+
],
|
|
36
|
+
"auxiliaryScript": {
|
|
37
|
+
"context-compression": [
|
|
38
|
+
{
|
|
39
|
+
"text": "Read so far: logs/one.log failed with E41, logs/two.log failed with E42."
|
|
40
|
+
}
|
|
41
|
+
]
|
|
42
|
+
},
|
|
43
|
+
"expect": {
|
|
44
|
+
"description": "Una compresión con el resumen recortado (sin nada de caché, como sin la opción) y el agente termina sin errores.",
|
|
45
|
+
"checks": [
|
|
46
|
+
{
|
|
47
|
+
"type": "output_contains",
|
|
48
|
+
"value": "E43"
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
"type": "metric",
|
|
52
|
+
"metric": "compressions",
|
|
53
|
+
"mode": "mock",
|
|
54
|
+
"equals": 1
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
"type": "llm_call",
|
|
58
|
+
"origin": "context-compression",
|
|
59
|
+
"status": "ok",
|
|
60
|
+
"min": 1,
|
|
61
|
+
"max": 1,
|
|
62
|
+
"mode": "mock"
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"type": "metric",
|
|
66
|
+
"metric": "auxiliaryCachedReadTokens",
|
|
67
|
+
"equals": 0,
|
|
68
|
+
"mode": "mock",
|
|
69
|
+
"label": "el resumen recortado no comparte prefijo con el loop"
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
"type": "metric",
|
|
73
|
+
"metric": "auxiliaryPromptTokens",
|
|
74
|
+
"max": 2000,
|
|
75
|
+
"mode": "mock",
|
|
76
|
+
"label": "y es pequeño: mucho menos que el historial entero"
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
"type": "metric",
|
|
80
|
+
"metric": "toolErrors",
|
|
81
|
+
"mode": "mock",
|
|
82
|
+
"equals": 0
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
"type": "metric",
|
|
86
|
+
"metric": "llmErrors",
|
|
87
|
+
"mode": "mock",
|
|
88
|
+
"equals": 0
|
|
89
|
+
}
|
|
90
|
+
]
|
|
91
|
+
}
|
|
92
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "cache-compression-reuses-prefix",
|
|
3
|
+
"group": "cache",
|
|
4
|
+
"title": "Con agent.compressionReusePrefix, el resumen reutiliza el prefijo que el backend acaba de cachear",
|
|
5
|
+
"description": "Ocho lecturas medianas y una novena que cruza el umbral. Lo que entró desde la última petición del loop (la novena) es menos que el resumen de siempre de las ocho anteriores, así que la compresión manda el historial tal cual (mismas tools, en tool_choice none) con la instrucción al final, y casi toda su entrada sale de caché. Su pareja, cache-compression-prefix-not-worth, es el caso contrario.",
|
|
6
|
+
"difficulty": "intermediate",
|
|
7
|
+
"requires": {
|
|
8
|
+
"commands": ["node"]
|
|
9
|
+
},
|
|
10
|
+
"setup": {
|
|
11
|
+
"commands": [
|
|
12
|
+
[
|
|
13
|
+
"node",
|
|
14
|
+
"-e",
|
|
15
|
+
"const fs = require('fs'); const nl = String.fromCharCode(10); fs.mkdirSync('docs'); const write = (name, n) => { const lines = []; for (let i = 0; i < n; i++) lines.push(name + ' line ' + String(i).padStart(3, '0') + ' lorem ipsum dolor sit amet consectetur'); fs.writeFileSync('docs/' + name + '.md', lines.join(nl) + nl); }; for (let k = 1; k <= 8; k++) write('part' + k, 100); write('notes', 100);"
|
|
16
|
+
]
|
|
17
|
+
],
|
|
18
|
+
"config": {
|
|
19
|
+
"agent": {
|
|
20
|
+
"compressionThreshold": 0.67,
|
|
21
|
+
"compressionKeepRounds": 1,
|
|
22
|
+
"compressionReusePrefix": true
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
},
|
|
26
|
+
"input": "Lee docs/part1.md a docs/part8.md, de uno en uno, y después docs/notes.md. Dime cuántos ficheros has leído.",
|
|
27
|
+
"script": [
|
|
28
|
+
{ "toolCalls": [{ "name": "read_file", "args": { "path": "docs/part1.md" } }] },
|
|
29
|
+
{ "toolCalls": [{ "name": "read_file", "args": { "path": "docs/part2.md" } }] },
|
|
30
|
+
{ "toolCalls": [{ "name": "read_file", "args": { "path": "docs/part3.md" } }] },
|
|
31
|
+
{ "toolCalls": [{ "name": "read_file", "args": { "path": "docs/part4.md" } }] },
|
|
32
|
+
{ "toolCalls": [{ "name": "read_file", "args": { "path": "docs/part5.md" } }] },
|
|
33
|
+
{ "toolCalls": [{ "name": "read_file", "args": { "path": "docs/part6.md" } }] },
|
|
34
|
+
{ "toolCalls": [{ "name": "read_file", "args": { "path": "docs/part7.md" } }] },
|
|
35
|
+
{ "toolCalls": [{ "name": "read_file", "args": { "path": "docs/part8.md" } }] },
|
|
36
|
+
{ "toolCalls": [{ "name": "read_file", "args": { "path": "docs/notes.md" } }] },
|
|
37
|
+
{ "text": "He leído 9 ficheros." }
|
|
38
|
+
],
|
|
39
|
+
"auxiliaryScript": {
|
|
40
|
+
"context-compression": [
|
|
41
|
+
{
|
|
42
|
+
"text": "Read so far: docs/part1.md to docs/part8.md, 100 lines each."
|
|
43
|
+
}
|
|
44
|
+
]
|
|
45
|
+
},
|
|
46
|
+
"expect": {
|
|
47
|
+
"description": "Una sola compresión, reconocida como llamada auxiliar, que lee de caché casi toda su entrada; el agente termina sin errores.",
|
|
48
|
+
"checks": [
|
|
49
|
+
{
|
|
50
|
+
"type": "output_contains",
|
|
51
|
+
"value": "9 ficheros"
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
"type": "metric",
|
|
55
|
+
"metric": "compressions",
|
|
56
|
+
"mode": "mock",
|
|
57
|
+
"equals": 1
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"type": "llm_call",
|
|
61
|
+
"origin": "context-compression",
|
|
62
|
+
"status": "ok",
|
|
63
|
+
"min": 1,
|
|
64
|
+
"max": 1,
|
|
65
|
+
"mode": "mock"
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
"type": "metric",
|
|
69
|
+
"metric": "auxiliaryCachedReadTokens",
|
|
70
|
+
"min": 1,
|
|
71
|
+
"mode": "mock",
|
|
72
|
+
"label": "el resumen lee de caché el prompt que el loop acaba de mandar"
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
"type": "metric",
|
|
76
|
+
"metric": "auxiliaryCacheHitRate",
|
|
77
|
+
"min": 0.85,
|
|
78
|
+
"mode": "mock"
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
"type": "cache_break",
|
|
82
|
+
"cause": "compression",
|
|
83
|
+
"min": 1,
|
|
84
|
+
"max": 1,
|
|
85
|
+
"mode": "mock"
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
"type": "metric",
|
|
89
|
+
"metric": "toolErrors",
|
|
90
|
+
"mode": "mock",
|
|
91
|
+
"equals": 0
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
"type": "metric",
|
|
95
|
+
"metric": "llmErrors",
|
|
96
|
+
"mode": "mock",
|
|
97
|
+
"equals": 0
|
|
98
|
+
}
|
|
99
|
+
]
|
|
100
|
+
}
|
|
101
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "replay-rename-across-files",
|
|
3
|
+
"group": "replay",
|
|
4
|
+
"title": "Replay: renombrar una función en todo el proyecto",
|
|
5
|
+
"description": "Sesión sess_20261028_144055_qtx reproducida desde su traza: 4 respuestas del modelo como guion. Corre sin modelo (--mock) y puntúa el workspace que dejó la sesión.",
|
|
6
|
+
"difficulty": "basic",
|
|
7
|
+
"setup": {
|
|
8
|
+
"files": {
|
|
9
|
+
"lib/cart.js": "function calcTotal(items) {\n return items.reduce((sum, item) => sum + item.price * item.qty, 0);\n}\n\nmodule.exports = { calcTotal };\n",
|
|
10
|
+
"lib/report.js": "const { calcTotal } = require('./cart');\n\nfunction report(items) {\n return `Total: ${calcTotal(items)}`;\n}\n\nmodule.exports = { report };\n",
|
|
11
|
+
"main.js": "const { calcTotal } = require('./lib/cart');\nconst { report } = require('./lib/report');\n\nconst items = [{ price: 10, qty: 3 }, { price: 6, qty: 2 }];\nconsole.log(calcTotal(items));\nconsole.log(report(items));\n"
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
"input": "Renombra la función `calcTotal` a `computeTotal` en todo el proyecto y comprueba que `node main.js` sigue funcionando.",
|
|
15
|
+
"script": [
|
|
16
|
+
{
|
|
17
|
+
"toolCalls": [
|
|
18
|
+
{
|
|
19
|
+
"name": "grep",
|
|
20
|
+
"args": {
|
|
21
|
+
"pattern": "calcTotal"
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
]
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"toolCalls": [
|
|
28
|
+
{
|
|
29
|
+
"name": "edit_file",
|
|
30
|
+
"args": {
|
|
31
|
+
"path": "lib/cart.js",
|
|
32
|
+
"old_string": "calcTotal",
|
|
33
|
+
"new_string": "computeTotal",
|
|
34
|
+
"replace_all": true
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
"name": "edit_file",
|
|
39
|
+
"args": {
|
|
40
|
+
"path": "lib/report.js",
|
|
41
|
+
"old_string": "calcTotal",
|
|
42
|
+
"new_string": "computeTotal",
|
|
43
|
+
"replace_all": true
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
"name": "edit_file",
|
|
48
|
+
"args": {
|
|
49
|
+
"path": "main.js",
|
|
50
|
+
"old_string": "calcTotal",
|
|
51
|
+
"new_string": "computeTotal",
|
|
52
|
+
"replace_all": true
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
]
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
"toolCalls": [
|
|
59
|
+
{
|
|
60
|
+
"name": "exec",
|
|
61
|
+
"args": {
|
|
62
|
+
"command": "node main.js"
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
]
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
"text": "Renombrada en los tres ficheros; `node main.js` imprime 42."
|
|
69
|
+
}
|
|
70
|
+
],
|
|
71
|
+
"expect": {
|
|
72
|
+
"description": "El workspace queda como al terminar la sesión grabada.",
|
|
73
|
+
"checks": [
|
|
74
|
+
{
|
|
75
|
+
"type": "workspace_expected",
|
|
76
|
+
"files": {
|
|
77
|
+
"lib/cart.js": "function computeTotal(items) {\n return items.reduce((sum, item) => sum + item.price * item.qty, 0);\n}\n\nmodule.exports = { computeTotal };\n",
|
|
78
|
+
"lib/report.js": "const { computeTotal } = require('./cart');\n\nfunction report(items) {\n return `Total: ${computeTotal(items)}`;\n}\n\nmodule.exports = { report };\n",
|
|
79
|
+
"main.js": "const { computeTotal } = require('./lib/cart');\nconst { report } = require('./lib/report');\n\nconst items = [{ price: 10, qty: 3 }, { price: 6, qty: 2 }];\nconsole.log(computeTotal(items));\nconsole.log(report(items));\n"
|
|
80
|
+
},
|
|
81
|
+
"strict": true
|
|
82
|
+
}
|
|
83
|
+
]
|
|
84
|
+
}
|
|
85
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "stratum-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.0",
|
|
4
4
|
"description": "Agente CLI extensible sobre un loop ReAct — provider-agnostic (Ollama, llama.cpp, vLLM, LiteLLM, OpenAI), con plan-and-execute, subagentes, MCP y SSH nativo",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"cli",
|