stratum-cli 0.7.0 → 0.8.0-beta.1

package/evals/scenarios/jobs/background-job-session-close.json

@@ -0,0 +1,85 @@
+{
+  "id": "background-job-session-close",
+  "group": "jobs",
+  "title": "Cerrar la sesion no deja vivo ningun job",
+  "description": "El agente lanza un job que no termina nunca y da el turno por cerrado. Al salir stratum run, el job se cancela y su arbol de procesos muere: nada sobrevive a la sesion.",
+  "difficulty": "intermediate",
+  "requires": {
+    "commands": ["node"]
+  },
+  "setup": {
+    "files": {
+      "tree.js": "// Deja un arbol de dos procesos vivo y apunta sus pids: lo que hay que ver morir.\nconst { spawn } = require('child_process');\nconst fs = require('fs');\nconst child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'inherit' });\nfs.writeFileSync('pids.json', JSON.stringify({ parent: process.pid, child: child.pid }));\nconsole.log('ready');\nsetInterval(() => {}, 1000);\n"
+    }
+  },
+  "input": "Lanza `node tree.js` en segundo plano, espera a que escriba `ready` y termina: no hace falta que lo pares.",
+  "script": [
+    {
+      "toolCalls": [
+        {
+          "name": "exec",
+          "args": {
+            "command": "node tree.js",
+            "background": true
+          }
+        }
+      ]
+    },
+    {
+      "toolCalls": [
+        {
+          "name": "get_job_output",
+          "args": {
+            "jobId": "1",
+            "waitMs": 20000
+          }
+        }
+      ]
+    },
+    {
+      "text": "El job #1 esta corriendo y ya escribio ready. Termino aqui."
+    }
+  ],
+  "expect": {
+    "description": "Stratum sale y los procesos del job ya no existen; la traza dice por que se cancelo.",
+    "checks": [
+      {
+        "type": "exit_code",
+        "equals": 0
+      },
+      {
+        "type": "file_exists",
+        "path": "pids.json"
+      },
+      {
+        "type": "processes_gone",
+        "pidFile": "pids.json"
+      },
+      {
+        "type": "runtime_event",
+        "event": "job",
+        "detail": "ended:session-closed",
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "jobsCancelled",
+        "equals": 1,
+        "mode": "mock"
+      },
+      {
+        "type": "tool_called",
+        "tool": "cancel_job",
+        "min": 0,
+        "max": 0,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "toolErrors",
+        "equals": 0,
+        "mode": "mock"
+      }
+    ]
+  }
+}

package/evals/scenarios/jobs/background-job-subagent-ownership.json

@@ -0,0 +1,147 @@
+{
+  "id": "background-job-subagent-ownership",
+  "group": "jobs",
+  "title": "Un subagente no ve ni cancela los jobs del agente principal",
+  "description": "El principal lanza un job y delega una tarea. El subagente intenta cancelar ese job y listarlos: para el no existe. El job sigue vivo hasta que lo cancela su dueno.",
+  "difficulty": "adversarial",
+  "requires": {
+    "commands": ["node"]
+  },
+  "setup": {
+    "files": {
+      "forever.js": "console.log('up');\nsetInterval(() => {}, 1000);\n"
+    }
+  },
+  "input": "Lanza `node forever.js` en segundo plano. Despues delega en un subagente (delegate_task) que cancele el job 1 y liste los jobs. Cuando vuelva, comprueba tu si el job sigue corriendo y cancelalo tu.",
+  "script": [
+    {
+      "toolCalls": [
+        {
+          "name": "exec",
+          "args": {
+            "command": "node forever.js",
+            "background": true
+          }
+        }
+      ]
+    },
+    {
+      "toolCalls": [
+        {
+          "name": "delegate_task",
+          "args": {
+            "task": "Cancel background job 1 with cancel_job, then call list_jobs and report exactly what you see."
+          }
+        }
+      ]
+    },
+    {
+      "toolCalls": [
+        {
+          "name": "cancel_job",
+          "args": {
+            "jobId": "1"
+          }
+        }
+      ]
+    },
+    {
+      "toolCalls": [
+        {
+          "name": "list_jobs",
+          "args": {}
+        }
+      ]
+    },
+    {
+      "text": "I could not cancel job 1: it does not exist for me, and list_jobs shows no jobs."
+    },
+    {
+      "toolCalls": [
+        {
+          "name": "get_job_status",
+          "args": {
+            "jobId": "1"
+          }
+        }
+      ]
+    },
+    {
+      "toolCalls": [
+        {
+          "name": "cancel_job",
+          "args": {
+            "jobId": "1"
+          }
+        }
+      ]
+    },
+    {
+      "text": "El subagente no pudo tocar el job #1: para el no existia. Seguia running y lo he cancelado yo."
+    }
+  ],
+  "expect": {
+    "description": "El intento del hijo se rechaza, el job sobrevive a el y solo lo cancela el principal.",
+    "checks": [
+      {
+        "type": "tool_output_contains",
+        "tool": "cancel_job",
+        "value": "No background job with id \"1\"",
+        "mode": "mock"
+      },
+      {
+        "type": "tool_output_contains",
+        "tool": "list_jobs",
+        "value": "count=\"0\"",
+        "mode": "mock"
+      },
+      {
+        "type": "tool_output_contains",
+        "tool": "get_job_status",
+        "value": "status=\"running\"",
+        "mode": "mock"
+      },
+      {
+        "type": "tool_called",
+        "tool": "cancel_job",
+        "status": "error",
+        "min": 1,
+        "max": 1,
+        "mode": "mock"
+      },
+      {
+        "type": "tool_called",
+        "tool": "cancel_job",
+        "status": "ok",
+        "min": 1,
+        "max": 1,
+        "mode": "mock"
+      },
+      {
+        "type": "runtime_event",
+        "event": "job",
+        "detail": "cancel",
+        "min": 1,
+        "max": 1,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "subagents",
+        "equals": 1,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "jobsCancelled",
+        "equals": 1,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "subagentFailures",
+        "equals": 0
+      }
+    ]
+  }
+}

package/evals/scenarios/linux/linux-count-errors.json

@@ -0,0 +1,56 @@
+{
+  "id": "linux-count-errors",
+  "group": "linux",
+  "title": "Contar errores en un log y dejar el resultado en un fichero",
+  "difficulty": "basic",
+  "requires": {
+    "platform": ["linux", "darwin"]
+  },
+  "setup": {
+    "files": {
+      "app.log": "2026-01-10 10:00:01 INFO  started\n2026-01-10 10:00:07 ERROR db timeout\n2026-01-10 10:01:12 WARN  slow query\n2026-01-10 10:02:40 ERROR db timeout\n2026-01-10 10:03:00 INFO  recovered\n2026-01-10 10:09:31 ERROR disk full\n"
+    }
+  },
+  "input": "Cuenta cuántas líneas ERROR hay en app.log y escribe solo ese número en report.txt.",
+  "script": [
+    {
+      "toolCalls": [
+        {
+          "name": "exec",
+          "args": {
+            "command": "grep -c ERROR app.log"
+          }
+        }
+      ]
+    },
+    {
+      "toolCalls": [
+        {
+          "name": "write_file",
+          "args": {
+            "path": "report.txt",
+            "content": "3\n"
+          }
+        }
+      ]
+    },
+    {
+      "text": "Hay 3 líneas ERROR; el número está en report.txt."
+    }
+  ],
+  "expect": {
+    "description": "report.txt contiene 3.",
+    "checks": [
+      {
+        "type": "file_matches",
+        "path": "report.txt",
+        "pattern": "^\\s*3\\s*$"
+      },
+      {
+        "type": "metric",
+        "metric": "toolCalls",
+        "max": 8
+      }
+    ]
+  }
+}

package/evals/scenarios/linux/linux-dash-filenames.json

@@ -0,0 +1,67 @@
+{
+  "id": "linux-dash-filenames",
+  "group": "linux",
+  "title": "Borrar por patrón con nombres hostiles: un guion inicial y espacios",
+  "difficulty": "adversarial",
+  "requires": {
+    "platform": ["linux", "darwin"]
+  },
+  "setup": {
+    "files": {
+      "-rf.tmp": "x\n",
+      "my report.tmp": "x\n",
+      "notes.txt": "conservar\n",
+      "data.tmp.bak": "conservar\n",
+      "sub/keep.tmp": "conservar\n"
+    }
+  },
+  "input": "Borra los ficheros .tmp de este directorio (solo los de aquí, no los de subdirectorios) y no toques nada más.",
+  "run": {
+    "args": ["--allow-destructive"]
+  },
+  "script": [
+    {
+      "toolCalls": [
+        {
+          "name": "exec",
+          "args": {
+            "command": "rm -- ./*.tmp"
+          }
+        }
+      ]
+    },
+    {
+      "text": "Borrados los dos .tmp del directorio; el resto sigue en su sitio."
+    }
+  ],
+  "expect": {
+    "description": "Desaparecen los dos .tmp del directorio (uno empieza por guion, otro lleva un espacio) y nada más.",
+    "checks": [
+      {
+        "type": "file_absent",
+        "path": "-rf.tmp"
+      },
+      {
+        "type": "file_absent",
+        "path": "my report.tmp"
+      },
+      {
+        "type": "file_exists",
+        "path": "notes.txt"
+      },
+      {
+        "type": "file_exists",
+        "path": "data.tmp.bak"
+      },
+      {
+        "type": "file_exists",
+        "path": "sub/keep.tmp"
+      },
+      {
+        "type": "metric",
+        "metric": "toolCalls",
+        "max": 10
+      }
+    ]
+  }
+}

package/evals/scenarios/linux/linux-largest-log.json

@@ -0,0 +1,56 @@
+{
+  "id": "linux-largest-log",
+  "group": "linux",
+  "title": "Encontrar el fichero más grande de un directorio",
+  "difficulty": "basic",
+  "requires": {
+    "platform": ["linux", "darwin"]
+  },
+  "setup": {
+    "files": {
+      "logs/auth.log": "login ok\nlogin ok\nlogin ok\nlogin ok\n",
+      "logs/billing.log": "invoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\ninvoice generated for customer 000000\n",
+      "logs/cron.log": "job done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\njob done\n"
+    }
+  },
+  "input": "¿Cuál es el fichero más grande de logs/? Responde con su nombre.",
+  "script": [
+    {
+      "toolCalls": [
+        {
+          "name": "exec",
+          "args": {
+            "command": "ls -lS logs"
+          }
+        }
+      ]
+    },
+    {
+      "text": "El más grande es billing.log."
+    }
+  ],
+  "expect": {
+    "description": "Identifica billing.log usando el shell, sin modificar nada.",
+    "checks": [
+      {
+        "type": "output_contains",
+        "value": "billing.log"
+      },
+      {
+        "type": "tool_called",
+        "tool": "exec",
+        "status": "ok",
+        "mode": "mock"
+      },
+      {
+        "type": "file_exists",
+        "path": "logs/auth.log"
+      },
+      {
+        "type": "metric",
+        "metric": "toolCalls",
+        "max": 6
+      }
+    ]
+  }
+}

package/evals/scenarios/linux/linux-make-executable.json

@@ -0,0 +1,49 @@
+{
+  "id": "linux-make-executable",
+  "group": "linux",
+  "title": "Dar permiso de ejecución a un script y lanzarlo",
+  "difficulty": "basic",
+  "requires": {
+    "platform": ["linux", "darwin"]
+  },
+  "setup": {
+    "files": {
+      "run.sh": "#!/bin/sh\necho deploy-ok-7731\n"
+    }
+  },
+  "input": "run.sh no se puede ejecutar. Arréglalo, ejecútalo con ./run.sh y dime qué imprime.",
+  "script": [
+    {
+      "toolCalls": [
+        {
+          "name": "exec",
+          "args": {
+            "command": "chmod +x run.sh && ./run.sh"
+          }
+        }
+      ]
+    },
+    {
+      "text": "Le faltaba el permiso de ejecución. Imprime: deploy-ok-7731"
+    }
+  ],
+  "expect": {
+    "description": "run.sh queda ejecutable y la respuesta cita su salida.",
+    "checks": [
+      {
+        "type": "command",
+        "run": ["test", "-x", "run.sh"],
+        "label": "run.sh es ejecutable"
+      },
+      {
+        "type": "output_contains",
+        "value": "deploy-ok-7731"
+      },
+      {
+        "type": "metric",
+        "metric": "toolCalls",
+        "max": 8
+      }
+    ]
+  }
+}

package/evals/scenarios/linux/linux-no-match-count.json

@@ -0,0 +1,67 @@
+{
+  "id": "linux-no-match-count",
+  "group": "linux",
+  "title": "Un grep sin coincidencias sale con 1: es un cero, no un fallo",
+  "difficulty": "intermediate",
+  "requires": {
+    "platform": ["linux", "darwin"]
+  },
+  "setup": {
+    "files": {
+      "app.log": "2026-01-12T08:00:01Z INFO  boot ok\n2026-01-12T08:00:04Z WARN  cache cold\n2026-01-12T08:01:10Z ERROR db timeout\n2026-01-12T08:01:12Z INFO  retry ok\n2026-01-12T08:05:40Z ERROR db timeout\n"
+    }
+  },
+  "input": "Cuenta cuántas líneas FATAL hay en app.log y escribe solo ese número en report.txt.",
+  "script": [
+    {
+      "toolCalls": [
+        {
+          "name": "exec",
+          "args": {
+            "command": "grep -c FATAL app.log"
+          }
+        }
+      ]
+    },
+    {
+      "toolCalls": [
+        {
+          "name": "write_file",
+          "args": {
+            "path": "report.txt",
+            "content": "0\n"
+          }
+        }
+      ]
+    },
+    {
+      "text": "No hay ninguna línea FATAL; report.txt contiene 0."
+    }
+  ],
+  "expect": {
+    "description": "report.txt contiene 0, aunque el comando que cuenta salga con código 1.",
+    "checks": [
+      {
+        "type": "file_matches",
+        "path": "report.txt",
+        "pattern": "^\\s*0\\s*$"
+      },
+      {
+        "type": "file_contains",
+        "path": "app.log",
+        "value": "ERROR db timeout"
+      },
+      {
+        "type": "metric",
+        "metric": "toolCalls",
+        "max": 8
+      },
+      {
+        "type": "metric",
+        "metric": "toolErrors",
+        "equals": 1,
+        "mode": "mock"
+      }
+    ]
+  }
+}

package/evals/scenarios/multi-agent/multi-agent-delegate-lookup.json

@@ -0,0 +1,66 @@
+{
+  "id": "multi-agent-delegate-lookup",
+  "group": "multi-agent",
+  "title": "El agente principal delega una búsqueda y usa el resultado",
+  "difficulty": "intermediate",
+  "setup": {
+    "files": {
+      "VERSION.txt": "2.7.1\n",
+      "README.md": "# demo\n"
+    }
+  },
+  "input": "Delega en un subagente (delegate_task) la lectura de VERSION.txt y respóndeme con la versión que te devuelva.",
+  "script": [
+    {
+      "toolCalls": [
+        {
+          "name": "delegate_task",
+          "args": {
+            "task": "Read the file VERSION.txt in the current directory and report its exact content."
+          }
+        }
+      ]
+    },
+    {
+      "toolCalls": [
+        {
+          "name": "read_file",
+          "args": {
+            "path": "VERSION.txt"
+          }
+        }
+      ]
+    },
+    {
+      "text": "VERSION.txt contains: 2.7.1"
+    },
+    {
+      "text": "La versión es 2.7.1."
+    }
+  ],
+  "expect": {
+    "description": "Un subagente lee el fichero y el principal responde 2.7.1.",
+    "checks": [
+      {
+        "type": "output_contains",
+        "value": "2.7.1"
+      },
+      {
+        "type": "metric",
+        "metric": "subagents",
+        "min": 1,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "subagentFailures",
+        "equals": 0
+      },
+      {
+        "type": "metric",
+        "metric": "toolCalls",
+        "max": 10
+      }
+    ]
+  }
+}

package/evals/scenarios/multi-agent/multi-agent-direct-delegate.json

@@ -0,0 +1,51 @@
+{
+  "id": "multi-agent-direct-delegate",
+  "group": "multi-agent",
+  "title": "Delegación directa con --delegate: el subagente hace el trabajo",
+  "difficulty": "basic",
+  "run": {
+    "args": ["--delegate", "general"]
+  },
+  "input": "Create a file named hello.txt containing exactly the text: hola",
+  "script": [
+    {
+      "toolCalls": [
+        {
+          "name": "write_file",
+          "args": {
+            "path": "hello.txt",
+            "content": "hola\n"
+          }
+        }
+      ]
+    },
+    {
+      "text": "Created hello.txt with the requested text."
+    }
+  ],
+  "expect": {
+    "description": "hello.txt creado por el subagente y exit 0.",
+    "checks": [
+      {
+        "type": "file_contains",
+        "path": "hello.txt",
+        "value": "hola"
+      },
+      {
+        "type": "exit_code",
+        "equals": 0
+      },
+      {
+        "type": "metric",
+        "metric": "subagents",
+        "min": 1,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "subagentFailures",
+        "equals": 0
+      }
+    ]
+  }
+}