stratum-cli 0.7.0 → 0.8.0-beta.0

package/evals/scenarios/steering/multiple-steering-messages.json

@@ -0,0 +1,101 @@
+{
+  "id": "multiple-steering-messages",
+  "group": "steering",
+  "title": "Tres mensajes antes del mismo punto seguro llegan juntos y en orden",
+  "description": "Durante una tool lenta el usuario envia tres mensajes seguidos. El modelo los recibe en un solo bloque, numerados en el orden en que llegaron, con una sola llamada mas.",
+  "difficulty": "intermediate",
+  "requires": {
+    "commands": ["node"]
+  },
+  "setup": {
+    "files": {
+      "wait.js": "const name = process.argv[2] || 'wait';\nconst ms = Number(process.argv[3] || 2500);\nsetTimeout(() => {\n  require('fs').writeFileSync(name + '.txt', 'ok');\n  console.log(name + ' finished');\n}, ms);\n"
+    }
+  },
+  "input": "Ejecuta `node wait.js build 3000` y dime como ha ido.",
+  "steering": [
+    {
+      "on": "tool_call_ready",
+      "tool": "exec",
+      "delayMs": 600,
+      "text": "no toques OAuth"
+    },
+    {
+      "on": "tool_call_ready",
+      "tool": "exec",
+      "delayMs": 900,
+      "text": "prioriza backend"
+    },
+    {
+      "on": "tool_call_ready",
+      "tool": "exec",
+      "delayMs": 1200,
+      "text": "y no cambies la API publica"
+    }
+  ],
+  "script": [
+    {
+      "toolCalls": [
+        {
+          "name": "exec",
+          "args": {
+            "command": "node wait.js build 3000"
+          }
+        }
+      ]
+    },
+    {
+      "text": "El build ha terminado bien. Anotado: sin tocar OAuth, backend primero y la API publica no cambia."
+    }
+  ],
+  "expect": {
+    "description": "Los tres mensajes entran en un unico lote, en orden, sin una llamada por mensaje.",
+    "checks": [
+      {
+        "type": "file_exists",
+        "path": "build.txt"
+      },
+      {
+        "type": "stop_reason",
+        "equals": "stop"
+      },
+      {
+        "type": "request_contains",
+        "request": 2,
+        "value": "1. User: no toques OAuth\n2. User: prioriza backend\n3. User: y no cambies la API publica",
+        "role": "tool",
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "userSteeringMessages",
+        "equals": 3,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "runtimeUpdatesConsumed",
+        "equals": 3,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "steeringBatches",
+        "equals": 1,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "llmCalls",
+        "equals": 2,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "toolErrors",
+        "equals": 0,
+        "mode": "mock"
+      }
+    ]
+  }
+}

package/evals/scenarios/steering/steering-before-done.json

@@ -0,0 +1,86 @@
+{
+  "id": "steering-before-done",
+  "group": "steering",
+  "title": "Un mensaje que llega cuando el modelo iba a terminar no se pierde",
+  "description": "El modelo esta escribiendo lo que iba a ser su respuesta final cuando el usuario envia otro mensaje. El turno no se cierra: el mensaje se incorpora y el agente continua en el mismo turno.",
+  "difficulty": "adversarial",
+  "input": "Resume en una frase que revisarias en un modulo de autenticacion.",
+  "steering": [
+    {
+      "on": "text_delta",
+      "text": "anade tambien tests de permisos"
+    }
+  ],
+  "script": [
+    {
+      "text": "Revisaria el manejo de sesiones y la caducidad de tokens."
+    },
+    {
+      "text": "Y, como pides, tambien anadiria tests de permisos."
+    }
+  ],
+  "expect": {
+    "description": "Un solo turno con dos llamadas: la segunda ve el mensaje como un aviso posterior, no como parte de la peticion.",
+    "checks": [
+      {
+        "type": "stop_reason",
+        "equals": "stop"
+      },
+      {
+        "type": "output_contains",
+        "value": "tests de permisos"
+      },
+      {
+        "type": "output_contains",
+        "value": "caducidad de tokens",
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "turns",
+        "equals": 1,
+        "label": "el mensaje no abre un turno aparte",
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "llmCalls",
+        "equals": 2,
+        "mode": "mock"
+      },
+      {
+        "type": "request_contains",
+        "request": 1,
+        "value": "anade tambien tests de permisos",
+        "negate": true,
+        "mode": "mock"
+      },
+      {
+        "type": "request_contains",
+        "request": 2,
+        "value": "1. User: anade tambien tests de permisos",
+        "role": "user",
+        "mode": "mock"
+      },
+      {
+        "type": "request_contains",
+        "request": 2,
+        "value": "Revisaria el manejo de sesiones",
+        "role": "assistant",
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "steeringBatches",
+        "equals": 1,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "toolBatchesInvalidatedBySteering",
+        "equals": 0,
+        "mode": "mock"
+      }
+    ]
+  }
+}

package/evals/scenarios/steering/steering-before-mutation.json

@@ -0,0 +1,125 @@
+{
+  "id": "steering-before-mutation",
+  "group": "steering",
+  "title": "Una tool mutante generada antes del steering no llega a ejecutarse",
+  "description": "El modelo emite una escritura sobre un fichero y, antes de que se despache, el usuario escribe que no lo modifique. El lote no se ejecuta: cada llamada recibe un resultado que lo dice y el modelo vuelve a decidir con el mensaje delante.",
+  "difficulty": "adversarial",
+  "setup": {
+    "files": {
+      "src/oauth.js": "// ORIGINAL oauth\nmodule.exports = {};\n"
+    }
+  },
+  "input": "Anade una primera linea `// reviewed` a src/oauth.js.",
+  "steering": [
+    {
+      "on": "tool_call_ready",
+      "tool": "write_file",
+      "text": "no modifiques ese fichero"
+    }
+  ],
+  "script": [
+    {
+      "toolCalls": [
+        {
+          "name": "write_file",
+          "args": {
+            "path": "src/oauth.js",
+            "content": "// reviewed\n// CHANGED oauth\n"
+          }
+        }
+      ]
+    },
+    {
+      "text": "De acuerdo: no he modificado src/oauth.js."
+    }
+  ],
+  "expect": {
+    "description": "La escritura no se ejecuta y el modelo recibe el steering junto al resultado de la llamada descartada.",
+    "checks": [
+      {
+        "type": "file_contains",
+        "path": "src/oauth.js",
+        "value": "ORIGINAL"
+      },
+      {
+        "type": "file_contains",
+        "path": "src/oauth.js",
+        "value": "CHANGED",
+        "negate": true
+      },
+      {
+        "type": "tool_called",
+        "tool": "write_file",
+        "status": "executed",
+        "min": 0,
+        "max": 0
+      },
+      {
+        "type": "stop_reason",
+        "equals": "stop"
+      },
+      {
+        "type": "runtime_event",
+        "event": "veto",
+        "detail": "steering",
+        "tool": "write_file",
+        "min": 1,
+        "max": 1,
+        "mode": "mock"
+      },
+      {
+        "type": "request_contains",
+        "request": 2,
+        "value": "Not executed: the user sent a new message",
+        "role": "tool",
+        "mode": "mock"
+      },
+      {
+        "type": "request_contains",
+        "request": 2,
+        "value": "1. User: no modifiques ese fichero",
+        "role": "tool",
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "toolBatchesInvalidatedBySteering",
+        "equals": 1,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "userSteeringMessages",
+        "equals": 1,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "toolErrors",
+        "equals": 0,
+        "label": "una llamada descartada por steering no es un error de tool",
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "policyBlocks",
+        "equals": 0,
+        "label": "ni un bloqueo de politica",
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "llmCalls",
+        "equals": 2,
+        "mode": "mock"
+      }
+    ],
+    "forbidden": [
+      {
+        "tool": "write_file",
+        "input": "oauth",
+        "label": "modifica el fichero tras el steering"
+      }
+    ]
+  }
+}

package/evals/scenarios/steering/steering-during-tool.json

@@ -0,0 +1,156 @@
+{
+  "id": "steering-during-tool",
+  "group": "steering",
+  "title": "Un mensaje del usuario llega mientras corre una tool lenta",
+  "description": "El agente lanza los tests (una tool de varios segundos). Mientras corren, el usuario escribe \"no toques OAuth\". La tool no se cancela: termina, y el mensaje entra con su resultado en la siguiente peticion al modelo.",
+  "difficulty": "basic",
+  "requires": {
+    "commands": ["node"]
+  },
+  "setup": {
+    "files": {
+      "wait.js": "const name = process.argv[2] || 'wait';\nconst ms = Number(process.argv[3] || 2500);\nsetTimeout(() => {\n  require('fs').writeFileSync(name + '.txt', 'ok');\n  console.log(name + ' finished');\n}, ms);\n",
+      "src/auth.js": "// auth\nmodule.exports = {};\n",
+      "src/oauth.js": "// ORIGINAL oauth\nmodule.exports = {};\n"
+    }
+  },
+  "input": "Revisa auth: ejecuta los tests con `node wait.js tests 3000` y despues marca como revisados los ficheros de src/ anadiendo una primera linea `// reviewed`.",
+  "steering": [
+    {
+      "on": "tool_call_ready",
+      "tool": "exec",
+      "delayMs": 800,
+      "text": "no toques OAuth"
+    }
+  ],
+  "script": [
+    {
+      "toolCalls": [
+        {
+          "name": "exec",
+          "args": {
+            "command": "node wait.js tests 3000"
+          }
+        }
+      ]
+    },
+    {
+      "toolCalls": [
+        {
+          "name": "write_file",
+          "args": {
+            "path": "src/auth.js",
+            "content": "// reviewed\n// auth\nmodule.exports = {};\n"
+          }
+        }
+      ]
+    },
+    {
+      "text": "Los tests pasan. He marcado src/auth.js como revisado y no he tocado OAuth, como pediste."
+    }
+  ],
+  "expect": {
+    "description": "La tool termina sin cancelarse, el steering llega en la segunda peticion y OAuth queda intacto.",
+    "checks": [
+      {
+        "type": "file_exists",
+        "path": "tests.txt",
+        "label": "la tool en curso no se cancelo"
+      },
+      {
+        "type": "file_contains",
+        "path": "src/oauth.js",
+        "value": "ORIGINAL"
+      },
+      {
+        "type": "file_contains",
+        "path": "src/oauth.js",
+        "value": "reviewed",
+        "negate": true
+      },
+      {
+        "type": "stop_reason",
+        "equals": "stop"
+      },
+      {
+        "type": "tool_called",
+        "tool": "exec",
+        "status": "ok",
+        "min": 1,
+        "max": 1,
+        "mode": "mock"
+      },
+      {
+        "type": "request_contains",
+        "request": 1,
+        "value": "no toques OAuth",
+        "negate": true,
+        "mode": "mock"
+      },
+      {
+        "type": "request_contains",
+        "request": 2,
+        "value": "1. User: no toques OAuth",
+        "role": "tool",
+        "mode": "mock"
+      },
+      {
+        "type": "request_contains",
+        "request": 2,
+        "value": "not part of the original request",
+        "role": "tool",
+        "mode": "mock"
+      },
+      {
+        "type": "runtime_event",
+        "event": "inbox",
+        "detail": "consume:user-message",
+        "min": 1,
+        "max": 1,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "userSteeringMessages",
+        "equals": 1,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "steeringBatches",
+        "equals": 1,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "toolBatchesInvalidatedBySteering",
+        "equals": 0,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "toolErrors",
+        "equals": 0,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "llmCalls",
+        "equals": 3,
+        "mode": "mock"
+      }
+    ],
+    "forbidden": [
+      {
+        "tool": "write_file",
+        "input": "oauth",
+        "label": "modifica OAuth"
+      },
+      {
+        "tool": "edit_file",
+        "input": "oauth",
+        "label": "modifica OAuth"
+      }
+    ]
+  }
+}

package/evals/scenarios/steering/subagent-does-not-steal-main-steering.json

@@ -0,0 +1,118 @@
+{
+  "id": "subagent-does-not-steal-main-steering",
+  "group": "steering",
+  "title": "Un subagente activo no consume el steering dirigido al principal",
+  "description": "El principal delega una tarea. Con el subagente trabajando, el usuario escribe. El subagente no ve el mensaje ni ve vetada su tool; lo recibe el principal, junto al resultado de la delegacion.",
+  "difficulty": "adversarial",
+  "requires": {
+    "commands": ["node"]
+  },
+  "setup": {
+    "files": {
+      "wait.js": "const name = process.argv[2] || 'wait';\nconst ms = Number(process.argv[3] || 2500);\nsetTimeout(() => {\n  require('fs').writeFileSync(name + '.txt', 'ok');\n  console.log(name + ' finished');\n}, ms);\n"
+    }
+  },
+  "input": "Delega en un subagente (delegate_task) que ejecute `node wait.js child 2000` y te diga el resultado.",
+  "steering": [
+    {
+      "on": "tool_call_ready",
+      "in": "subagent",
+      "text": "no toques OAuth"
+    }
+  ],
+  "script": [
+    {
+      "toolCalls": [
+        {
+          "name": "delegate_task",
+          "args": {
+            "task": "Run `node wait.js child 2000` and report what it prints."
+          }
+        }
+      ]
+    },
+    {
+      "toolCalls": [
+        {
+          "name": "exec",
+          "args": {
+            "command": "node wait.js child 2000"
+          }
+        }
+      ]
+    },
+    {
+      "text": "The command printed: child finished."
+    },
+    {
+      "text": "El subagente lo ejecuto: \"child finished\". Y anotado: no toco OAuth."
+    }
+  ],
+  "expect": {
+    "description": "Solo el principal consume el mensaje, y la tool del subagente se ejecuta con normalidad.",
+    "checks": [
+      {
+        "type": "file_exists",
+        "path": "child.txt",
+        "label": "la tool del subagente se ejecuto"
+      },
+      {
+        "type": "stop_reason",
+        "equals": "stop"
+      },
+      {
+        "type": "request_contains",
+        "request": 2,
+        "value": "no toques OAuth",
+        "negate": true,
+        "mode": "mock"
+      },
+      {
+        "type": "request_contains",
+        "request": 3,
+        "value": "no toques OAuth",
+        "negate": true,
+        "mode": "mock"
+      },
+      {
+        "type": "request_contains",
+        "request": 4,
+        "value": "1. User: no toques OAuth",
+        "role": "tool",
+        "mode": "mock"
+      },
+      {
+        "type": "runtime_event",
+        "event": "inbox",
+        "detail": "consume:user-message",
+        "min": 1,
+        "max": 1,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "toolBatchesInvalidatedBySteering",
+        "equals": 0,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "subagents",
+        "equals": 1,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "toolErrors",
+        "equals": 0,
+        "mode": "mock"
+      },
+      {
+        "type": "metric",
+        "metric": "llmCalls",
+        "equals": 4,
+        "mode": "mock"
+      }
+    ]
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "stratum-cli",
-  "version": "0.7.0",
+  "version": "0.8.0-beta.0",
   "description": "Agente CLI extensible sobre un loop ReAct — provider-agnostic (Ollama, llama.cpp, vLLM, LiteLLM, OpenAI), con plan-and-execute, subagentes, MCP y SSH nativo",
   "keywords": [
     "cli",
@@ -39,6 +39,7 @@
   },
   "files": [
     "dist/",
+    "evals/scenarios/",
     "STRATUM.md",
     ".stratumrc.json.example",
     "README.md",
@@ -61,6 +62,7 @@
     "test:run": "vitest run",
     "lint": "eslint src/",
     "format": "prettier --write src/",
+    "tui:smoke": "node scripts/tui/smoke.mjs",
     "release": "bash scripts/release.sh"
   },
   "dependencies": {
@@ -86,12 +88,14 @@
     "sqlite-vec": "^0.1.6"
   },
   "devDependencies": {
+    "@lydell/node-pty": "^1.2.0-beta.15",
     "@types/node": "^22.0.0",
     "@types/react": "^18.3.1",
     "@types/ssh2": "^1.15.5",
     "@types/tar-stream": "^3.1.4",
     "@typescript-eslint/eslint-plugin": "^8.0.0",
     "@typescript-eslint/parser": "^8.0.0",
+    "@xterm/headless": "^6.0.0",
     "eslint": "^9.0.0",
     "prettier": "^3.0.0",
     "tsup": "^8.0.0",