stageflow 0.40.0 → 0.44.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (519) hide show
  1. package/README.md +15 -4
  2. package/dist/agent/activity.d.ts +18 -0
  3. package/dist/agent/claudeAdapter.js +7 -0
  4. package/dist/agent/claudeTools.d.ts +27 -18
  5. package/dist/agent/claudeTools.js +54 -15
  6. package/dist/agent/compactionEvents.d.ts +34 -0
  7. package/dist/agent/compactionEvents.js +37 -0
  8. package/dist/agent/openRouterRouting.d.ts +7 -0
  9. package/dist/agent/openRouterRouting.js +23 -0
  10. package/dist/agent/piAdapter.d.ts +6 -0
  11. package/dist/agent/piAdapter.js +128 -27
  12. package/dist/agent/piSettings.d.ts +7 -0
  13. package/dist/agent/piSettings.js +9 -0
  14. package/dist/agent/sandboxPiTools.d.ts +3 -1
  15. package/dist/agent/sandboxPiTools.js +8 -2
  16. package/dist/agent/stageTurnDriver.d.ts +11 -2
  17. package/dist/agent/stageTurnDriver.js +45 -4
  18. package/dist/agent/toolOutputBudget.d.ts +15 -1
  19. package/dist/agent/toolOutputBudget.js +18 -0
  20. package/dist/agent/toolOutputBudgetExtension.d.ts +2 -0
  21. package/dist/agent/toolOutputBudgetExtension.js +40 -3
  22. package/dist/agents/authorChat.d.ts +5 -0
  23. package/dist/agents/authorChat.js +120 -12
  24. package/dist/agents/authorSessionStore.d.ts +38 -0
  25. package/dist/agents/authorSessionStore.js +167 -0
  26. package/dist/agents/chatTurn.d.ts +2 -0
  27. package/dist/agents/chatTurn.js +17 -1
  28. package/dist/agents/errors.d.ts +1 -1
  29. package/dist/agents/errors.js +1 -0
  30. package/dist/agents/evalGate.d.ts +61 -0
  31. package/dist/agents/evalGate.js +103 -0
  32. package/dist/agents/evalSession.d.ts +26 -0
  33. package/dist/agents/evalSession.js +29 -0
  34. package/dist/agents/hostContext.d.ts +4 -1
  35. package/dist/agents/hostContext.js +4 -1
  36. package/dist/agents/runtime/piRuntime.d.ts +6 -2
  37. package/dist/agents/runtime/piRuntime.js +50 -16
  38. package/dist/agents/runtime/types.d.ts +3 -1
  39. package/dist/agents/service.d.ts +27 -2
  40. package/dist/agents/service.js +91 -15
  41. package/dist/agents/sessions.d.ts +8 -0
  42. package/dist/agents/sessions.js +53 -11
  43. package/dist/agents/store.d.ts +14 -3
  44. package/dist/agents/store.js +45 -7
  45. package/dist/agents/toolRisk.d.ts +21 -0
  46. package/dist/agents/toolRisk.js +50 -0
  47. package/dist/cli/doctorCommand.js +26 -0
  48. package/dist/cli/evalCommand.d.ts +20 -0
  49. package/dist/cli/evalCommand.js +720 -0
  50. package/dist/cli/exportAllCommand.js +1 -1
  51. package/dist/cli/hostClient.d.ts +60 -0
  52. package/dist/cli/hostClient.js +71 -0
  53. package/dist/cli/validateOutput.d.ts +2 -2
  54. package/dist/cli/validateOutput.js +2 -2
  55. package/dist/cli.js +10 -0
  56. package/dist/compaction/directives.d.ts +18 -0
  57. package/dist/compaction/directives.js +119 -0
  58. package/dist/compaction/extension.d.ts +20 -0
  59. package/dist/compaction/extension.js +78 -0
  60. package/dist/compaction/mask.d.ts +18 -0
  61. package/dist/compaction/mask.js +155 -0
  62. package/dist/compaction/pin.d.ts +21 -0
  63. package/dist/compaction/pin.js +270 -0
  64. package/dist/compaction/scrub.d.ts +7 -0
  65. package/dist/compaction/scrub.js +26 -0
  66. package/dist/compaction/sessionWiring.d.ts +22 -0
  67. package/dist/compaction/sessionWiring.js +26 -0
  68. package/dist/compaction/summarize.d.ts +79 -0
  69. package/dist/compaction/summarize.js +171 -0
  70. package/dist/compaction/toolPolicy.d.ts +3 -0
  71. package/dist/compaction/toolPolicy.js +33 -0
  72. package/dist/compaction/types.d.ts +45 -0
  73. package/dist/compaction/types.js +7 -0
  74. package/dist/compaction/workingState.d.ts +56 -0
  75. package/dist/compaction/workingState.js +198 -0
  76. package/dist/config/atomicWrite.d.ts +15 -0
  77. package/dist/config/atomicWrite.js +43 -0
  78. package/dist/config/browseCatalog.d.ts +2 -0
  79. package/dist/config/browseCatalog.js +4 -2
  80. package/dist/config/createPipeline.js +2 -0
  81. package/dist/config/createTrigger.js +7 -1
  82. package/dist/config/csv.d.ts +9 -0
  83. package/dist/config/csv.js +81 -0
  84. package/dist/config/datasetAgentCheck.d.ts +18 -0
  85. package/dist/config/datasetAgentCheck.js +109 -0
  86. package/dist/config/datasetCandidates.d.ts +22 -0
  87. package/dist/config/datasetCandidates.js +86 -0
  88. package/dist/config/datasetFromSessions.d.ts +34 -0
  89. package/dist/config/datasetFromSessions.js +105 -0
  90. package/dist/config/datasetHash.d.ts +1 -0
  91. package/dist/config/datasetHash.js +4 -0
  92. package/dist/config/datasetImport.d.ts +21 -0
  93. package/dist/config/datasetImport.js +120 -0
  94. package/dist/config/datasetStageCheck.d.ts +9 -0
  95. package/dist/config/datasetStageCheck.js +17 -0
  96. package/dist/config/datasetTypes.d.ts +27 -0
  97. package/dist/config/datasetTypes.js +1 -0
  98. package/dist/config/ensureCatalogScanRoot.d.ts +4 -1
  99. package/dist/config/ensureCatalogScanRoot.js +8 -6
  100. package/dist/config/evalCatalog.d.ts +22 -0
  101. package/dist/config/evalCatalog.js +95 -0
  102. package/dist/config/evalWriteRoots.d.ts +17 -0
  103. package/dist/config/evalWriteRoots.js +109 -0
  104. package/dist/config/githubTriggerAction.d.ts +8 -0
  105. package/dist/config/githubTriggerAction.js +29 -0
  106. package/dist/config/hostConfig.js +3 -0
  107. package/dist/config/installEvalSample.d.ts +34 -0
  108. package/dist/config/installEvalSample.js +182 -0
  109. package/dist/config/loadDataset.d.ts +16 -0
  110. package/dist/config/loadDataset.js +318 -0
  111. package/dist/config/loadEval.d.ts +32 -0
  112. package/dist/config/loadEval.js +1103 -0
  113. package/dist/config/loadOutcome.d.ts +5 -0
  114. package/dist/config/loadStageflowManifest.js +64 -0
  115. package/dist/config/pipelineNeeds.d.ts +2 -0
  116. package/dist/config/pipelineNeeds.js +16 -3
  117. package/dist/config/pipelineRoute.js +5 -4
  118. package/dist/config/resolveAgentTarget.d.ts +32 -0
  119. package/dist/config/resolveAgentTarget.js +118 -0
  120. package/dist/config/resolveEvalTarget.d.ts +21 -0
  121. package/dist/config/resolveEvalTarget.js +89 -0
  122. package/dist/config/scanCatalogPaths.d.ts +1 -1
  123. package/dist/config/scanCatalogPaths.js +17 -5
  124. package/dist/config/validateCatalog.d.ts +2 -2
  125. package/dist/config/validateCatalog.js +21 -1
  126. package/dist/config/validateEval.d.ts +36 -0
  127. package/dist/config/validateEval.js +446 -0
  128. package/dist/config/writeDataset.d.ts +54 -0
  129. package/dist/config/writeDataset.js +186 -0
  130. package/dist/config/writeEval.d.ts +25 -0
  131. package/dist/config/writeEval.js +398 -0
  132. package/dist/diagnostics/checks.js +1 -1
  133. package/dist/envelope/assertEmittedArtifacts.d.ts +12 -0
  134. package/dist/envelope/assertEmittedArtifacts.js +43 -0
  135. package/dist/envelope/preEmitChecks.d.ts +3 -0
  136. package/dist/evals/agentDiff.d.ts +13 -0
  137. package/dist/evals/agentDiff.js +11 -0
  138. package/dist/evals/canonical.d.ts +3 -0
  139. package/dist/evals/canonical.js +24 -0
  140. package/dist/evals/caseTask.d.ts +3 -0
  141. package/dist/evals/caseTask.js +9 -0
  142. package/dist/evals/compare.d.ts +94 -0
  143. package/dist/evals/compare.js +201 -0
  144. package/dist/evals/constants.d.ts +62 -0
  145. package/dist/evals/constants.js +97 -0
  146. package/dist/evals/engine/FakeEngine.d.ts +33 -0
  147. package/dist/evals/engine/FakeEngine.js +177 -0
  148. package/dist/evals/engine/index.d.ts +9 -0
  149. package/dist/evals/engine/index.js +19 -0
  150. package/dist/evals/engine/promptfoo/PromptfooEngine.d.ts +52 -0
  151. package/dist/evals/engine/promptfoo/PromptfooEngine.js +312 -0
  152. package/dist/evals/engine/promptfoo/errors.d.ts +5 -0
  153. package/dist/evals/engine/promptfoo/errors.js +8 -0
  154. package/dist/evals/engine/promptfoo/generate.d.ts +30 -0
  155. package/dist/evals/engine/promptfoo/generate.js +345 -0
  156. package/dist/evals/engine/promptfoo/guard.d.ts +10 -0
  157. package/dist/evals/engine/promptfoo/guard.js +116 -0
  158. package/dist/evals/engine/promptfoo/install.d.ts +17 -0
  159. package/dist/evals/engine/promptfoo/install.js +73 -0
  160. package/dist/evals/engine/promptfoo/judgeProviders.d.ts +2 -0
  161. package/dist/evals/engine/promptfoo/judgeProviders.js +9 -0
  162. package/dist/evals/engine/promptfoo/judgeServices.d.ts +6 -0
  163. package/dist/evals/engine/promptfoo/judgeServices.js +41 -0
  164. package/dist/evals/engine/promptfoo/locate.d.ts +21 -0
  165. package/dist/evals/engine/promptfoo/locate.js +68 -0
  166. package/dist/evals/engine/promptfoo/nodeVersion.d.ts +7 -0
  167. package/dist/evals/engine/promptfoo/nodeVersion.js +14 -0
  168. package/dist/evals/engine/promptfoo/normalize.d.ts +66 -0
  169. package/dist/evals/engine/promptfoo/normalize.js +261 -0
  170. package/dist/evals/engine/promptfoo/pinned.d.ts +5 -0
  171. package/dist/evals/engine/promptfoo/pinned.js +5 -0
  172. package/dist/evals/engine/promptfoo/rubricPrompt.d.ts +14 -0
  173. package/dist/evals/engine/promptfoo/rubricPrompt.js +105 -0
  174. package/dist/evals/engine/promptfoo/runTokens.d.ts +18 -0
  175. package/dist/evals/engine/promptfoo/runTokens.js +31 -0
  176. package/dist/evals/engine/promptfoo/status.d.ts +13 -0
  177. package/dist/evals/engine/promptfoo/status.js +16 -0
  178. package/dist/evals/estimate.d.ts +56 -0
  179. package/dist/evals/estimate.js +66 -0
  180. package/dist/evals/graders/codeGrader.d.ts +6 -0
  181. package/dist/evals/graders/codeGrader.js +85 -0
  182. package/dist/evals/graders/conversationCheckGrader.d.ts +6 -0
  183. package/dist/evals/graders/conversationCheckGrader.js +62 -0
  184. package/dist/evals/graders/fieldGrader.d.ts +11 -0
  185. package/dist/evals/graders/fieldGrader.js +117 -0
  186. package/dist/evals/graders/goal.d.ts +3 -0
  187. package/dist/evals/graders/goal.js +14 -0
  188. package/dist/evals/graders/index.d.ts +18 -0
  189. package/dist/evals/graders/index.js +34 -0
  190. package/dist/evals/graders/notApplicable.d.ts +5 -0
  191. package/dist/evals/graders/notApplicable.js +9 -0
  192. package/dist/evals/graders/path.d.ts +26 -0
  193. package/dist/evals/graders/path.js +75 -0
  194. package/dist/evals/graders/referenceGrader.d.ts +5 -0
  195. package/dist/evals/graders/referenceGrader.js +59 -0
  196. package/dist/evals/graders/schemaGrader.d.ts +5 -0
  197. package/dist/evals/graders/schemaGrader.js +29 -0
  198. package/dist/evals/graders/subject.d.ts +6 -0
  199. package/dist/evals/graders/subject.js +33 -0
  200. package/dist/evals/graders/toolCallsGrader.d.ts +6 -0
  201. package/dist/evals/graders/toolCallsGrader.js +100 -0
  202. package/dist/evals/graders/transcript.d.ts +12 -0
  203. package/dist/evals/graders/transcript.js +57 -0
  204. package/dist/evals/graders/types.d.ts +20 -0
  205. package/dist/evals/graders/types.js +1 -0
  206. package/dist/evals/judgeProviders.d.ts +16 -0
  207. package/dist/evals/judgeProviders.js +27 -0
  208. package/dist/evals/lineDiff.d.ts +5 -0
  209. package/dist/evals/lineDiff.js +34 -0
  210. package/dist/evals/listStatus.d.ts +31 -0
  211. package/dist/evals/listStatus.js +48 -0
  212. package/dist/evals/manager.d.ts +187 -0
  213. package/dist/evals/manager.js +795 -0
  214. package/dist/evals/plan.d.ts +65 -0
  215. package/dist/evals/plan.js +621 -0
  216. package/dist/evals/planInternalAgent.d.ts +19 -0
  217. package/dist/evals/planInternalAgent.js +205 -0
  218. package/dist/evals/providerKeys.d.ts +1 -0
  219. package/dist/evals/providerKeys.js +21 -0
  220. package/dist/evals/read/agentTargets.d.ts +37 -0
  221. package/dist/evals/read/agentTargets.js +101 -0
  222. package/dist/evals/read/api.d.ts +208 -0
  223. package/dist/evals/read/api.js +363 -0
  224. package/dist/evals/read/compare.d.ts +4 -0
  225. package/dist/evals/read/compare.js +39 -0
  226. package/dist/evals/read/datasets.d.ts +105 -0
  227. package/dist/evals/read/datasets.js +318 -0
  228. package/dist/evals/read/list.d.ts +66 -0
  229. package/dist/evals/read/list.js +182 -0
  230. package/dist/evals/read/results.d.ts +46 -0
  231. package/dist/evals/read/results.js +107 -0
  232. package/dist/evals/read/runs.d.ts +154 -0
  233. package/dist/evals/read/runs.js +295 -0
  234. package/dist/evals/read/suggestions.d.ts +43 -0
  235. package/dist/evals/read/suggestions.js +73 -0
  236. package/dist/evals/runner/FakeAgentBackend.d.ts +40 -0
  237. package/dist/evals/runner/FakeAgentBackend.js +82 -0
  238. package/dist/evals/runner/agentCaseRunner.d.ts +24 -0
  239. package/dist/evals/runner/agentCaseRunner.js +388 -0
  240. package/dist/evals/runner/caseRunner.d.ts +8 -0
  241. package/dist/evals/runner/caseRunner.js +28 -0
  242. package/dist/evals/runner/conversationSubject.d.ts +18 -0
  243. package/dist/evals/runner/conversationSubject.js +123 -0
  244. package/dist/evals/runner/internalAgentCaseRunner.d.ts +6 -0
  245. package/dist/evals/runner/internalAgentCaseRunner.js +250 -0
  246. package/dist/evals/runner/simulatorModel.d.ts +51 -0
  247. package/dist/evals/runner/simulatorModel.js +58 -0
  248. package/dist/evals/runner/simulatorPrompt.d.ts +29 -0
  249. package/dist/evals/runner/simulatorPrompt.js +59 -0
  250. package/dist/evals/runner/stageCaseRunner.d.ts +30 -0
  251. package/dist/evals/runner/stageCaseRunner.js +279 -0
  252. package/dist/evals/runner/userDriver.d.ts +71 -0
  253. package/dist/evals/runner/userDriver.js +104 -0
  254. package/dist/evals/scoring.d.ts +34 -0
  255. package/dist/evals/scoring.js +67 -0
  256. package/dist/evals/snapshot.d.ts +191 -0
  257. package/dist/evals/snapshot.js +70 -0
  258. package/dist/evals/stageDiff.d.ts +8 -0
  259. package/dist/evals/stageDiff.js +48 -0
  260. package/dist/evals/stats.d.ts +95 -0
  261. package/dist/evals/stats.js +220 -0
  262. package/dist/evals/statsAdapter.d.ts +4 -0
  263. package/dist/evals/statsAdapter.js +30 -0
  264. package/dist/evals/statsTypes.d.ts +51 -0
  265. package/dist/evals/statsTypes.js +18 -0
  266. package/dist/evals/storeError.d.ts +5 -0
  267. package/dist/evals/storeError.js +8 -0
  268. package/dist/evals/storeTypes.d.ts +207 -0
  269. package/dist/evals/storeTypes.js +1 -0
  270. package/dist/evals/tTable.d.ts +3 -0
  271. package/dist/evals/tTable.js +15 -0
  272. package/dist/evals/triage/constants.d.ts +26 -0
  273. package/dist/evals/triage/constants.js +33 -0
  274. package/dist/evals/triage/credentials.d.ts +13 -0
  275. package/dist/evals/triage/credentials.js +20 -0
  276. package/dist/evals/triage/jevClassifier.d.ts +14 -0
  277. package/dist/evals/triage/jevClassifier.js +124 -0
  278. package/dist/evals/triage/judgeClassifier.d.ts +10 -0
  279. package/dist/evals/triage/judgeClassifier.js +76 -0
  280. package/dist/evals/triage/prompts.d.ts +16 -0
  281. package/dist/evals/triage/prompts.js +64 -0
  282. package/dist/evals/triage/service.d.ts +79 -0
  283. package/dist/evals/triage/service.js +217 -0
  284. package/dist/evals/triage/stateBuilder.d.ts +19 -0
  285. package/dist/evals/triage/stateBuilder.js +125 -0
  286. package/dist/evals/triage/types.d.ts +60 -0
  287. package/dist/evals/triage/types.js +8 -0
  288. package/dist/evals/types.d.ts +357 -0
  289. package/dist/evals/types.js +4 -0
  290. package/dist/guide/cardContract.d.ts +133 -0
  291. package/dist/guide/cardContract.js +156 -0
  292. package/dist/guide/chat.d.ts +45 -0
  293. package/dist/guide/chat.js +378 -0
  294. package/dist/guide/knowledge/index.d.ts +14 -0
  295. package/dist/guide/knowledge/index.js +109 -0
  296. package/dist/guide/knowledge/topics/agents.d.ts +2 -0
  297. package/dist/guide/knowledge/topics/agents.js +58 -0
  298. package/dist/guide/knowledge/topics/bestPractices.d.ts +2 -0
  299. package/dist/guide/knowledge/topics/bestPractices.js +67 -0
  300. package/dist/guide/knowledge/topics/connections.d.ts +2 -0
  301. package/dist/guide/knowledge/topics/connections.js +66 -0
  302. package/dist/guide/knowledge/topics/consoleMap.d.ts +2 -0
  303. package/dist/guide/knowledge/topics/consoleMap.js +55 -0
  304. package/dist/guide/knowledge/topics/evals.d.ts +2 -0
  305. package/dist/guide/knowledge/topics/evals.js +72 -0
  306. package/dist/guide/knowledge/topics/limitations.d.ts +2 -0
  307. package/dist/guide/knowledge/topics/limitations.js +84 -0
  308. package/dist/guide/knowledge/topics/mcpSkillsBrowser.d.ts +2 -0
  309. package/dist/guide/knowledge/topics/mcpSkillsBrowser.js +57 -0
  310. package/dist/guide/knowledge/topics/memories.d.ts +2 -0
  311. package/dist/guide/knowledge/topics/memories.js +63 -0
  312. package/dist/guide/knowledge/topics/modelsAndProviders.d.ts +2 -0
  313. package/dist/guide/knowledge/topics/modelsAndProviders.js +56 -0
  314. package/dist/guide/knowledge/topics/overview.d.ts +2 -0
  315. package/dist/guide/knowledge/topics/overview.js +56 -0
  316. package/dist/guide/knowledge/topics/pipelines.d.ts +2 -0
  317. package/dist/guide/knowledge/topics/pipelines.js +93 -0
  318. package/dist/guide/knowledge/topics/stages.d.ts +2 -0
  319. package/dist/guide/knowledge/topics/stages.js +54 -0
  320. package/dist/guide/knowledge/topics/tasks.d.ts +2 -0
  321. package/dist/guide/knowledge/topics/tasks.js +52 -0
  322. package/dist/guide/knowledge/topics/triggers.d.ts +2 -0
  323. package/dist/guide/knowledge/topics/triggers.js +68 -0
  324. package/dist/guide/knowledge/topics/troubleshooting.d.ts +2 -0
  325. package/dist/guide/knowledge/topics/troubleshooting.js +85 -0
  326. package/dist/guide/knowledgeTools.d.ts +2 -0
  327. package/dist/guide/knowledgeTools.js +56 -0
  328. package/dist/guide/showUi.d.ts +6 -0
  329. package/dist/guide/showUi.js +546 -0
  330. package/dist/guide/threadStore.d.ts +48 -0
  331. package/dist/guide/threadStore.js +189 -0
  332. package/dist/guide/writeRoot.d.ts +11 -0
  333. package/dist/guide/writeRoot.js +70 -0
  334. package/dist/hosted/translate.js +2 -0
  335. package/dist/mcp/agentTools.js +12 -0
  336. package/dist/mcp/catalogTools.js +18 -20
  337. package/dist/mcp/catalogWriteTools.d.ts +8 -0
  338. package/dist/mcp/catalogWriteTools.js +225 -0
  339. package/dist/mcp/connectionTools.d.ts +3 -0
  340. package/dist/mcp/connectionTools.js +85 -0
  341. package/dist/mcp/controlTools.js +34 -1
  342. package/dist/mcp/deps.d.ts +16 -0
  343. package/dist/mcp/evalTools.d.ts +3 -0
  344. package/dist/mcp/evalTools.js +943 -0
  345. package/dist/mcp/memoryHostTools.d.ts +3 -0
  346. package/dist/mcp/memoryHostTools.js +72 -0
  347. package/dist/mcp/pipelineAuthorTools.js +14 -1
  348. package/dist/mcp/server.js +6 -0
  349. package/dist/mcp/tools.js +8 -0
  350. package/dist/mcp/triggerTools.js +77 -31
  351. package/dist/models/response.d.ts +7 -2
  352. package/dist/models/response.js +6 -1
  353. package/dist/operatorAgent/agentAuthorContext.d.ts +1 -1
  354. package/dist/operatorAgent/agentAuthorContext.js +1 -1
  355. package/dist/operatorAgent/evalGate.d.ts +22 -0
  356. package/dist/operatorAgent/evalGate.js +77 -0
  357. package/dist/operatorAgent/fakeHost.js +10 -2
  358. package/dist/operatorAgent/host.js +18 -2
  359. package/dist/operatorAgent/index.d.ts +1 -0
  360. package/dist/operatorAgent/index.js +1 -0
  361. package/dist/operatorAgent/internalAgents.d.ts +11 -0
  362. package/dist/operatorAgent/internalAgents.js +13 -0
  363. package/dist/operatorAgent/piModel.d.ts +5 -1
  364. package/dist/operatorAgent/piModel.js +32 -14
  365. package/dist/operatorAgent/profiles/agentAuthor.d.ts +3 -1
  366. package/dist/operatorAgent/profiles/agentAuthor.js +201 -32
  367. package/dist/operatorAgent/profiles/agentAuthorPlaybook.d.ts +1 -1
  368. package/dist/operatorAgent/profiles/agentAuthorPlaybook.js +48 -15
  369. package/dist/operatorAgent/profiles/authorReadable.d.ts +2 -0
  370. package/dist/operatorAgent/profiles/authorReadable.js +26 -0
  371. package/dist/operatorAgent/profiles/guideConfirm.d.ts +21 -0
  372. package/dist/operatorAgent/profiles/guideConfirm.js +100 -0
  373. package/dist/operatorAgent/profiles/guideTools.d.ts +7 -0
  374. package/dist/operatorAgent/profiles/guideTools.js +123 -0
  375. package/dist/operatorAgent/profiles/stageflowGuide.d.ts +7 -0
  376. package/dist/operatorAgent/profiles/stageflowGuide.js +132 -0
  377. package/dist/operatorAgent/profiles/workshopAuthor.js +23 -0
  378. package/dist/operatorAgent/profiles/workshopAuthorPlaybook.d.ts +1 -1
  379. package/dist/operatorAgent/profiles/workshopAuthorPlaybook.js +9 -1
  380. package/dist/operatorAgent/profiles/workshopAuthorTools.d.ts +2 -0
  381. package/dist/operatorAgent/profiles/workshopAuthorTools.js +197 -9
  382. package/dist/operatorAgent/types.d.ts +23 -0
  383. package/dist/package-meta.d.ts +1 -1
  384. package/dist/package-meta.js +1 -1
  385. package/dist/runstore/port.d.ts +43 -0
  386. package/dist/runstore/retention.d.ts +2 -1
  387. package/dist/runstore/retention.js +3 -7
  388. package/dist/runstore/runStatusClock.d.ts +17 -0
  389. package/dist/runstore/runStatusClock.js +19 -0
  390. package/dist/runstore/sqlite/SqliteRunStore.d.ts +29 -0
  391. package/dist/runstore/sqlite/SqliteRunStore.js +144 -50
  392. package/dist/runstore/sqlite/evalStore.d.ts +20 -0
  393. package/dist/runstore/sqlite/evalStore.js +541 -0
  394. package/dist/runstore/sqlite/migrations/015-evals.d.ts +12 -0
  395. package/dist/runstore/sqlite/migrations/015-evals.js +157 -0
  396. package/dist/runstore/sqlite/migrations/016-eval-subject-ref.d.ts +12 -0
  397. package/dist/runstore/sqlite/migrations/016-eval-subject-ref.js +20 -0
  398. package/dist/runstore/sqlite/migrations/017-eval-result-suggestions.d.ts +11 -0
  399. package/dist/runstore/sqlite/migrations/017-eval-result-suggestions.js +36 -0
  400. package/dist/runstore/sqlite/migrations/index.js +6 -0
  401. package/dist/runstore/sqlite/suggestionStore.d.ts +14 -0
  402. package/dist/runstore/sqlite/suggestionStore.js +132 -0
  403. package/dist/runstore/workspaceLayout.js +6 -1
  404. package/dist/runtime/artifactDiskInspect.d.ts +14 -0
  405. package/dist/runtime/artifactDiskInspect.js +52 -0
  406. package/dist/runtime/artifactPath.d.ts +24 -0
  407. package/dist/runtime/artifactPath.js +84 -0
  408. package/dist/runtime/completionCheckRunner.js +15 -43
  409. package/dist/runtime/githubPollSource.d.ts +22 -4
  410. package/dist/runtime/githubPollSource.js +93 -8
  411. package/dist/runtime/pipelineRunner.d.ts +2 -1
  412. package/dist/runtime/pipelineRunner.js +2 -0
  413. package/dist/runtime/priorArtifactGrants.d.ts +8 -0
  414. package/dist/runtime/priorArtifactGrants.js +105 -0
  415. package/dist/runtime/runActionEligibility.d.ts +46 -0
  416. package/dist/runtime/runActionEligibility.js +123 -0
  417. package/dist/runtime/runManager.d.ts +6 -2
  418. package/dist/runtime/runManager.js +60 -16
  419. package/dist/runtime/runRetentionSweep.js +2 -2
  420. package/dist/runtime/runRetryCoordinator.d.ts +2 -14
  421. package/dist/runtime/runRetryCoordinator.js +3 -67
  422. package/dist/runtime/sandboxContainer.d.ts +7 -0
  423. package/dist/runtime/sandboxContainer.js +48 -6
  424. package/dist/runtime/sandboxDesk.d.ts +2 -1
  425. package/dist/runtime/sandboxDesk.js +11 -12
  426. package/dist/runtime/stageEnvironment.js +1 -0
  427. package/dist/runtime/stageRoots.d.ts +12 -0
  428. package/dist/runtime/stageSandbox.d.ts +2 -1
  429. package/dist/runtime/stageSandbox.js +18 -2
  430. package/dist/runtime/stageWorker.js +20 -0
  431. package/dist/server/agentAuthorRoutes.js +59 -1
  432. package/dist/server/bootstrap.d.ts +13 -0
  433. package/dist/server/bootstrap.js +69 -0
  434. package/dist/server/controlToken.js +3 -0
  435. package/dist/server/evalInternalRoutes.d.ts +14 -0
  436. package/dist/server/evalInternalRoutes.js +87 -0
  437. package/dist/server/evalRouteContext.d.ts +46 -0
  438. package/dist/server/evalRouteContext.js +43 -0
  439. package/dist/server/evalRoutes.d.ts +5 -0
  440. package/dist/server/evalRoutes.js +1456 -0
  441. package/dist/server/evalTriageRoutes.d.ts +2 -0
  442. package/dist/server/evalTriageRoutes.js +95 -0
  443. package/dist/server/guideRoutes.d.ts +5 -0
  444. package/dist/server/guideRoutes.js +166 -0
  445. package/dist/server/http.d.ts +15 -0
  446. package/dist/server/http.js +67 -3
  447. package/dist/server/internalAgentRoutes.d.ts +6 -0
  448. package/dist/server/internalAgentRoutes.js +12 -0
  449. package/dist/server/mcpHost.js +5 -0
  450. package/dist/toolkit/meta.d.ts +1 -1
  451. package/dist/toolkit/meta.js +53 -0
  452. package/dist/toolkit/profiles.d.ts +5 -0
  453. package/dist/toolkit/profiles.js +13 -1
  454. package/dist/tools/askOperator.js +2 -1
  455. package/dist/tools/emitStageEnvelope.d.ts +17 -16
  456. package/dist/tools/emitStageEnvelope.js +46 -32
  457. package/dist/tools/stageToolNames.d.ts +4 -0
  458. package/dist/tools/stageToolNames.js +4 -0
  459. package/dist/tools/workingState.d.ts +40 -0
  460. package/dist/tools/workingState.js +108 -0
  461. package/dist/tools/writeStageArtifact.d.ts +0 -9
  462. package/dist/tools/writeStageArtifact.js +2 -1
  463. package/dist/types/stageflowManifest.d.ts +6 -0
  464. package/dist/ui/assets/{MarkdownText-D7tYaIFO.js → MarkdownText-S5C_vemc.js} +2 -2
  465. package/dist/ui/assets/adapters-C0l4ruiR.js +17 -0
  466. package/dist/ui/assets/chat-CjnsmvqR.js +2 -0
  467. package/dist/ui/assets/chat-DFDd4UjS.css +2 -0
  468. package/dist/ui/assets/{main-BFo79krC.css → main-Del1GeLf.css} +1 -1
  469. package/dist/ui/assets/main-_yXVqcW9.js +277 -0
  470. package/dist/ui/chat.html +3 -4
  471. package/dist/ui/index.html +3 -4
  472. package/dist/workshop/chatTurn.js +1 -1
  473. package/dist/workshop/workshopPin.d.ts +3 -3
  474. package/dist/workshop/workshopPin.js +16 -15
  475. package/examples/README.md +2 -0
  476. package/examples/compaction-stress/README.md +30 -0
  477. package/examples/compaction-stress/check-run.mjs +53 -0
  478. package/examples/compaction-stress/collect-evidence.yaml +60 -0
  479. package/examples/compaction-stress/compaction-stress.pipeline.yaml +141 -0
  480. package/examples/compaction-stress/compaction-stress.task.yaml +15 -0
  481. package/examples/compaction-stress/generate-evidence.mjs +110 -0
  482. package/examples/compaction-stress/root-cause.yaml +26 -0
  483. package/examples/compaction-stress/stageflow.yaml +9 -0
  484. package/examples/compaction-stress/write-postmortem.yaml +40 -0
  485. package/examples/document-policy-review/README.md +35 -0
  486. package/examples/document-policy-review/agents/compliance.agent.yaml +42 -0
  487. package/examples/document-policy-review/documents/acme-vendor-sow.md +15 -0
  488. package/examples/document-policy-review/documents/support-runbook.md +13 -0
  489. package/examples/document-policy-review/policies/acceptable-use.md +13 -0
  490. package/examples/document-policy-review/policies/data-handling.md +17 -0
  491. package/examples/document-policy-review/policies/vendor-security.md +17 -0
  492. package/examples/evals/README.md +141 -0
  493. package/examples/evals/critique-sample.dataset.jsonl +8 -0
  494. package/examples/evals/critique-sample.eval.yaml +34 -0
  495. package/examples/evals/critique-sample.task.yaml +4 -0
  496. package/examples/evals/eval-sample.pipeline.yaml +35 -0
  497. package/examples/evals/guide-behavior.dataset.jsonl +15 -0
  498. package/examples/evals/guide-behavior.eval.yaml +35 -0
  499. package/examples/evals/guide-depth.dataset.jsonl +35 -0
  500. package/examples/evals/guide-depth.eval.yaml +169 -0
  501. package/examples/evals/support-sample.agent.yaml +20 -0
  502. package/examples/evals/support-sample.dataset.jsonl +6 -0
  503. package/examples/evals/support-sample.eval.yaml +38 -0
  504. package/examples/memory/evals/guide-behavior.dataset.jsonl +15 -0
  505. package/examples/memory/evals/guide-behavior.eval.yaml +35 -0
  506. package/examples/memory/evals/guide-depth.dataset.jsonl +35 -0
  507. package/examples/memory/evals/guide-depth.eval.yaml +169 -0
  508. package/examples/stageflow.yaml +6 -0
  509. package/package.json +1 -1
  510. package/skills/stageflow/SKILL.md +5 -2
  511. package/skills/stageflow/references/control-surface.md +4 -2
  512. package/skills/stageflow/references/evals.md +68 -0
  513. package/skills/stageflow-author/SKILL.md +4 -0
  514. package/skills/stageflow-run/SKILL.md +4 -0
  515. package/dist/ui/assets/AssistantRuntimeProvider-CsiPFhCT.js +0 -9
  516. package/dist/ui/assets/ReadonlyStore-BfXqHykm.js +0 -6
  517. package/dist/ui/assets/chat-BbLukq9W.js +0 -5
  518. package/dist/ui/assets/chat-zIZU_bP1.css +0 -2
  519. package/dist/ui/assets/main-CijYMeM0.js +0 -259
@@ -0,0 +1,3 @@
1
+ import type { McpServer } from "@modelcontextprotocol/server";
2
+ import type { McpToolDeps } from "./deps.js";
3
+ export declare function registerMemoryHostTools(server: McpServer, deps: McpToolDeps): void;
@@ -0,0 +1,72 @@
1
+ import { z } from "zod";
2
+ import { MemoryError } from "../memory/host.js";
3
+ import { textResult } from "./toolResults.js";
4
+ async function call(deps, work) {
5
+ const host = deps.memories;
6
+ if (!host)
7
+ return textResult({ error: "MEMORIES_UNAVAILABLE" }, true);
8
+ try {
9
+ return textResult(await work(host));
10
+ }
11
+ catch (error) {
12
+ if (error instanceof MemoryError)
13
+ return textResult({ error: error.code }, true);
14
+ throw error;
15
+ }
16
+ }
17
+ export function registerMemoryHostTools(server, deps) {
18
+ server.registerTool("list_memories", {
19
+ description: "List host memories (id, description, and card count).",
20
+ inputSchema: z.object({}),
21
+ }, () => call(deps, async (host) => ({ memories: await host.list() })));
22
+ server.registerTool("get_memory", {
23
+ description: "Read one host memory: its description, cover page, and card keys.",
24
+ inputSchema: z.object({
25
+ id: z.string(),
26
+ }),
27
+ }, ({ id }) => call(deps, async (host) => {
28
+ const meta = await host.readMeta(id);
29
+ const body = await host.read(id);
30
+ const keys = await host.listCards(id);
31
+ return { ...meta, body, keys };
32
+ }));
33
+ server.registerTool("create_memory", {
34
+ description: "Create a host memory with an id and description. The cover page starts empty.",
35
+ inputSchema: z.object({
36
+ id: z.string(),
37
+ description: z.string(),
38
+ }),
39
+ }, ({ id, description }) => call(deps, (host) => host.create({ id, description })));
40
+ server.registerTool("write_memory", {
41
+ description: "Replace the cover page of a host memory.",
42
+ inputSchema: z.object({
43
+ id: z.string(),
44
+ body: z.string(),
45
+ }),
46
+ }, ({ id, body }) => call(deps, async (host) => {
47
+ await host.replace(id, body);
48
+ const meta = await host.readMeta(id);
49
+ return { ...meta, body };
50
+ }));
51
+ server.registerTool("write_memory_card", {
52
+ description: "Replace an existing host memory card. A missing card is not created.",
53
+ inputSchema: z.object({
54
+ id: z.string(),
55
+ key: z.string(),
56
+ body: z.string(),
57
+ }),
58
+ }, ({ id, key, body }) => call(deps, async (host) => {
59
+ await host.updateCard(id, key, body);
60
+ return { body };
61
+ }));
62
+ server.registerTool("delete_memory_card", {
63
+ description: "Delete one card from a host memory.",
64
+ inputSchema: z.object({
65
+ id: z.string(),
66
+ key: z.string(),
67
+ }),
68
+ }, ({ id, key }) => call(deps, async (host) => {
69
+ await host.deleteCard(id, key);
70
+ return { removed: true };
71
+ }));
72
+ }
@@ -88,7 +88,20 @@ export function registerPipelineAuthorTools(server, deps) {
88
88
  }, ({ path: relPath, allow_invalid, ...args }) => mutate(relPath, "overwrite", (draft) => buildCreateStageDraft(draft, args), { allowInvalid: allow_invalid }));
89
89
  server.registerTool("edit_stage", {
90
90
  description: "Edit an existing stage on disk by id: body fields and/or its pipeline entry (route, entry, uses).",
91
- inputSchema: z.looseObject({ path: pipelinePath, id: z.string(), allow_invalid: allowInvalid }),
91
+ inputSchema: z.object({
92
+ path: pipelinePath,
93
+ id: z.string(),
94
+ patch: record.optional(),
95
+ body: record.optional(),
96
+ pipeline_entry: record.optional(),
97
+ entry: z.boolean().optional(),
98
+ route: z.array(record).optional(),
99
+ uses: z.string().optional(),
100
+ system_prompt: z.string().optional(),
101
+ model: z.string().optional(),
102
+ io: record.optional(),
103
+ allow_invalid: allowInvalid,
104
+ }),
92
105
  }, ({ path: relPath, allow_invalid, ...args }) => mutate(relPath, "overwrite", (draft) => buildEditStageDraft(draft, args), { allowInvalid: allow_invalid }));
93
106
  server.registerTool("create_task", {
94
107
  description: "Create or replace the task beside a pipeline on disk (id and goal, optional body).",
@@ -49,6 +49,12 @@ function toolDepsFrom(deps) {
49
49
  ...(deps.agentConversationBackend !== undefined
50
50
  ? { agentConversationBackend: deps.agentConversationBackend }
51
51
  : {}),
52
+ ...(deps.evalRunManager !== undefined ? { evalRunManager: deps.evalRunManager } : {}),
53
+ ...(deps.evalEngineStatus !== undefined ? { evalEngineStatus: deps.evalEngineStatus } : {}),
54
+ ...(deps.triage !== undefined ? { triage: deps.triage } : {}),
55
+ ...(deps.channels !== undefined ? { channels: deps.channels } : {}),
56
+ ...(deps.memories !== undefined ? { memories: deps.memories } : {}),
57
+ ...(deps.issueFetch !== undefined ? { issueFetch: deps.issueFetch } : {}),
52
58
  };
53
59
  }
54
60
  function attachBusNotifications(server, bus) {
package/dist/mcp/tools.js CHANGED
@@ -5,11 +5,15 @@ import { registerControlTools } from "./controlTools.js";
5
5
  import { registerProjectMcpTools } from "./projectMcpTools.js";
6
6
  import { registerPipelineAuthorTools } from "./pipelineAuthorTools.js";
7
7
  import { registerProviderTools } from "./providerTools.js";
8
+ import { registerEvalTools } from "./evalTools.js";
8
9
  import { registerTriggerTools } from "./triggerTools.js";
9
10
  import { textResult } from "./toolResults.js";
10
11
  import { DEFAULT_TIMEOUT_MS, waitRun } from "./waitRun.js";
11
12
  import { registerAgentTools } from "./agentTools.js";
13
+ import { registerCatalogWriteTools } from "./catalogWriteTools.js";
14
+ import { registerConnectionTools } from "./connectionTools.js";
12
15
  import { registerGetStartedTool } from "./getStartedTool.js";
16
+ import { registerMemoryHostTools } from "./memoryHostTools.js";
13
17
  export function registerMcpTools(server, deps) {
14
18
  const { store } = deps;
15
19
  registerGetStartedTool(server, deps);
@@ -19,7 +23,11 @@ export function registerMcpTools(server, deps) {
19
23
  registerProviderTools(server, deps);
20
24
  registerProjectMcpTools(server, deps);
21
25
  registerTriggerTools(server, deps);
26
+ registerEvalTools(server, deps);
22
27
  registerPipelineAuthorTools(server, deps);
28
+ registerCatalogWriteTools(server, deps);
29
+ registerConnectionTools(server, deps);
30
+ registerMemoryHostTools(server, deps);
23
31
  registerAgentTools(server, deps);
24
32
  server.registerTool("wait_run", {
25
33
  description: "Long-poll until a run is waiting for HITL input and/or terminal (succeeded/failed), or until timeout_ms. Default until=any wakes on waiting OR terminal. Abort cancels only this wait (not the run). Optional notifications/progress when the client supplies _meta.progressToken. Compose: wait_run → answer_gate → wait_run.",
@@ -2,6 +2,7 @@ import { z } from "zod";
2
2
  import { CatalogPathError, catalogPathErrorBody, resolveWritableCatalogRoot, } from "../config/catalogRelativePath.js";
3
3
  import { createTrigger, parseCreateTriggerBody } from "../config/createTrigger.js";
4
4
  import { parsePatchTriggerBody, parsePutTriggerBody, updateTriggerDefinition, updateTriggerEnabled } from "../config/updateTrigger.js";
5
+ import { readTriggerAdapterStatus } from "../config/triggerAdapterStatus.js";
5
6
  import { loadCatalogTriggers } from "../config/triggerCatalog.js";
6
7
  import { writeAudit } from "../logging/audit.js";
7
8
  import { logger as rootLogger } from "../logging/logger.js";
@@ -41,23 +42,58 @@ const triggerEventSchema = z.object({
41
42
  match: z.record(z.string(), z.unknown()).optional(),
42
43
  config: z.record(z.string(), z.unknown()).optional(),
43
44
  });
44
- const createTriggerSchema = z.object({
45
- project_root: z
46
- .string()
47
- .optional()
48
- .describe("Catalog project root to write into; required when multiple catalog roots are configured and no sole non-seeded root exists"),
45
+ function findListedTrigger(items, id, projectRoot) {
46
+ if (projectRoot) {
47
+ const found = items.find((item) => item.id === id && item.project_root === projectRoot);
48
+ if (!found)
49
+ return { error: `Trigger not found: ${id}`, status: 404 };
50
+ return found;
51
+ }
52
+ const matches = items.filter((item) => item.id === id);
53
+ if (matches.length === 0)
54
+ return { error: `Trigger not found: ${id}`, status: 404 };
55
+ if (matches.length > 1) {
56
+ return {
57
+ error: `Trigger id is ambiguous across catalog roots: ${id}`,
58
+ status: 409,
59
+ project_roots: [...new Set(matches.map((item) => item.project_root))].sort((a, b) => a.localeCompare(b)),
60
+ };
61
+ }
62
+ return matches[0];
63
+ }
64
+ const triggerProjectRoot = z
65
+ .string()
66
+ .optional()
67
+ .describe("Catalog project root to write into; required when multiple catalog roots are configured and no sole non-seeded root exists");
68
+ const triggerTask = z
69
+ .string()
70
+ .optional()
71
+ .describe("Catalog task id; omit to create a dynamic-mode trigger whose task is supplied at fire time");
72
+ const createTriggerBase = {
73
+ project_root: triggerProjectRoot,
49
74
  directory: z.string(),
50
75
  id: z.string(),
51
76
  pipeline: z.string(),
52
- task: z
53
- .string()
54
- .optional()
55
- .describe("Catalog task id; omit to create a dynamic-mode trigger whose task is supplied at fire time"),
56
- kind: z.enum(["manual", "schedule", "event"]),
57
- schedule: triggerScheduleSchema.optional(),
58
- event: triggerEventSchema.optional(),
77
+ task: triggerTask,
59
78
  enabled: z.boolean().optional(),
60
- });
79
+ };
80
+ const createTriggerSchema = z.discriminatedUnion("kind", [
81
+ z.object({ ...createTriggerBase, kind: z.literal("manual") }),
82
+ z.object({ ...createTriggerBase, kind: z.literal("schedule"), schedule: triggerScheduleSchema }),
83
+ z.object({ ...createTriggerBase, kind: z.literal("event"), event: triggerEventSchema }),
84
+ ]);
85
+ const updateTriggerBase = {
86
+ id: z.string(),
87
+ project_root: z.string().optional(),
88
+ pipeline: z.string(),
89
+ task: z.string().optional(),
90
+ enabled: z.boolean(),
91
+ };
92
+ const updateTriggerSchema = z.discriminatedUnion("kind", [
93
+ z.object({ ...updateTriggerBase, kind: z.literal("manual") }),
94
+ z.object({ ...updateTriggerBase, kind: z.literal("schedule"), schedule: triggerScheduleSchema }),
95
+ z.object({ ...updateTriggerBase, kind: z.literal("event"), event: triggerEventSchema }),
96
+ ]);
61
97
  export function registerTriggerTools(server, deps) {
62
98
  const { store, cwd, manager } = deps;
63
99
  server.registerTool("list_triggers", {
@@ -65,32 +101,51 @@ export function registerTriggerTools(server, deps) {
65
101
  inputSchema: z.object({}),
66
102
  }, async () => {
67
103
  const items = await loadCatalogTriggers({ store, bootCwd: cwd });
68
- return textResult({ triggers: items });
104
+ const triggers = await Promise.all(items.map(async (item) => ({
105
+ ...item,
106
+ adapter_status: await readTriggerAdapterStatus(item, store),
107
+ })));
108
+ return textResult({ triggers });
69
109
  });
70
110
  server.registerTool("get_trigger", {
71
111
  description: "Look up one catalog trigger by id (same source as GET /api/triggers/:id).",
72
112
  inputSchema: z.object({
73
113
  id: z.string(),
114
+ project_root: z
115
+ .string()
116
+ .optional()
117
+ .describe("Catalog project root; required when the same trigger id exists in more than one root"),
74
118
  }),
75
- }, async ({ id }) => {
119
+ }, async ({ id, project_root }) => {
76
120
  const items = await loadCatalogTriggers({ store, bootCwd: cwd });
77
- const found = items.find((item) => item.id === id);
78
- if (!found) {
79
- return textResult({ error: `Trigger not found: ${id}`, status: 404 }, true);
121
+ const found = findListedTrigger(items, id, project_root);
122
+ if ("error" in found) {
123
+ return textResult({ error: found.error, status: found.status, ...(found.project_roots ? { project_roots: found.project_roots } : {}) }, true);
80
124
  }
81
- return textResult(found);
125
+ return textResult({
126
+ ...found,
127
+ adapter_status: await readTriggerAdapterStatus(found, store),
128
+ });
82
129
  });
83
130
  server.registerTool("fire_trigger", {
84
131
  description: "Fire a catalog trigger by id: resolves its pipeline/task refs and starts a run through the same RunManager.startRun path as start_run (same as POST /api/triggers/:id/fire). Returns { runId } or { runId, queued: true, queuePosition }. Fails not_found (404) for an unknown id, or 409 when the trigger is disabled.",
85
132
  inputSchema: z.object({
86
133
  id: z.string(),
134
+ project_root: z
135
+ .string()
136
+ .optional()
137
+ .describe("Catalog project root; required when the same trigger id exists in more than one root"),
87
138
  task: taskFileSchema
88
139
  .optional()
89
140
  .describe("Inline TaskFile, required to fire a dynamic-mode trigger (no catalog task) and rejected for a catalog-mode trigger"),
90
141
  }),
91
- }, async ({ id, task }) => {
142
+ }, async ({ id, task, project_root }) => {
92
143
  const callerId = callerIdFromRequestAuth();
93
- const result = await fireTrigger(id, store, manager, { cwd, task });
144
+ const result = await fireTrigger(id, store, manager, {
145
+ cwd,
146
+ task,
147
+ ...(project_root ? { project_root } : {}),
148
+ });
94
149
  if (!result.ok) {
95
150
  writeAudit(auditLog, {
96
151
  caller_id: callerId,
@@ -170,16 +225,7 @@ export function registerTriggerTools(server, deps) {
170
225
  });
171
226
  server.registerTool("update_trigger", {
172
227
  description: "Replace a trigger's definition (pipeline, task, kind, schedule or event, enabled). Same validation and write path as PUT /api/triggers/:id; the id cannot change.",
173
- inputSchema: z.object({
174
- id: z.string(),
175
- project_root: z.string().optional(),
176
- pipeline: z.string(),
177
- task: z.string().optional(),
178
- kind: z.enum(["manual", "schedule", "event"]),
179
- schedule: triggerScheduleSchema.optional(),
180
- event: triggerEventSchema.optional(),
181
- enabled: z.boolean().optional(),
182
- }),
228
+ inputSchema: updateTriggerSchema,
183
229
  }, async ({ id, project_root, ...body }) => {
184
230
  const parsed = parsePutTriggerBody(body, id);
185
231
  if ("ok" in parsed)
@@ -1,11 +1,16 @@
1
1
  import type { RunStore } from "../runstore/port.js";
2
2
  import { listModelsMultiProject } from "../config/multiProjectCatalog.js";
3
- import { type CatalogProvider } from "./catalog.js";
3
+ import { type CatalogModel, type CatalogProvider } from "./catalog.js";
4
+ export type ModelsResponseProvider = Omit<CatalogProvider, "models"> & {
5
+ models: Array<CatalogModel & {
6
+ judge_capable: boolean;
7
+ }>;
8
+ };
4
9
  export type ModelsResponse = {
5
10
  /** Catalog ids plus ids pipelines already use, sorted. Back-compatible with the old flat list. */
6
11
  models: string[];
7
12
  /** Connected providers with their models. */
8
- providers: CatalogProvider[];
13
+ providers: ModelsResponseProvider[];
9
14
  /** The model a new chat, stage or agent starts on. */
10
15
  default: string;
11
16
  fetchedAt: string;
@@ -1,4 +1,5 @@
1
1
  import { listModelsMultiProject } from "../config/multiProjectCatalog.js";
2
+ import { judgeCapableModels } from "../evals/judgeProviders.js";
2
3
  import { getModelCatalog } from "./catalog.js";
3
4
  /** The shared answer for GET /api/models, the MCP `list_models` tool and every picker. */
4
5
  export async function buildModelsResponse(options) {
@@ -9,12 +10,16 @@ export async function buildModelsResponse(options) {
9
10
  catalog.defaultModel(),
10
11
  ]);
11
12
  const listed = new Set(snapshot.models);
13
+ const judgeCapable = new Set(judgeCapableModels(snapshot.providers.flatMap((p) => p.models.map((m) => m.id))));
12
14
  const unlisted = used.models.filter((id) => !listed.has(id));
13
15
  return {
14
16
  used,
15
17
  response: {
16
18
  models: [...new Set([...snapshot.models, ...used.models])].sort((a, b) => a.localeCompare(b)),
17
- providers: snapshot.providers,
19
+ providers: snapshot.providers.map((provider) => ({
20
+ ...provider,
21
+ models: provider.models.map((model) => ({ ...model, judge_capable: judgeCapable.has(model.id) })),
22
+ })),
18
23
  default: defaultModel,
19
24
  fetchedAt: snapshot.fetchedAt,
20
25
  unlisted,
@@ -10,7 +10,7 @@ export type AgentAuthorContext = {
10
10
  projectRoot?: string;
11
11
  draft: AgentAuthorDraft;
12
12
  };
13
- export declare const AGENT_AUTHOR_GREETING = "What should this agent do? Tell me the job, the tools it needs, and anything it must never do without asking.";
13
+ export declare const AGENT_AUTHOR_GREETING = "Who is this agent for, and what does a good result look like? I'll ask a few questions before I fill the config.";
14
14
  export declare function emptyAgentAuthorDraft(): AgentAuthorDraft;
15
15
  export declare function isAgentAuthorContext(value: unknown): value is AgentAuthorContext;
16
16
  export declare function readAgentAuthorDraft(context: unknown): AgentAuthorDraft;
@@ -1,4 +1,4 @@
1
- export const AGENT_AUTHOR_GREETING = "What should this agent do? Tell me the job, the tools it needs, and anything it must never do without asking.";
1
+ export const AGENT_AUTHOR_GREETING = "Who is this agent for, and what does a good result look like? I'll ask a few questions before I fill the config.";
2
2
  export function emptyAgentAuthorDraft() {
3
3
  return {};
4
4
  }
@@ -0,0 +1,22 @@
1
+ import { type EvalToolCallRecord, type EvalToolPolicy } from "../agents/evalGate.js";
2
+ import type { OperatorAgentToolResult } from "./types.js";
3
+ export type OperatorEvalPolicy = {
4
+ tools: EvalToolPolicy;
5
+ mocks: Record<string, {
6
+ result?: unknown;
7
+ error?: string;
8
+ }>;
9
+ };
10
+ /**
11
+ * Tool gate for an eval of a built-in operator agent. Reads may run. Writes, sends and deletes
12
+ * are stubbed or mocked and never reach the handler. `real` is treated as read-only: these
13
+ * agents write the operator's own project, so an eval cannot opt into that.
14
+ */
15
+ export declare function createOperatorEvalGate(options: {
16
+ policy: OperatorEvalPolicy;
17
+ turn: () => number;
18
+ }): {
19
+ decide(name: string, args: Record<string, unknown>): OperatorAgentToolResult | null;
20
+ observe(name: string, result: OperatorAgentToolResult): void;
21
+ records(): EvalToolCallRecord[];
22
+ };
@@ -0,0 +1,77 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import { STUB_RESULT } from "../agents/evalGate.js";
3
+ import { TOOL_META } from "../toolkit/meta.js";
4
+ const ALWAYS_BLOCKED = new Set([
5
+ "run_eval",
6
+ "agent_chat",
7
+ "fire_trigger",
8
+ "sort_eval_failures",
9
+ "cancel_eval_run",
10
+ "start_run",
11
+ "run_stage",
12
+ "rerun",
13
+ "retry_stage",
14
+ "resume_stage",
15
+ "delete_run",
16
+ "cancel_run",
17
+ "gc_runs",
18
+ "remove_connection",
19
+ "delete_eval_run",
20
+ "delete_memory_card",
21
+ "archive_agent",
22
+ "enable_agent",
23
+ "set_trigger_enabled",
24
+ "test_connection",
25
+ "import_task_file",
26
+ "probe_project_mcp",
27
+ ]);
28
+ /** Tools that only talk to the person or read Stageflow's own docs. They never change the project. */
29
+ const SAFE = new Set(["confirm_action", "show_ui", "list_guide_topics", "read_guide_topic"]);
30
+ /**
31
+ * Tool gate for an eval of a built-in operator agent. Reads may run. Writes, sends and deletes
32
+ * are stubbed or mocked and never reach the handler. `real` is treated as read-only: these
33
+ * agents write the operator's own project, so an eval cannot opt into that.
34
+ */
35
+ export function createOperatorEvalGate(options) {
36
+ const records = [];
37
+ let pending;
38
+ const push = (record) => {
39
+ records.push(record);
40
+ };
41
+ return {
42
+ decide(name, args) {
43
+ const base = { id: randomUUID(), turn: options.turn(), name, args };
44
+ const mock = options.policy.mocks[name];
45
+ const deny = (result, policyOutcome) => {
46
+ push({ ...base, policyOutcome, status: "not_run", result });
47
+ return { ok: false, error: "eval.tool_not_run", content: { error: "eval.tool_not_run", message: result } };
48
+ };
49
+ if (ALWAYS_BLOCKED.has(name))
50
+ return deny(`${name} cannot run in an evaluation.`, "denied");
51
+ if (mock && (options.policy.tools === "mocked" || !mayRun(name))) {
52
+ const text = mock.error ?? (typeof mock.result === "string" ? mock.result : JSON.stringify(mock.result ?? null));
53
+ push({ ...base, policyOutcome: "mocked", status: mock.error !== undefined ? "error" : "ok", result: text });
54
+ return { ok: mock.error === undefined, ...(mock.error !== undefined ? { error: mock.error } : {}), content: mock.result ?? mock.error };
55
+ }
56
+ if (!mayRun(name))
57
+ return deny(STUB_RESULT(name), "stubbed");
58
+ pending = { ...base, policyOutcome: "ran", status: "ok" };
59
+ push(pending);
60
+ return null;
61
+ },
62
+ observe(_name, result) {
63
+ if (!pending)
64
+ return;
65
+ pending.result = typeof result.content === "string" ? result.content : JSON.stringify(result.content ?? null);
66
+ if (!result.ok)
67
+ pending.status = "error";
68
+ pending = undefined;
69
+ },
70
+ records: () => records,
71
+ };
72
+ }
73
+ function mayRun(name) {
74
+ if (SAFE.has(name))
75
+ return true;
76
+ return TOOL_META[name]?.risk === "read";
77
+ }
@@ -1,5 +1,6 @@
1
1
  import { createOperatorAgentHost, invokeProfileTool, } from "./host.js";
2
2
  import { createAgentAuthorProfile } from "./profiles/agentAuthor.js";
3
+ import { createStageflowGuideProfile } from "./profiles/stageflowGuide.js";
3
4
  import { createWorkshopAuthorProfile, isTaskProposalIntent, proposeStageFromUserMessage, proposeTaskFromUserMessage, WORKSHOP_AUTHOR_PROFILE_ID, } from "./profiles/workshopAuthor.js";
4
5
  /**
5
6
  * Deterministic Operator Agent Host for Workshop and tests.
@@ -8,7 +9,7 @@ import { createWorkshopAuthorProfile, isTaskProposalIntent, proposeStageFromUser
8
9
  export function createFakeOperatorAgentModel(script = [{ type: "propose_stage" }]) {
9
10
  let index = 0;
10
11
  return {
11
- async complete({ profile, message, tools }) {
12
+ async complete({ profile, message, tools, onActivity }) {
12
13
  const turn = script[Math.min(index, script.length - 1)] ?? {
13
14
  type: "propose_stage",
14
15
  };
@@ -28,7 +29,13 @@ export function createFakeOperatorAgentModel(script = [{ type: "propose_stage" }
28
29
  };
29
30
  }
30
31
  if (turn.type === "call_tool") {
31
- const result = await invokeProfileTool(profile, turn.name, turn.args ?? {}, tools);
32
+ const args = turn.args ?? {};
33
+ const callId = `${turn.name}-${index}`;
34
+ onActivity?.({ id: callId, name: turn.name, status: "running", args });
35
+ const result = await invokeProfileTool(profile, turn.name, args, tools);
36
+ onActivity?.(result.ok
37
+ ? { id: callId, name: turn.name, status: "complete", result: result.content }
38
+ : { id: callId, name: turn.name, status: "error", errorMessage: result.error ?? "unknown error" });
32
39
  const events = [
33
40
  { type: "tool_result", name: turn.name, result },
34
41
  ];
@@ -88,6 +95,7 @@ export function createWorkshopOperatorHost(scriptOrOptions) {
88
95
  projectRoot: options.projectRoot,
89
96
  }),
90
97
  createAgentAuthorProfile(),
98
+ createStageflowGuideProfile(),
91
99
  ]);
92
100
  return host;
93
101
  }
@@ -1,7 +1,7 @@
1
1
  import { profileMayUse } from "../toolkit/profiles.js";
2
2
  import { enrichProposal, isMutationConflict, operatorContextFingerprint, UNDO_MUTATION_CONFLICT_NOTICE, } from "./proposals.js";
3
3
  import { rejectUnlinkedWorkshopDraftTool } from "../workshop/workshopPin.js";
4
- function createSession(profile, initialContext, model) {
4
+ function createSession(profile, initialContext, model, evalGate) {
5
5
  let context = initialContext;
6
6
  let closed = false;
7
7
  const mutations = new Map();
@@ -23,6 +23,7 @@ function createSession(profile, initialContext, model) {
23
23
  lastMutationId = applied.id;
24
24
  lastEmitted = applied;
25
25
  },
26
+ ...(evalGate ? { evalGate } : {}),
26
27
  };
27
28
  function resolveMutation(mutationId) {
28
29
  if (mutations.size === 0)
@@ -165,6 +166,16 @@ export function createOperatorAgentHost(model, seedProfiles = []) {
165
166
  }
166
167
  return createSession(profile, context, model);
167
168
  },
169
+ openEvalSession({ profileId, context, gate }) {
170
+ if (!gate) {
171
+ throw new Error("An internal-agent eval session requires a tool gate");
172
+ }
173
+ const profile = profiles.get(profileId);
174
+ if (!profile) {
175
+ throw new Error(`Unknown operator agent profile: ${profileId}`);
176
+ }
177
+ return createSession(profile, context, model, gate);
178
+ },
168
179
  };
169
180
  }
170
181
  export async function invokeProfileTool(profile, name, args, ctx) {
@@ -178,5 +189,10 @@ export async function invokeProfileTool(profile, name, args, ctx) {
178
189
  const blocked = rejectUnlinkedWorkshopDraftTool(ctx.getContext(), name);
179
190
  if (blocked)
180
191
  return blocked;
181
- return tool.handler(args, ctx);
192
+ const refused = ctx.evalGate?.decide(name, args);
193
+ if (refused)
194
+ return refused;
195
+ const result = await tool.handler(args, ctx);
196
+ ctx.evalGate?.observe(name, result);
197
+ return result;
182
198
  }
@@ -6,6 +6,7 @@ export { createLiveWorkshopOperatorHost, createPiOperatorAgentModel, type LiveWo
6
6
  export { createFailingDocsRetriever, createFilesystemDocsRetriever, createStubDocsRetriever, resolveWorkshopCatalogRoot, type DocsRetrievalHit, type DocsRetrievalResult, type DocsRetriever, } from "./docsRetrieval.js";
7
7
  export { agentAuthorContextAdapter, emptyAgentAuthorDraft, isAgentAuthorContext, readAgentAuthorDraft, AGENT_AUTHOR_GREETING, type AgentAuthorContext, type AgentAuthorDraft, } from "./agentAuthorContext.js";
8
8
  export { AGENT_AUTHOR_PROFILE_ID, AGENT_AUTHOR_TOOL_NAMES, createAgentAuthorContext, createAgentAuthorProfile, } from "./profiles/agentAuthor.js";
9
+ export { STAGEFLOW_GUIDE_GREETING, STAGEFLOW_GUIDE_PLAYBOOK, STAGEFLOW_GUIDE_PROFILE_ID, STAGEFLOW_GUIDE_TOOL_NAMES, createStageflowGuideProfile, } from "./profiles/stageflowGuide.js";
9
10
  export { AGENT_AUTHOR_PLAYBOOK } from "./profiles/agentAuthorPlaybook.js";
10
11
  export { operatorContextFingerprint } from "./proposals.js";
11
12
  export { createWorkshopDraftContext, draftTrackStages, emptyDraftPackage, isWorkshopDraftContext, readDraftFromContext, withDestination, withDraft, withProjectRoot, workshopDraftContextAdapter, WORKSHOP_AUTHOR_GREETING, } from "./draftContext.js";
@@ -5,6 +5,7 @@ export { createLiveWorkshopOperatorHost, createPiOperatorAgentModel, } from "./p
5
5
  export { createFailingDocsRetriever, createFilesystemDocsRetriever, createStubDocsRetriever, resolveWorkshopCatalogRoot, } from "./docsRetrieval.js";
6
6
  export { agentAuthorContextAdapter, emptyAgentAuthorDraft, isAgentAuthorContext, readAgentAuthorDraft, AGENT_AUTHOR_GREETING, } from "./agentAuthorContext.js";
7
7
  export { AGENT_AUTHOR_PROFILE_ID, AGENT_AUTHOR_TOOL_NAMES, createAgentAuthorContext, createAgentAuthorProfile, } from "./profiles/agentAuthor.js";
8
+ export { STAGEFLOW_GUIDE_GREETING, STAGEFLOW_GUIDE_PLAYBOOK, STAGEFLOW_GUIDE_PROFILE_ID, STAGEFLOW_GUIDE_TOOL_NAMES, createStageflowGuideProfile, } from "./profiles/stageflowGuide.js";
8
9
  export { AGENT_AUTHOR_PLAYBOOK } from "./profiles/agentAuthorPlaybook.js";
9
10
  export { operatorContextFingerprint } from "./proposals.js";
10
11
  export { createWorkshopDraftContext, draftTrackStages, emptyDraftPackage, isWorkshopDraftContext, readDraftFromContext, withDestination, withDraft, withProjectRoot, workshopDraftContextAdapter, WORKSHOP_AUTHOR_GREETING, } from "./draftContext.js";
@@ -0,0 +1,11 @@
1
+ import type { OperatorAgentHost } from "./types.js";
2
+ export declare const SHOW_INTERNAL_AGENTS_ENV = "STAGEFLOW_SHOW_INTERNAL_AGENTS";
3
+ export type InternalAgentInfo = {
4
+ id: string;
5
+ title: string;
6
+ greeting?: string;
7
+ playbook: string;
8
+ tools: string[];
9
+ };
10
+ export declare function internalAgentsVisible(env?: NodeJS.ProcessEnv): boolean;
11
+ export declare function describeInternalAgents(host: OperatorAgentHost): InternalAgentInfo[];
@@ -0,0 +1,13 @@
1
+ export const SHOW_INTERNAL_AGENTS_ENV = "STAGEFLOW_SHOW_INTERNAL_AGENTS";
2
+ export function internalAgentsVisible(env = process.env) {
3
+ return env[SHOW_INTERNAL_AGENTS_ENV] === "1";
4
+ }
5
+ export function describeInternalAgents(host) {
6
+ return host.listProfiles().map((profile) => ({
7
+ id: profile.id,
8
+ title: profile.title,
9
+ ...(profile.greeting !== undefined ? { greeting: profile.greeting } : {}),
10
+ playbook: profile.playbook,
11
+ tools: profile.tools.map((tool) => tool.name),
12
+ }));
13
+ }
@@ -1,5 +1,6 @@
1
1
  import { type AgentSession, defineTool, SessionManager } from "@earendil-works/pi-coding-agent";
2
2
  import "../agent/cursorProvider.js";
3
+ import type { McpToolDeps } from "../mcp/deps.js";
3
4
  import { type OperatorAgentModel } from "./host.js";
4
5
  import { type WorkshopAuthorProfileOptions } from "./profiles/workshopAuthor.js";
5
6
  import type { OperatorAgentHost, OperatorAgentProfile, OperatorAgentToolContext, OperatorAgentToolResult, WorkshopToolActivityUpdate } from "./types.js";
@@ -76,5 +77,8 @@ export declare function workshopToolActivity(id: string, name: string, phase: "s
76
77
  * Pi OperatorAgentModel: durable session per host tool-context, Workshop tools only.
77
78
  */
78
79
  export declare function createPiOperatorAgentModel(options: PiOperatorAgentModelOptions): PiOperatorAgentModel;
79
- export type LiveWorkshopOperatorHostOptions = WorkshopAuthorProfileOptions & PiOperatorAgentModelOptions;
80
+ export type LiveWorkshopOperatorHostOptions = WorkshopAuthorProfileOptions & PiOperatorAgentModelOptions & {
81
+ /** When set, Guide can call the Stageflow catalog. Without it, Guide only talks. */
82
+ guideDeps?: McpToolDeps;
83
+ };
80
84
  export declare function createLiveWorkshopOperatorHost(options: LiveWorkshopOperatorHostOptions): OperatorAgentHost;