synabun 2.0.10 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +86 -0
- package/README.md +25 -2
- package/THIRD-PARTY-LICENSES.md +22 -0
- package/bin/synabun.js +71 -24
- package/docs/whatsapp.md +2 -2
- package/hooks/claude-code/pre-websearch.mjs +12 -1
- package/hooks/claude-code/shared.mjs +5 -0
- package/hooks/claude-code/stop.mjs +1 -1
- package/lib/claude-hooks.js +172 -18
- package/lib/client-config-repair.js +17 -8
- package/lib/desktop-app.js +7 -1
- package/lib/external-tools.js +1 -1
- package/lib/first-launch-protection.js +57 -12
- package/lib/mcp-dist.js +176 -0
- package/lib/release-npm-ci.mjs +138 -0
- package/lib/release-publish.mjs +91 -0
- package/lib/release-scan.mjs +50 -0
- package/lib/server-crash-record.js +123 -0
- package/lib/server-exit.js +318 -0
- package/lib/server-supervisor.js +508 -0
- package/lib/start-launcher.js +37 -4
- package/lib/tools-cli.js +45 -0
- package/lib/update-snapshot.js +228 -0
- package/lib/virustotal.mjs +192 -0
- package/lib/windows-crash-event.js +269 -0
- package/mcp-server/dist/http.d.ts +20 -1
- package/mcp-server/dist/http.d.ts.map +1 -1
- package/mcp-server/dist/http.js +59 -12
- package/mcp-server/dist/http.js.map +1 -1
- package/mcp-server/dist/index.d.ts +33 -27
- package/mcp-server/dist/index.d.ts.map +1 -1
- package/mcp-server/dist/index.js +128 -143
- package/mcp-server/dist/index.js.map +1 -1
- package/mcp-server/dist/preload.js +1 -1
- package/mcp-server/dist/preload.js.map +1 -1
- package/mcp-server/dist/services/codex-config-heal.d.ts +6 -2
- package/mcp-server/dist/services/codex-config-heal.d.ts.map +1 -1
- package/mcp-server/dist/services/codex-config-heal.js +81 -28
- package/mcp-server/dist/services/codex-config-heal.js.map +1 -1
- package/mcp-server/dist/services/memory-judge-backfill.d.ts +6 -0
- package/mcp-server/dist/services/memory-judge-backfill.d.ts.map +1 -1
- package/mcp-server/dist/services/memory-judge-backfill.js +7 -1
- package/mcp-server/dist/services/memory-judge-backfill.js.map +1 -1
- package/mcp-server/dist/services/neural-interface.d.ts +16 -10
- package/mcp-server/dist/services/neural-interface.d.ts.map +1 -1
- package/mcp-server/dist/services/neural-interface.js +67 -11
- package/mcp-server/dist/services/neural-interface.js.map +1 -1
- package/mcp-server/dist/services/server-instructions.d.ts +7 -2
- package/mcp-server/dist/services/server-instructions.d.ts.map +1 -1
- package/mcp-server/dist/services/server-instructions.js +20 -62
- package/mcp-server/dist/services/server-instructions.js.map +1 -1
- package/mcp-server/dist/services/tool-catalog.d.ts +120 -0
- package/mcp-server/dist/services/tool-catalog.d.ts.map +1 -0
- package/mcp-server/dist/services/tool-catalog.js +339 -0
- package/mcp-server/dist/services/tool-catalog.js.map +1 -0
- package/mcp-server/dist/services/tool-list-refresh-barrier.d.ts +12 -0
- package/mcp-server/dist/services/tool-list-refresh-barrier.d.ts.map +1 -0
- package/mcp-server/dist/services/tool-list-refresh-barrier.js +40 -0
- package/mcp-server/dist/services/tool-list-refresh-barrier.js.map +1 -0
- package/mcp-server/dist/services/tool-list.d.ts +108 -0
- package/mcp-server/dist/services/tool-list.d.ts.map +1 -0
- package/mcp-server/dist/services/tool-list.js +334 -0
- package/mcp-server/dist/services/tool-list.js.map +1 -0
- package/mcp-server/dist/services/tool-runtime.d.ts +161 -0
- package/mcp-server/dist/services/tool-runtime.d.ts.map +1 -0
- package/mcp-server/dist/services/tool-runtime.js +400 -0
- package/mcp-server/dist/services/tool-runtime.js.map +1 -0
- package/mcp-server/dist/services/whiteboard-geometry.d.ts +26 -2
- package/mcp-server/dist/services/whiteboard-geometry.d.ts.map +1 -1
- package/mcp-server/dist/services/whiteboard-geometry.js +91 -8
- package/mcp-server/dist/services/whiteboard-geometry.js.map +1 -1
- package/mcp-server/dist/tools/agents.d.ts +10 -10
- package/mcp-server/dist/tools/agents.d.ts.map +1 -1
- package/mcp-server/dist/tools/agents.js +22 -13
- package/mcp-server/dist/tools/agents.js.map +1 -1
- package/mcp-server/dist/tools/browser-batch.d.ts +3 -3
- package/mcp-server/dist/tools/image-tools.d.ts +1 -1
- package/mcp-server/dist/tools/image-tools.js +1 -1
- package/mcp-server/dist/tools/skill-tools.d.ts +505 -0
- package/mcp-server/dist/tools/skill-tools.d.ts.map +1 -0
- package/mcp-server/dist/tools/skill-tools.js +639 -0
- package/mcp-server/dist/tools/skill-tools.js.map +1 -0
- package/mcp-server/dist/tools/skill.d.ts +7 -0
- package/mcp-server/dist/tools/skill.d.ts.map +1 -0
- package/mcp-server/dist/tools/skill.js +11 -0
- package/mcp-server/dist/tools/skill.js.map +1 -0
- package/mcp-server/dist/tools/tool-groups.d.ts +68 -0
- package/mcp-server/dist/tools/tool-groups.d.ts.map +1 -0
- package/mcp-server/dist/tools/tool-groups.js +147 -0
- package/mcp-server/dist/tools/tool-groups.js.map +1 -0
- package/mcp-server/dist/tools/whiteboard-format.d.ts.map +1 -1
- package/mcp-server/dist/tools/whiteboard-format.js +7 -4
- package/mcp-server/dist/tools/whiteboard-format.js.map +1 -1
- package/mcp-server/dist/tools/whiteboard-tools.d.ts +64 -13
- package/mcp-server/dist/tools/whiteboard-tools.d.ts.map +1 -1
- package/mcp-server/dist/tools/whiteboard-tools.js +30 -5
- package/mcp-server/dist/tools/whiteboard-tools.js.map +1 -1
- package/mcp-server/package-lock.json +2 -2
- package/mcp-server/package.json +1 -1
- package/mcp-server/src/http.ts +75 -12
- package/mcp-server/src/index.ts +147 -154
- package/mcp-server/src/preload.ts +12 -12
- package/mcp-server/src/services/codex-config-heal.ts +80 -29
- package/mcp-server/src/services/memory-judge-backfill.ts +7 -1
- package/mcp-server/src/services/neural-interface.ts +79 -20
- package/mcp-server/src/services/server-instructions.ts +27 -63
- package/mcp-server/src/services/tool-catalog.ts +403 -0
- package/mcp-server/src/services/tool-list-refresh-barrier.ts +33 -0
- package/mcp-server/src/services/tool-list.ts +410 -0
- package/mcp-server/src/services/tool-runtime.ts +457 -0
- package/mcp-server/src/services/whiteboard-geometry.ts +94 -9
- package/mcp-server/src/tools/agents.ts +22 -14
- package/mcp-server/src/tools/image-tools.ts +1 -1
- package/mcp-server/src/tools/skill-tools.ts +589 -0
- package/mcp-server/src/tools/skill.ts +13 -0
- package/mcp-server/src/tools/tool-groups.ts +164 -0
- package/mcp-server/src/tools/whiteboard-format.ts +9 -3
- package/mcp-server/src/tools/whiteboard-tools.ts +39 -5
- package/neural-interface/i18n/en.json +3463 -200
- package/neural-interface/i18n/pt-BR.json +3411 -151
- package/neural-interface/lib/assistant-api.js +127 -28
- package/neural-interface/lib/assistant-brains/claude.js +4 -3
- package/neural-interface/lib/assistant-brains/codex-browser-policy-hook.mjs +12 -2
- package/neural-interface/lib/assistant-brains/codex-route-gate-hook.mjs +32 -9
- package/neural-interface/lib/assistant-brains/codex.js +9 -5
- package/neural-interface/lib/assistant-brains/opencode.js +120 -54
- package/neural-interface/lib/assistant-brains/opencode2-browser-policy/index.js +28 -13
- package/neural-interface/lib/assistant-brains/opencode2-route-gate/index.js +59 -19
- package/neural-interface/lib/assistant-catalog.js +6 -7
- package/neural-interface/lib/assistant-dispatch.js +871 -156
- package/neural-interface/lib/assistant-envelope.js +3 -1
- package/neural-interface/lib/assistant-persona.js +80 -11
- package/neural-interface/lib/assistant-plan-permissions.js +10 -9
- package/neural-interface/lib/assistant-route-gate-codex.js +3 -0
- package/neural-interface/lib/assistant-route-gate.js +6 -4
- package/neural-interface/lib/assistant-router.js +13 -8
- package/neural-interface/lib/assistant-runtime.js +56 -18
- package/neural-interface/lib/assistant-task-prompt.js +78 -14
- package/neural-interface/lib/assistant-tools.js +51 -0
- package/neural-interface/lib/backup-service.js +620 -3
- package/neural-interface/lib/backup-zip-reader.js +10 -5
- package/neural-interface/lib/backup-zip-writer.js +257 -0
- package/neural-interface/lib/browser-execution.js +10 -2
- package/neural-interface/lib/browser-guard-cdp.js +83 -0
- package/neural-interface/lib/browser-network-guard.js +284 -0
- package/neural-interface/lib/browser-policy-roots.js +64 -0
- package/neural-interface/lib/browser-tool-policy.js +186 -31
- package/neural-interface/lib/claude-agent-bridge.js +4 -2
- package/neural-interface/lib/claude-hooks-status.js +103 -0
- package/neural-interface/lib/claude-temporary.js +5 -11
- package/neural-interface/lib/cli-install.js +18 -138
- package/neural-interface/lib/cli-resolver.js +4 -1
- package/neural-interface/lib/cli-update-planner.js +32 -31
- package/neural-interface/lib/cloudflared/tunnel.js +42 -2
- package/neural-interface/lib/codex-mcp-approvals.js +11 -0
- package/neural-interface/lib/codex-worker-app-server.js +172 -0
- package/neural-interface/lib/confined-fs.js +30 -3
- package/neural-interface/lib/crash-notice-api.js +98 -0
- package/neural-interface/lib/editor-completion-opencode.js +68 -68
- package/neural-interface/lib/exec-loop-command.js +4 -8
- package/neural-interface/lib/http-guards.js +8 -2
- package/neural-interface/lib/install-details.js +4 -1
- package/neural-interface/lib/mcp-agent-servers.js +67 -0
- package/neural-interface/lib/mcp-client-config.js +0 -29
- package/neural-interface/lib/mcp-external-servers.js +323 -0
- package/neural-interface/lib/{mcp-profile-refresh-coordinator.js → mcp-tools-refresh-coordinator.js} +4 -4
- package/neural-interface/lib/morelogin.js +30 -0
- package/neural-interface/lib/native-loop-providers.js +547 -47
- package/neural-interface/lib/native-loop-runtime.js +268 -41
- package/neural-interface/lib/opencode-config.js +557 -0
- package/neural-interface/lib/opencode-db-schema.js +17 -34
- package/neural-interface/lib/opencode-history.js +48 -218
- package/neural-interface/lib/opencode-mcp-ready.js +108 -0
- package/neural-interface/lib/opencode-native-boundary.js +44 -0
- package/neural-interface/lib/opencode-protocol.js +249 -112
- package/neural-interface/lib/opencode-runtime-config.js +32 -22
- package/neural-interface/lib/opencode-serve-records.js +39 -21
- package/neural-interface/lib/opencode-v2-client.js +48 -153
- package/neural-interface/lib/opencode-v2-ws-requests.js +36 -210
- package/neural-interface/lib/opencode-worker-data.js +27 -0
- package/neural-interface/lib/opencode2-client.js +73 -37
- package/neural-interface/lib/opencode2-proxy.js +23 -63
- package/neural-interface/lib/opencode2-translate.js +51 -2
- package/neural-interface/lib/owner-authority.js +249 -0
- package/neural-interface/lib/page-session.js +275 -0
- package/neural-interface/lib/peer-identity.js +763 -0
- package/neural-interface/lib/request-gate.js +17 -2
- package/neural-interface/lib/rulesets/api.js +1 -1
- package/neural-interface/lib/rulesets/fs-safe.js +1 -1
- package/neural-interface/lib/rulesets/installer.js +92 -27
- package/neural-interface/lib/rulesets/json-edit.js +1 -1
- package/neural-interface/lib/session-cache-builder.js +17 -30
- package/neural-interface/lib/session-title-generator.js +58 -36
- package/neural-interface/lib/skills/api-v2.js +495 -0
- package/neural-interface/lib/skills/bundle.js +418 -0
- package/neural-interface/lib/skills/catalog.js +294 -0
- package/neural-interface/lib/skills/creation-api.js +191 -0
- package/neural-interface/lib/skills/creation-brief.js +247 -0
- package/neural-interface/lib/skills/creation-sources.js +267 -0
- package/neural-interface/lib/skills/creation.js +843 -0
- package/neural-interface/lib/skills/deploy.js +564 -0
- package/neural-interface/lib/skills/digest.js +119 -0
- package/neural-interface/lib/skills/discover.js +111 -0
- package/neural-interface/lib/skills/draft-confinement.js +214 -0
- package/neural-interface/lib/skills/eval-confinement.js +397 -0
- package/neural-interface/lib/skills/eval-sandbox.js +280 -0
- package/neural-interface/lib/skills/eval-trace.js +119 -0
- package/neural-interface/lib/skills/evals-api.js +212 -0
- package/neural-interface/lib/skills/evals.js +1573 -0
- package/neural-interface/lib/skills/export.js +200 -0
- package/neural-interface/lib/skills/frontmatter.js +64 -0
- package/neural-interface/lib/skills/health.js +75 -0
- package/neural-interface/lib/skills/json-edit.js +165 -0
- package/neural-interface/lib/skills/known-bridge-digests.json +387 -0
- package/neural-interface/lib/skills/known-bridges.js +96 -0
- package/neural-interface/lib/skills/library.js +823 -0
- package/neural-interface/lib/skills/local-store.js +21 -0
- package/neural-interface/lib/skills/migration.js +433 -0
- package/neural-interface/lib/skills/operations.js +45 -0
- package/neural-interface/lib/skills/ownership.js +427 -0
- package/neural-interface/lib/skills/packages.js +88 -0
- package/neural-interface/lib/skills/reviews.js +212 -0
- package/neural-interface/lib/skills/revision.js +80 -0
- package/neural-interface/lib/skills/roots.js +262 -0
- package/neural-interface/lib/skills/runtime-checks.js +355 -0
- package/neural-interface/lib/skills/scan-assets.js +252 -0
- package/neural-interface/lib/skills/scan-context.js +26 -0
- package/neural-interface/lib/skills/scan-shell.js +324 -0
- package/neural-interface/lib/skills/scan.js +364 -0
- package/neural-interface/lib/skills/scanners.js +54 -0
- package/neural-interface/lib/skills/service.js +522 -0
- package/neural-interface/lib/skills/telemetry.js +113 -0
- package/neural-interface/lib/sqlite-integrity-worker.js +22 -0
- package/neural-interface/lib/tool-lists.js +124 -0
- package/neural-interface/lib/whatsapp/bridge.js +6 -3
- package/neural-interface/lib/worker-confinement.js +1192 -0
- package/neural-interface/lib/worker-guard-setting.js +56 -0
- package/neural-interface/lib/worker-mcp-channel.js +175 -0
- package/neural-interface/lib/worker-relay-ports.js +227 -0
- package/neural-interface/lib/worker-resume.js +229 -0
- package/neural-interface/lib/worker-sandbox.js +299 -0
- package/neural-interface/package-lock.json +18 -12
- package/neural-interface/package.json +2 -2
- package/neural-interface/public/index.html +3 -1
- package/neural-interface/public/index2d.html +19 -11
- package/neural-interface/public/onboarding.html +2 -2
- package/neural-interface/public/shared/account-menu.js +8 -5
- package/neural-interface/public/shared/agent-runtime-options.js +35 -19
- package/neural-interface/public/shared/api.js +1527 -1336
- package/neural-interface/public/shared/app-frame.js +192 -0
- package/neural-interface/public/shared/assistant/asst-agents-dock.js +33 -2
- package/neural-interface/public/shared/assistant/asst-ask-styles.js +270 -0
- package/neural-interface/public/shared/assistant/asst-ask-tail.js +118 -0
- package/neural-interface/public/shared/assistant/asst-ask.js +397 -0
- package/neural-interface/public/shared/assistant/asst-brain-picker.js +49 -35
- package/neural-interface/public/shared/assistant/asst-control.js +157 -26
- package/neural-interface/public/shared/assistant/asst-followups-card.js +383 -0
- package/neural-interface/public/shared/assistant/asst-followups.js +417 -0
- package/neural-interface/public/shared/assistant/asst-limit-dock.js +40 -0
- package/neural-interface/public/shared/assistant/asst-limit-wait.js +294 -0
- package/neural-interface/public/shared/assistant/asst-limit.js +168 -0
- package/neural-interface/public/shared/assistant/asst-mascot.js +49 -12
- package/neural-interface/public/shared/assistant/asst-menu.js +7 -6
- package/neural-interface/public/shared/assistant/asst-panel.js +106 -11
- package/neural-interface/public/shared/assistant/asst-rack-styles.js +66 -49
- package/neural-interface/public/shared/assistant/asst-render.js +401 -77
- package/neural-interface/public/shared/assistant/asst-route.js +83 -12
- package/neural-interface/public/shared/assistant/asst-scenes.js +2 -0
- package/neural-interface/public/shared/assistant/asst-sidepanel.js +67 -4
- package/neural-interface/public/shared/assistant/asst-slash.js +9 -7
- package/neural-interface/public/shared/assistant/asst-state.js +11 -15
- package/neural-interface/public/shared/assistant/asst-stop.js +127 -0
- package/neural-interface/public/shared/assistant/asst-styles.js +59 -21
- package/neural-interface/public/shared/assistant/asst-tool-kinds.js +156 -16
- package/neural-interface/public/shared/automation-context.js +5 -2
- package/neural-interface/public/shared/cc-dropdown.js +89 -19
- package/neural-interface/public/shared/cdx/cdx-context-menu.js +10 -9
- package/neural-interface/public/shared/cdx/cdx-copy.js +16 -0
- package/neural-interface/public/shared/cdx/cdx-output-model.js +22 -1
- package/neural-interface/public/shared/cdx/cdx-output.js +8 -7
- package/neural-interface/public/shared/cdx/cdx-panel.js +45 -149
- package/neural-interface/public/shared/cdx/cdx-protocol.js +28 -1
- package/neural-interface/public/shared/cdx/cdx-render.js +47 -29
- package/neural-interface/public/shared/cdx/cdx-requests.js +3 -3
- package/neural-interface/public/shared/cdx/cdx-skill-hints.js +54 -0
- package/neural-interface/public/shared/cdx/cdx-styles.js +11 -3
- package/neural-interface/public/shared/cdx/cdx-tabs.js +52 -39
- package/neural-interface/public/shared/cdx/cdx-ws.js +0 -1
- package/neural-interface/public/shared/cli-install-view.js +4 -8
- package/neural-interface/public/shared/cli-status.js +2 -2
- package/neural-interface/public/shared/constants.js +2 -2
- package/neural-interface/public/shared/context-menu-areas.js +34 -0
- package/neural-interface/public/shared/context-menu-logic.js +243 -0
- package/neural-interface/public/shared/context-menu-model.js +366 -0
- package/neural-interface/public/shared/context-menu.js +372 -0
- package/neural-interface/public/shared/cp/cp-context-menu.js +10 -9
- package/neural-interface/public/shared/crash-notice-view.js +168 -0
- package/neural-interface/public/shared/desktop-update-view.js +20 -0
- package/neural-interface/public/shared/editor/editor-panel-logic.js +2 -2
- package/neural-interface/public/shared/html-shell.js +9 -11
- package/neural-interface/public/shared/i18n.js +2 -0
- package/neural-interface/public/shared/ocp-v2/ocp-v2-approvals.js +7 -7
- package/neural-interface/public/shared/ocp-v2/ocp-v2-binding.js +4 -4
- package/neural-interface/public/shared/ocp-v2/ocp-v2-changes-logic.js +3 -101
- package/neural-interface/public/shared/ocp-v2/ocp-v2-childpanel.js +2 -0
- package/neural-interface/public/shared/ocp-v2/ocp-v2-compact-button.js +1 -3
- package/neural-interface/public/shared/ocp-v2/ocp-v2-composer-logic.js +2 -6
- package/neural-interface/public/shared/ocp-v2/ocp-v2-context-menu.js +11 -11
- package/neural-interface/public/shared/ocp-v2/ocp-v2-context-model.js +0 -3
- package/neural-interface/public/shared/ocp-v2/ocp-v2-events.js +8 -4
- package/neural-interface/public/shared/ocp-v2/ocp-v2-manager.js +7 -7
- package/neural-interface/public/shared/ocp-v2/ocp-v2-outcome-notifications.js +16 -0
- package/neural-interface/public/shared/ocp-v2/ocp-v2-panel.js +47 -252
- package/neural-interface/public/shared/ocp-v2/ocp-v2-plan.js +1 -2
- package/neural-interface/public/shared/ocp-v2/ocp-v2-projectbar.js +95 -202
- package/neural-interface/public/shared/ocp-v2/ocp-v2-render.js +6 -21
- package/neural-interface/public/shared/ocp-v2/ocp-v2-send.js +4 -4
- package/neural-interface/public/shared/ocp-v2/ocp-v2-session-actions.js +8 -61
- package/neural-interface/public/shared/ocp-v2/ocp-v2-sessions-logic.js +2 -91
- package/neural-interface/public/shared/ocp-v2/ocp-v2-slash-hints.js +16 -15
- package/neural-interface/public/shared/ocp-v2/ocp-v2-state.js +8 -21
- package/neural-interface/public/shared/ocp-v2/ocp-v2-status-logic.js +1 -99
- package/neural-interface/public/shared/ocp-v2/ocp-v2-status.js +5 -35
- package/neural-interface/public/shared/ocp-v2/ocp-v2-styles.js +0 -21
- package/neural-interface/public/shared/ocp-v2/{ocp-v2-profile-inheritance.js → ocp-v2-tools-inheritance.js} +16 -16
- package/neural-interface/public/shared/ocp-v2/ocp-v2-ws.js +14 -47
- package/neural-interface/public/shared/page-session.js +96 -0
- package/neural-interface/public/shared/registry.js +5 -3
- package/neural-interface/public/shared/retired-2d.js +3 -0
- package/neural-interface/public/shared/schedules-view.js +195 -61
- package/neural-interface/public/shared/settings/settings-claude-hooks.js +135 -0
- package/neural-interface/public/shared/settings/settings-data.js +3 -1
- package/neural-interface/public/shared/settings/settings-ia.js +6 -6
- package/neural-interface/public/shared/settings/settings-opencode.js +89 -0
- package/neural-interface/public/shared/settings/settings-skill-folders-pane.js +265 -0
- package/neural-interface/public/shared/settings/settings-skill-folders.js +130 -0
- package/neural-interface/public/shared/settings/settings-worker-guard.js +72 -0
- package/neural-interface/public/shared/skills/ss-args.js +96 -0
- package/neural-interface/public/shared/skills/ss-creation.js +219 -0
- package/neural-interface/public/shared/skills/ss-dialog.js +130 -0
- package/neural-interface/public/shared/skills/ss-doc.js +173 -0
- package/neural-interface/public/shared/skills/ss-eval-adapters.js +269 -0
- package/neural-interface/public/shared/skills/ss-eval-grade.js +359 -0
- package/neural-interface/public/shared/skills/ss-eval-optimise.js +153 -0
- package/neural-interface/public/shared/skills/ss-eval-plan.js +315 -0
- package/neural-interface/public/shared/skills/ss-evals.js +417 -0
- package/neural-interface/public/shared/skills/ss-form.js +159 -0
- package/neural-interface/public/shared/skills/ss-frontmatter.js +376 -0
- package/neural-interface/public/shared/skills/ss-ids.js +32 -0
- package/neural-interface/public/shared/skills/ss-lint.js +141 -0
- package/neural-interface/public/shared/skills/ss-presets.js +117 -0
- package/neural-interface/public/shared/skills/ss-profiles.js +475 -0
- package/neural-interface/public/shared/skills/ss-resources.js +165 -0
- package/neural-interface/public/shared/skills/ss-review-view.js +190 -0
- package/neural-interface/public/shared/skills/ss-tabs.js +79 -0
- package/neural-interface/public/shared/skills/ss-unicode.js +135 -0
- package/neural-interface/public/shared/skills/studio/sx-actions.js +222 -0
- package/neural-interface/public/shared/skills/studio/sx-cm.js +101 -0
- package/neural-interface/public/shared/skills/studio/sx-create-ai-view.js +749 -0
- package/neural-interface/public/shared/skills/studio/sx-create-ai.js +236 -0
- package/neural-interface/public/shared/skills/studio/sx-create-view.js +254 -0
- package/neural-interface/public/shared/skills/studio/sx-create.js +126 -0
- package/neural-interface/public/shared/skills/studio/sx-diff-view.js +43 -0
- package/neural-interface/public/shared/skills/studio/sx-diff.js +133 -0
- package/neural-interface/public/shared/skills/studio/sx-docs.js +480 -0
- package/neural-interface/public/shared/skills/studio/sx-dom.js +229 -0
- package/neural-interface/public/shared/skills/studio/sx-edit-view.js +848 -0
- package/neural-interface/public/shared/skills/studio/sx-editor-model.js +193 -0
- package/neural-interface/public/shared/skills/studio/sx-errors.js +22 -0
- package/neural-interface/public/shared/skills/studio/sx-geometry.js +82 -0
- package/neural-interface/public/shared/skills/studio/sx-import.js +262 -0
- package/neural-interface/public/shared/skills/studio/sx-inspector.js +308 -0
- package/neural-interface/public/shared/skills/studio/sx-install-flow.js +279 -0
- package/neural-interface/public/shared/skills/studio/sx-install.js +296 -0
- package/neural-interface/public/shared/skills/studio/sx-library-view.js +639 -0
- package/neural-interface/public/shared/skills/studio/sx-library.js +475 -0
- package/neural-interface/public/shared/skills/studio/sx-main.js +681 -0
- package/neural-interface/public/shared/skills/studio/sx-operations-view.js +98 -0
- package/neural-interface/public/shared/skills/studio/sx-operations.js +11 -0
- package/neural-interface/public/shared/skills/studio/sx-plan.js +158 -0
- package/neural-interface/public/shared/skills/studio/sx-publish-view.js +621 -0
- package/neural-interface/public/shared/skills/studio/sx-review-view.js +369 -0
- package/neural-interface/public/shared/skills/studio/sx-review.js +148 -0
- package/neural-interface/public/shared/skills/studio/sx-scan.js +35 -0
- package/neural-interface/public/shared/skills/studio/sx-snippets.js +96 -0
- package/neural-interface/public/shared/skills/studio/sx-subject.js +248 -0
- package/neural-interface/public/shared/skills/studio/sx-test-view.js +994 -0
- package/neural-interface/public/shared/skills/studio/sx-test.js +328 -0
- package/neural-interface/public/shared/skills/studio/sx-versions-view.js +241 -0
- package/neural-interface/public/shared/skills/studio/sx-workspace-view.js +425 -0
- package/neural-interface/public/shared/styleguide/sg-panel.js +4 -2
- package/neural-interface/public/shared/styles.css +1525 -1779
- package/neural-interface/public/shared/synabun-mascot.js +1115 -9
- package/neural-interface/public/shared/tool-list-picker.js +55 -0
- package/neural-interface/public/shared/tool-list-storage.js +24 -0
- package/neural-interface/public/shared/tool-list-view.js +112 -0
- package/neural-interface/public/shared/tour/tour-actor.js +268 -0
- package/neural-interface/public/shared/tour/tour-beats.js +336 -0
- package/neural-interface/public/shared/tour/tour-engine.js +884 -0
- package/neural-interface/public/shared/tour/tour-ink.js +390 -0
- package/neural-interface/public/shared/tour/tour-stage.js +440 -0
- package/neural-interface/public/shared/tour/tour-state.js +40 -0
- package/neural-interface/public/shared/tour/tour-tips.js +99 -0
- package/neural-interface/public/shared/tour/tour.css +144 -0
- package/neural-interface/public/shared/ui-automation-studio.js +91 -128
- package/neural-interface/public/shared/ui-claude-panel.js +77 -5
- package/neural-interface/public/shared/ui-command-runner.js +6 -3
- package/neural-interface/public/shared/ui-context-menu.js +295 -0
- package/neural-interface/public/shared/ui-cost-widget.js +57 -59
- package/neural-interface/public/shared/ui-crash-notice.js +187 -0
- package/neural-interface/public/shared/ui-detail.js +15 -3
- package/neural-interface/public/shared/ui-image-gallery.js +50 -47
- package/neural-interface/public/shared/ui-keybinds.js +26 -1
- package/neural-interface/public/shared/ui-menubar.js +8 -18
- package/neural-interface/public/shared/ui-native-loop-router-state.js +5 -0
- package/neural-interface/public/shared/ui-native-loop-router.js +7 -1
- package/neural-interface/public/shared/ui-navbar.js +4 -24
- package/neural-interface/public/shared/ui-panels.js +96 -65
- package/neural-interface/public/shared/ui-schedules-studio.js +507 -423
- package/neural-interface/public/shared/ui-sessions.js +5 -3
- package/neural-interface/public/shared/ui-settings.js +594 -462
- package/neural-interface/public/shared/ui-sidepanel-sessions.js +1 -1
- package/neural-interface/public/shared/ui-sidepanel-state.js +23 -12
- package/neural-interface/public/shared/ui-sidepanel-windows.js +16 -11
- package/neural-interface/public/shared/ui-skills.js +33 -2830
- package/neural-interface/public/shared/ui-styleguide.js +5 -2
- package/neural-interface/public/shared/ui-sync.js +15 -5
- package/neural-interface/public/shared/ui-terminal.js +124 -180
- package/neural-interface/public/shared/ui-tool-updates.js +3 -1
- package/neural-interface/public/shared/ui-tooltip.js +10 -5
- package/neural-interface/public/shared/ui-trash.js +6 -4
- package/neural-interface/public/shared/ui-tutorial-layout.js +3 -117
- package/neural-interface/public/shared/ui-tutorial-preview.js +103 -0
- package/neural-interface/public/shared/ui-tutorial-steps.js +3 -65
- package/neural-interface/public/shared/ui-tutorial.css +47 -100
- package/neural-interface/public/shared/ui-tutorial.js +285 -509
- package/neural-interface/public/shared/ui-update.js +38 -2
- package/neural-interface/public/shared/ui-whiteboard.js +311 -58
- package/neural-interface/public/shared/ui-worker-end.js +4 -0
- package/neural-interface/public/shared/ui-workspaces.js +3 -2
- package/neural-interface/public/shared/view-variant.js +66 -0
- package/neural-interface/public/shared/whatsapp/wa-view.js +1 -1
- package/neural-interface/public/shared/whiteboard-list.js +172 -0
- package/neural-interface/public/shared/worker-end.js +10 -3
- package/neural-interface/public/shared/worker-restart.js +12 -0
- package/neural-interface/public/sidepanel-session.html +1 -1
- package/neural-interface/public/variant/2d/background.js +2 -1
- package/neural-interface/public/variant/2d/context-menu.js +8 -2
- package/neural-interface/public/variant/2d/graph.js +42 -20
- package/neural-interface/public/variant/2d/hulls.js +14 -9
- package/neural-interface/public/variant/2d/lasso.js +11 -7
- package/neural-interface/public/variant/2d/main.js +32 -1
- package/neural-interface/public/variant/2d/minimap.js +11 -13
- package/neural-interface/public/variant/3d/main.js +69 -6
- package/neural-interface/public/variant/3d/map-controls.js +24 -9
- package/neural-interface/public/vendor/caveat/OFL.txt +93 -0
- package/neural-interface/public/vendor/caveat/caveat-latin-400-normal.woff2 +0 -0
- package/neural-interface/public/vendor/caveat/caveat-latin-700-normal.woff2 +0 -0
- package/neural-interface/public/vendor/caveat/caveat.css +17 -0
- package/neural-interface/public/vendor/caveat/manifest.json +12 -0
- package/neural-interface/public/vendor/yaml/LICENSE +13 -0
- package/neural-interface/public/vendor/yaml/manifest.json +4 -0
- package/neural-interface/public/vendor/yaml/yaml-2.9.1.min.js +137 -0
- package/neural-interface/scripts/build-vendor.mjs +32 -0
- package/neural-interface/server.js +3934 -2454
- package/neural-interface/templates/rulesets/core.md +1 -1
- package/neural-interface/templates/rulesets/legacy-hashes.json +18 -0
- package/neural-interface/templates/rulesets/manifest.json +8 -8
- package/npm-shrinkwrap.json +18 -12
- package/package.json +16 -3
- package/postinstall.js +9 -4
- package/preuninstall.js +1 -1
- package/setup.js +97 -322
- package/skills/skill-forge/SKILL.md +177 -0
- package/skills/skill-forge/evals/evals.json +54 -0
- package/skills/skill-forge/references/package.md +105 -0
- package/skills/skill-forge/references/review-and-recovery.md +63 -0
- package/skills/skill-forge/references/runtime-profiles.md +20 -0
- package/skills/skill-forge/references/sources-and-capabilities.md +51 -0
- package/skills/synabun/SKILL.md +10 -7
- package/skills/synabun/modules/audit.md +1 -5
- package/skills/synabun/modules/changelog.md +1 -5
- package/skills/synabun/modules/idea.md +1 -1
- package/skills/synabun/modules/memorize.md +6 -9
- package/skills/synabun/modules/schedule.md +4 -4
- package/mcp-server/dist/services/profiles.d.ts +0 -120
- package/mcp-server/dist/services/profiles.d.ts.map +0 -1
- package/mcp-server/dist/services/profiles.js +0 -387
- package/mcp-server/dist/services/profiles.js.map +0 -1
- package/mcp-server/dist/tools/leonardo-browser-tools.d.ts +0 -97
- package/mcp-server/dist/tools/leonardo-browser-tools.d.ts.map +0 -1
- package/mcp-server/dist/tools/leonardo-browser-tools.js +0 -228
- package/mcp-server/dist/tools/leonardo-browser-tools.js.map +0 -1
- package/mcp-server/dist/tools/leonardo.d.ts +0 -13
- package/mcp-server/dist/tools/leonardo.d.ts.map +0 -1
- package/mcp-server/dist/tools/leonardo.js +0 -21
- package/mcp-server/dist/tools/leonardo.js.map +0 -1
- package/mcp-server/dist/tools/profile.d.ts +0 -17
- package/mcp-server/dist/tools/profile.d.ts.map +0 -1
- package/mcp-server/dist/tools/profile.js +0 -114
- package/mcp-server/dist/tools/profile.js.map +0 -1
- package/mcp-server/src/services/profiles.ts +0 -428
- package/mcp-server/src/tools/leonardo-browser-tools.ts +0 -291
- package/mcp-server/src/tools/leonardo.ts +0 -30
- package/mcp-server/src/tools/profile.ts +0 -122
- package/neural-interface/lib/assistant-brains/opencode-browser-policy.js +0 -47
- package/neural-interface/lib/assistant-brains/opencode-route-gate.js +0 -62
- package/neural-interface/lib/mcp-profile-servers.js +0 -120
- package/neural-interface/public/shared/ui-tutorial-draw.js +0 -311
- package/skills/leonardo/SKILL.md +0 -168
- package/skills/leonardo/icon.svg +0 -1
- package/skills/leonardo/modules/browser-guide.md +0 -378
- package/skills/leonardo/modules/image-prompter.md +0 -283
- package/skills/leonardo/modules/model-advisor.md +0 -100
- package/skills/leonardo/modules/prompt-library.md +0 -110
- package/skills/leonardo/modules/style-guide.md +0 -186
- package/skills/leonardo/modules/video-prompter.md +0 -317
|
@@ -0,0 +1,994 @@
|
|
|
1
|
+
// ═══════════════════════════════════════════
|
|
2
|
+
// SynaBun Neural Interface — Skills Studio 2.0: the Test tab (P4)
|
|
3
|
+
// ═══════════════════════════════════════════
|
|
4
|
+
//
|
|
5
|
+
// A library skill's evaluations (lib/skills/evals.js over /api/skills-studio/v2/evals): one obvious action, Run evals
|
|
6
|
+
// (⌘/Ctrl+Enter), with defaults that are cheap (the train and validation cases once, on the first runtime that can, its
|
|
7
|
+
// cheapest model, against no skill, within $3 and 30 minutes) and an estimate that follows the form; the run's progress
|
|
8
|
+
// with its spend against its ceiling and Cancel; then the report: per runtime the task cases with and without the skill
|
|
9
|
+
// (candidate minus baseline, paired), the trigger queries (positives and near-misses), the notes about the suite; each
|
|
10
|
+
// trial opened beside its request (the answer, the files it wrote, the assertions collapsed with their evidence, the
|
|
11
|
+
// previous run's answer, the owner's note, saved as it is typed and acknowledged); the history of the skill's runs,
|
|
12
|
+
// two compared, a run exported. Also: the final check on the held-out cases, the description optimiser, and the import
|
|
13
|
+
// of another tool's test prompts into the draft. Every answer is fenced by the window's generation and the tab's own
|
|
14
|
+
// signal, and redraws only its part (never a field being typed in). Strings: skillsStudio.test.*.
|
|
15
|
+
|
|
16
|
+
import { dropdownHTML, wireDropdowns } from '../../cc-dropdown.js';
|
|
17
|
+
import { html, raw, setHTML, ICONS, relativeTime, fullTime, formatBytes, idempotencyKey } from './sx-dom.js';
|
|
18
|
+
import { changedFiles } from './sx-diff.js';
|
|
19
|
+
import { EVALS_PATH } from '../ss-evals.js';
|
|
20
|
+
import { importEvals } from '../ss-eval-adapters.js';
|
|
21
|
+
import { executedRuntime } from '../ss-eval-grade.js';
|
|
22
|
+
import {
|
|
23
|
+
BASELINE_CHOICES, BUDGET_CHOICES, EXPORT_FORMATS, MINUTE_CHOICES, REPETITION_CHOICES, TEST_FORM_KEY, TEST_KIND_CHOICES,
|
|
24
|
+
executionLabel as labelExecution, clockText, codexStatus, defaultForm, elapsedSeconds, hasSuite, historyRows, isActiveEvalRun, isOptInRuntime, money, pct, previousTrial, pts,
|
|
25
|
+
readSuiteText, reportModel, requestOf, rerunCases, runtimeEntries, safetyLines, storedForm, suiteAllows, suiteErrorCopy, testErrorText, testSubject, trialGroups, trialStep, trialTone,
|
|
26
|
+
} from './sx-test.js';
|
|
27
|
+
|
|
28
|
+
let _life = null;
|
|
29
|
+
const live = (panel, signal) => !signal.aborted && panel.isConnected;
|
|
30
|
+
|
|
31
|
+
/** The tab is left (another tab, another place, the window closed): its listeners, timers and polls end. */
|
|
32
|
+
export function leaveTestTab() { _life?.abort(); _life = null; }
|
|
33
|
+
|
|
34
|
+
function stateOf(ws) {
|
|
35
|
+
if (!ws.test) {
|
|
36
|
+
ws.test = {
|
|
37
|
+
options: null, optionsError: null, catalog: null, suite: null, suiteError: null, suiteFor: null,
|
|
38
|
+
history: null, historyError: null, form: null, estimate: null, estimateError: null, estimating: false, estimateSeq: 0,
|
|
39
|
+
openId: null, run: null, runFull: false, trialId: null, trial: null, trialError: null, previous: null,
|
|
40
|
+
compare: new Set(), comparison: null, starting: false, notice: null, feedbackState: {}, exportFormat: 'native',
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
return ws.test;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function readStored(ctx) { try { return JSON.parse(ctx.storage.getItem(TEST_FORM_KEY) || 'null'); } catch { return null; } }
|
|
47
|
+
function keepForm(ctx, form) { try { ctx.storage.setItem(TEST_FORM_KEY, JSON.stringify(storedForm(form))); } catch { /* the form is still used */ } }
|
|
48
|
+
|
|
49
|
+
// ── Words ──
|
|
50
|
+
|
|
51
|
+
const providerName = (t, p) => t(`skillsStudio.test.provider.${p}`);
|
|
52
|
+
function runtimeWords(ctx) {
|
|
53
|
+
const { t } = ctx;
|
|
54
|
+
return {
|
|
55
|
+
provider: (p) => providerName(t, p),
|
|
56
|
+
providerDefault: (p, model) => (model ? t('skillsStudio.test.setup.providerDefaultModel', { provider: providerName(t, p), model }) : t('skillsStudio.test.setup.providerDefault', { provider: providerName(t, p) })),
|
|
57
|
+
providerOff: (p, reason) => t('skillsStudio.test.setup.providerOff', { provider: providerName(t, p), reason }),
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
const runtimeLabel = (t, r) => `${providerName(t, r.provider)}${r.model ? ` · ${r.model}` : ''}`;
|
|
61
|
+
const executionLabel = (t, actual, requested) => labelExecution(t, actual, requested, r => runtimeLabel(t, r));
|
|
62
|
+
|
|
63
|
+
function stateWord(t, state) { return t(`skillsStudio.test.state.${state}`); }
|
|
64
|
+
function minutesText(t, est) { return est.minutes.low === est.minutes.high ? t('skillsStudio.test.estimate.minutesOne', { n: est.minutes.low }) : t('skillsStudio.test.estimate.minutes', { low: est.minutes.low, high: est.minutes.high }); }
|
|
65
|
+
|
|
66
|
+
// ── Drawing ──
|
|
67
|
+
|
|
68
|
+
/** Draw the Test tab. */
|
|
69
|
+
export function renderTestTab(panel, ctx, ws) {
|
|
70
|
+
_life?.abort();
|
|
71
|
+
_life = new AbortController();
|
|
72
|
+
const signal = _life.signal;
|
|
73
|
+
const { t } = ctx;
|
|
74
|
+
const st = stateOf(ws);
|
|
75
|
+
const subject = testSubject(ws, { caps: ctx.caps, capsKnown: ctx.capsKnown, guest: ctx.guest, changedFiles });
|
|
76
|
+
if (subject.mode !== 'ok') { drawUnavailable(panel, ctx, ws, subject, signal); return; }
|
|
77
|
+
if (!hasSuite(ws.revision)) { drawNoSuite(panel, ctx, ws, signal); return; }
|
|
78
|
+
st.revisionId = subject.revisionId;
|
|
79
|
+
st.draftChanged = subject.draftChanged;
|
|
80
|
+
if (!st.form) st.form = defaultForm(st.options, readStored(ctx));
|
|
81
|
+
setHTML(panel, html`<div class="sx-test" data-test-state="${st.run?.state || 'idle'}">
|
|
82
|
+
<div class="sx-test-scroll" id="sx-test-scroll">
|
|
83
|
+
<section class="sx-test-section" id="sx-test-head" aria-labelledby="sx-test-title">${headHTML(ctx, ws, st)}</section>
|
|
84
|
+
<section class="sx-test-section sx-test-setup" id="sx-test-setup" aria-label="${t('skillsStudio.test.setup.title')}">${setupHTML(ctx, ws, st)}</section>
|
|
85
|
+
<section class="sx-test-section" id="sx-test-run" aria-live="polite">${runHTML(ctx, ws, st)}</section>
|
|
86
|
+
<section class="sx-test-section" id="sx-test-trial">${trialHTML(ctx, ws, st)}</section>
|
|
87
|
+
<section class="sx-test-section" id="sx-test-history" aria-labelledby="sx-test-history-title">${historyHTML(ctx, ws, st)}</section>
|
|
88
|
+
</div>
|
|
89
|
+
<div class="sx-test-bar" id="sx-test-bar">${barHTML(ctx, ws, st)}</div>
|
|
90
|
+
</div>`);
|
|
91
|
+
wireDropdowns(panel, { signal });
|
|
92
|
+
wire(panel, ctx, ws, signal);
|
|
93
|
+
load(panel, ctx, ws, signal);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function drawUnavailable(panel, ctx, ws, subject, signal) {
|
|
97
|
+
const { t } = ctx;
|
|
98
|
+
const words = {
|
|
99
|
+
'older-server': [t('skillsStudio.test.unavailable.olderTitle'), t('skillsStudio.test.unavailable.older')],
|
|
100
|
+
'not-library': [t('skillsStudio.test.unavailable.notLibraryTitle'), t('skillsStudio.test.unavailable.notLibrary')],
|
|
101
|
+
'no-revision': [t('skillsStudio.test.unavailable.noRevisionTitle'), t('skillsStudio.test.unavailable.noRevision')],
|
|
102
|
+
guest: [t('skillsStudio.test.unavailable.guestTitle'), t('skillsStudio.test.unavailable.guest')],
|
|
103
|
+
}[subject.mode];
|
|
104
|
+
setHTML(panel, html`<div class="sx-empty" role="status"><span class="sx-placeholder-icon" aria-hidden="true">${ICONS.test}</span>
|
|
105
|
+
<h4 class="sx-empty-title">${words[0]}</h4><p>${words[1]}</p>
|
|
106
|
+
${subject.mode === 'not-library' && subject.canAdopt && !ctx.guest ? html`<button type="button" class="sx-btn sx-btn--primary" data-test="adopt">${t('skillsStudio.ws.adoptNow')}</button>` : ''}
|
|
107
|
+
${subject.mode === 'no-revision' && !ctx.guest ? html`<button type="button" class="sx-btn sx-btn--primary" data-test="go-versions">${t('skillsStudio.test.unavailable.saveVersion')}</button>` : ''}</div>`);
|
|
108
|
+
panel.addEventListener('click', async (e) => {
|
|
109
|
+
const btn = e.target.closest?.('[data-test]');
|
|
110
|
+
if (!btn) return;
|
|
111
|
+
if (btn.dataset.test === 'adopt') { const { adoptNow } = await import('./sx-actions.js'); adoptNow(ctx, ws); }
|
|
112
|
+
if (btn.dataset.test === 'go-versions') { ws.tab = 'versions'; ctx.render(); }
|
|
113
|
+
}, { signal });
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function drawNoSuite(panel, ctx, ws, signal) {
|
|
117
|
+
const { t } = ctx;
|
|
118
|
+
const st = stateOf(ws);
|
|
119
|
+
setHTML(panel, html`<div class="sx-test"><div class="sx-test-scroll"><div class="sx-empty sx-test-empty" role="status">
|
|
120
|
+
<span class="sx-placeholder-icon" aria-hidden="true">${ICONS.test}</span>
|
|
121
|
+
<h4 class="sx-empty-title">${t('skillsStudio.test.noSuite.title')}</h4>
|
|
122
|
+
<p>${t('skillsStudio.test.noSuite.body', { path: EVALS_PATH })}</p>
|
|
123
|
+
<div class="sx-row sx-test-empty-actions">
|
|
124
|
+
${ctx.guest ? '' : html`<button type="button" class="sx-btn sx-btn--primary" data-test="import">${ICONS.upload}<span>${t('skillsStudio.test.import.button')}</span></button>`}
|
|
125
|
+
<button type="button" class="sx-btn" data-test="go-edit">${t('skillsStudio.test.noSuite.edit')}</button>
|
|
126
|
+
</div>
|
|
127
|
+
<p class="sx-help">${t('skillsStudio.test.import.help')}</p>
|
|
128
|
+
<div id="sx-test-import" class="sx-test-import">${st.importPreview ? importHTML(ctx, st) : ''}</div>
|
|
129
|
+
</div></div></div>`);
|
|
130
|
+
panel.addEventListener('click', (e) => {
|
|
131
|
+
const btn = e.target.closest?.('[data-test]');
|
|
132
|
+
if (!btn || btn.disabled) return;
|
|
133
|
+
if (btn.dataset.test === 'go-edit') { ws.tab = 'edit'; ctx.render(); return; }
|
|
134
|
+
if (btn.dataset.test === 'import') pickImport(panel, ctx, ws, signal);
|
|
135
|
+
if (btn.dataset.test === 'import-save') saveImport(panel, ctx, ws, signal);
|
|
136
|
+
if (btn.dataset.test === 'import-drop') { st.importPreview = null; const box = panel.querySelector('#sx-test-import'); if (box) setHTML(box, ''); }
|
|
137
|
+
}, { signal });
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function headHTML(ctx, ws, st) {
|
|
141
|
+
const { t, tp } = ctx;
|
|
142
|
+
const counts = st.suite?.counts || null;
|
|
143
|
+
const rev = ws.revision;
|
|
144
|
+
const lines = [];
|
|
145
|
+
if (st.draftChanged) lines.push(html`<p class="sx-note sx-note--warn" role="note">${ICONS.warn} ${t('skillsStudio.test.head.draftChanged')} <button type="button" class="sx-link" data-test="save-version">${t('skillsStudio.ws.saveVersion')}</button></p>`);
|
|
146
|
+
if (st.suite && !st.suite.ok) {
|
|
147
|
+
lines.push(html`<div class="sx-note sx-note--warn" role="alert"><p>${tp('skillsStudio.test.head.suiteErrors', st.suite.errors.length)}</p>
|
|
148
|
+
<ul class="sx-plain-list">${st.suite.errors.slice(0, 3).map((e) => html`<li><code>${e.path}</code> ${suiteErrorText(ctx, e)}</li>`)}</ul>
|
|
149
|
+
<button type="button" class="sx-btn sx-btn--sm" data-test="go-edit">${t('skillsStudio.test.head.fixInEdit')}</button></div>`);
|
|
150
|
+
}
|
|
151
|
+
if (st.suiteError) lines.push(html`<p class="sx-note sx-note--warn" role="alert">${t('skillsStudio.test.head.suiteUnread', { error: st.suiteError })}</p>`);
|
|
152
|
+
const q = st.options?.queue;
|
|
153
|
+
const waiting = q ? q.waiting.filter((w) => w.skill?.skillId !== ws.skill?.skillId || w.evalRunId !== st.openId) : [];
|
|
154
|
+
return html`<div class="sx-test-headline">
|
|
155
|
+
<h4 class="sx-section-title" id="sx-test-title">${ICONS.test}<span>${t('skillsStudio.test.title')}</span></h4>
|
|
156
|
+
<span class="sx-tag" title="${rev?.revisionId || ''}">${t('skillsStudio.test.head.version', { id: String(rev?.contentDigest || '').replace(/^sha256:/, '').slice(0, 8) })}</span>
|
|
157
|
+
<span class="sx-spacer"></span>
|
|
158
|
+
${q && (q.active.length || q.waiting.length) ? html`<span class="sx-test-queue" id="sx-test-queue">${tp('skillsStudio.test.queue.busy', q.active.length + waiting.length)}</span>` : ''}
|
|
159
|
+
</div>
|
|
160
|
+
<p class="sx-help" id="sx-test-suite-line">${counts
|
|
161
|
+
? html`${tp('skillsStudio.test.head.tasks', counts.task)} · ${tp('skillsStudio.test.head.triggers', counts.trigger)}${counts.trigger ? html` (${t('skillsStudio.test.head.polarity', { positive: counts.positive, nearMiss: counts.nearMiss })})` : ''} · ${tp('skillsStudio.test.head.held', counts.held)}`
|
|
162
|
+
: t('skillsStudio.test.head.reading')}</p>
|
|
163
|
+
<div class="sx-test-safety-box" id="sx-test-safety">${safetyHTML(ctx, st)}</div>
|
|
164
|
+
${fixturesHTML(ctx, st)}
|
|
165
|
+
${lines}`;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* What happens to a case's test files, said once under the safety lines: the files refused (a runtime would load them
|
|
170
|
+
* by itself), what each runtime does with an instruction file deeper in the folder, git and the no-symlink shell rule.
|
|
171
|
+
*/
|
|
172
|
+
function fixturesHTML(ctx, st) {
|
|
173
|
+
const { t } = ctx;
|
|
174
|
+
return html`<details class="sx-details sx-test-fixtures" id="sx-test-fixtures"${st.fixturesOpen ? ' open' : ''}><summary>${t('skillsStudio.test.fixtures.title')}</summary>
|
|
175
|
+
<p class="sx-help">${t('skillsStudio.test.fixtures.refused')}</p>
|
|
176
|
+
<p class="sx-help">${t('skillsStudio.test.fixtures.nested')}</p>
|
|
177
|
+
<p class="sx-help">${t('skillsStudio.test.fixtures.git')}</p>
|
|
178
|
+
<p class="sx-help">${t('skillsStudio.test.fixtures.symlinks')}</p>
|
|
179
|
+
<p class="sx-help">${t('skillsStudio.test.fixtures.symlinkWorkaround')}</p></details>`;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** A problem of the suite as the page words it (a test file a runtime would load is said in the page's language). */
|
|
183
|
+
function suiteErrorText(ctx, e) {
|
|
184
|
+
const copy = suiteErrorCopy(e);
|
|
185
|
+
return copy ? ctx.t(copy.key, copy.params) : e.message;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* What keeps a run safe, said truthfully per runtime the form uses: Claude Code and OpenCode load only the skill under
|
|
190
|
+
* test and work in the run's folder (a shell, where a case asks, with no network and no home folder); Codex's shell can
|
|
191
|
+
* read anywhere, so it runs only versions the owner confirms.
|
|
192
|
+
*/
|
|
193
|
+
function safetyHTML(ctx, st) {
|
|
194
|
+
const { t } = ctx;
|
|
195
|
+
const rows = st.form ? safetyLines(st.form, st.options) : [];
|
|
196
|
+
return html`<p class="sx-help sx-test-safety">${ICONS.lock} <span>${t('skillsStudio.test.head.safety')}</span></p>
|
|
197
|
+
${rows.map((r) => html`<p class="sx-help sx-test-safety${r.code === 'codex' ? ' sx-test-safety--warn' : ''}" data-provider="${r.provider}">${r.code === 'codex' ? ICONS.warn : ICONS.lock} <span>${r.code === 'codex'
|
|
198
|
+
? t('skillsStudio.test.head.safetyCodex')
|
|
199
|
+
: html`${t('skillsStudio.test.head.safetyHeld', { provider: providerName(t, r.provider) })}${r.shell ? html` ${t('skillsStudio.test.head.safetyShell')}` : ''}`}</span></p>`)}`;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** Codex's state for the form: the estimate's, else the last one seen while the next estimate is on its way. */
|
|
203
|
+
const codexNow = (st) => codexStatus(st.form, st.estimate || (st.codexSeen ? { codex: st.codexSeen } : null), { revisionId: st.revisionId });
|
|
204
|
+
|
|
205
|
+
/** The Codex line under the setup while the form uses Codex: the warning, and the one-click confirmation of the version. */
|
|
206
|
+
function codexHTML(ctx, st) {
|
|
207
|
+
const { t } = ctx;
|
|
208
|
+
const c = codexNow(st);
|
|
209
|
+
if (!c) return '';
|
|
210
|
+
return html`<div class="sx-note sx-note--warn sx-test-codex" role="note" id="sx-test-codex" aria-live="polite">
|
|
211
|
+
<p class="sx-test-codex-line">${ICONS.warn}<strong>${t('skillsStudio.test.codex.warning')}</strong></p>
|
|
212
|
+
${c.confirmed
|
|
213
|
+
? html`<p class="sx-help">${c.at ? t('skillsStudio.test.codex.confirmedAt', { when: relativeTime(c.at, ctx.locale) }) : t('skillsStudio.test.codex.confirmed')} <button type="button" class="sx-link" data-test="codex-undo"${st.codexBusy ? ' disabled' : ''}>${t('skillsStudio.test.codex.undo')}</button></p>`
|
|
214
|
+
: html`<p class="sx-help">${t(c.revisionIds.length > 1 ? 'skillsStudio.test.codex.askBoth' : 'skillsStudio.test.codex.ask')}</p>
|
|
215
|
+
<div class="sx-row"><button type="button" class="sx-btn sx-btn--sm" data-test="codex-confirm"${st.codexBusy || !st.estimate ? ' disabled' : ''}>${ICONS.ok}<span>${t('skillsStudio.test.codex.confirm')}</span></button></div>`}
|
|
216
|
+
</div>`;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
function setupHTML(ctx, ws, st) {
|
|
220
|
+
const { t } = ctx;
|
|
221
|
+
const f = st.form;
|
|
222
|
+
const words = runtimeWords(ctx);
|
|
223
|
+
const allows = suiteAllows(st.suite?.cases || []);
|
|
224
|
+
const dd = (id, entries, label) => raw(dropdownHTML(id, entries, { label, classes: 'sx-dd' }));
|
|
225
|
+
const runtimeRows = f.runtimes.map((value, i) => html`<div class="sx-test-runtime" data-index="${i}">
|
|
226
|
+
${dd(`sx-test-runtime-${i}`, runtimeEntries({ options: st.options, catalog: st.catalog, selected: value, words }), t('skillsStudio.test.setup.runtime'))}
|
|
227
|
+
${i > 0 ? html`<button type="button" class="sx-icon-btn sx-icon-btn--sm" data-test="remove-runtime" data-index="${i}" aria-label="${t('skillsStudio.test.setup.removeRuntime')}" title="${t('skillsStudio.test.setup.removeRuntime')}">${ICONS.close}</button>` : ''}
|
|
228
|
+
</div>`);
|
|
229
|
+
// The skill's other versions, newest first (a baseline can be one of them).
|
|
230
|
+
const revisions = (Array.isArray(st.revisions) ? st.revisions : []).filter((r) => r.revisionId !== st.revisionId);
|
|
231
|
+
const kindsDisabled = (k) => (k === 'task' && !allows.tasks) || (k === 'trigger' && !allows.triggers);
|
|
232
|
+
// Open while nothing has run yet (or when the owner opened it); folded to one line over a report, so the results are
|
|
233
|
+
// in view and Run evals (the foot bar) still runs these settings.
|
|
234
|
+
const open = st.setupOpen ?? !(st.run || (st.history && st.history.length));
|
|
235
|
+
const firstRuntime = f.runtimes[0] ? runtimeOfValue(f.runtimes[0]) : null;
|
|
236
|
+
const summary = [t(`skillsStudio.test.setup.kinds.${f.kinds}`), ...(firstRuntime ? [runtimeLabel(t, { provider: firstRuntime.provider, model: firstRuntime.model || (st.options?.runtimes || []).find((r) => r.provider === firstRuntime.provider)?.defaultModel || null })] : []), ...(f.runtimes.length > 1 ? [`+${f.runtimes.length - 1}`] : []), t(`skillsStudio.test.setup.baselines.${f.baseline}`), t('skillsStudio.test.setup.budgetN', { n: f.budgetUsd }), t('skillsStudio.test.setup.minutesN', { n: f.maxMinutes })].join(' · ');
|
|
237
|
+
return html`<details class="sx-test-setup-box" id="sx-test-setup-box"${open ? ' open' : ''}><summary class="sx-test-setup-summary"><span class="sx-section-title">${t('skillsStudio.test.setup.title')}</span> <span class="sx-dim sx-test-setup-line">${summary}</span></summary>
|
|
238
|
+
<div class="sx-test-grid">
|
|
239
|
+
<div class="sx-field"><span class="sx-field-label" id="sx-test-kinds-label">${t('skillsStudio.test.setup.what')}</span>
|
|
240
|
+
<div class="sx-seg" role="group" aria-labelledby="sx-test-kinds-label">${TEST_KIND_CHOICES.map((k) => html`<button type="button" data-test="kinds" data-value="${k}" aria-pressed="${f.kinds === k ? 'true' : 'false'}"${k !== 'both' && kindsDisabled(k) ? ' disabled' : ''}>${t(`skillsStudio.test.setup.kinds.${k}`)}</button>`)}</div></div>
|
|
241
|
+
<div class="sx-field sx-test-runtimes"><span class="sx-field-label">${t('skillsStudio.test.setup.runtime')}</span>${runtimeRows}
|
|
242
|
+
${f.runtimes.length < 3 && (st.options?.runtimes || []).filter((r) => r.available).length ? html`<button type="button" class="sx-link sx-test-add" data-test="add-runtime">${ICONS.plus}<span>${t('skillsStudio.test.setup.addRuntime')}</span></button>` : ''}</div>
|
|
243
|
+
<div class="sx-field"><span class="sx-field-label" id="sx-test-baseline-label">${t('skillsStudio.test.setup.baseline')}</span>
|
|
244
|
+
${dd('sx-test-baseline', BASELINE_CHOICES.map((b) => ({ value: b, label: t(`skillsStudio.test.setup.baselines.${b}`), selected: f.baseline === b, disabled: b === 'revision' && !revisions.length })), t('skillsStudio.test.setup.baseline'))}
|
|
245
|
+
${f.baseline === 'revision' && revisions.length ? dd('sx-test-baseline-rev', revisions.map((r, i) => ({ value: r.revisionId, label: t('skillsStudio.test.setup.versionN', { n: `${String(r.contentDigest || '').replace(/^sha256:/, '').slice(0, 8)} · ${relativeTime(r.createdAt, ctx.locale)}` }), selected: f.baselineRevisionId ? f.baselineRevisionId === r.revisionId : i === 0 })), t('skillsStudio.test.setup.baselineVersion')) : ''}</div>
|
|
246
|
+
<div class="sx-field"><span class="sx-field-label">${t('skillsStudio.test.setup.ceilings')}</span>
|
|
247
|
+
<div class="sx-row">${dd('sx-test-budget', BUDGET_CHOICES.map((v) => ({ value: String(v), label: t('skillsStudio.test.setup.budgetN', { n: v }), selected: f.budgetUsd === v })), t('skillsStudio.test.setup.budget'))}
|
|
248
|
+
${dd('sx-test-minutes', MINUTE_CHOICES.map((v) => ({ value: String(v), label: t('skillsStudio.test.setup.minutesN', { n: v }), selected: f.maxMinutes === v })), t('skillsStudio.test.setup.minutes'))}</div></div>
|
|
249
|
+
</div>
|
|
250
|
+
<details class="sx-details sx-test-more"${st.moreOpen ? ' open' : ''}><summary data-test="more">${t('skillsStudio.test.setup.more')}</summary>
|
|
251
|
+
<div class="sx-test-grid">
|
|
252
|
+
<div class="sx-field"><span class="sx-field-label">${t('skillsStudio.test.setup.taskReps')}</span>${dd('sx-test-task-reps', REPETITION_CHOICES.map((v) => ({ value: String(v), label: String(v), selected: f.taskReps === v })), t('skillsStudio.test.setup.taskReps'))}</div>
|
|
253
|
+
<div class="sx-field"><span class="sx-field-label">${t('skillsStudio.test.setup.triggerReps')}</span>${dd('sx-test-trigger-reps', REPETITION_CHOICES.map((v) => ({ value: String(v), label: String(v), selected: f.triggerReps === v })), t('skillsStudio.test.setup.triggerReps'))}</div>
|
|
254
|
+
<div class="sx-field"><span class="sx-field-label">${t('skillsStudio.test.setup.grader')}</span>${dd('sx-test-grader', runtimeEntries({ options: st.options, catalog: st.catalog, selected: f.grader, words, allowNone: true, noneLabel: t('skillsStudio.test.setup.graderAuto') }), t('skillsStudio.test.setup.grader'))}</div>
|
|
255
|
+
<label class="sx-choice sx-test-judge"><input type="checkbox" class="sx-check" id="sx-test-judge"${f.judge ? ' checked' : ''}> <span>${t('skillsStudio.test.setup.judge')}</span></label>
|
|
256
|
+
</div>
|
|
257
|
+
<p class="sx-help">${t('skillsStudio.test.setup.moreHelp')}</p>
|
|
258
|
+
</details>
|
|
259
|
+
</details>
|
|
260
|
+
<div id="sx-test-codex-box">${codexHTML(ctx, st)}</div>
|
|
261
|
+
<div class="sx-test-estimate" id="sx-test-estimate" role="status">${estimateHTML(ctx, st)}</div>`;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/** A runtime value of the form as { provider, model }. */
|
|
265
|
+
function runtimeOfValue(value) { const [provider, ...rest] = String(value || '').split('|'); return { provider, model: rest.join('|') || null }; }
|
|
266
|
+
|
|
267
|
+
function estimateHTML(ctx, st) {
|
|
268
|
+
const { t, tp } = ctx;
|
|
269
|
+
if (st.estimateError) return html`<p class="sx-help sx-err-text">${st.estimateError}</p>`;
|
|
270
|
+
const e = st.estimate;
|
|
271
|
+
if (!e) return html`<p class="sx-help">${st.estimating ? t('skillsStudio.test.estimate.working') : t('skillsStudio.test.estimate.none')}</p>`;
|
|
272
|
+
const est = e.estimate;
|
|
273
|
+
const money_ = est.free ? t('skillsStudio.test.estimate.free') : est.usd.high > 0 ? t('skillsStudio.test.estimate.usd', { low: money(est.usd.low), high: money(est.usd.high) }) : t('skillsStudio.test.estimate.unpricedOnly');
|
|
274
|
+
const notes = (e.notes || []).map((n) => {
|
|
275
|
+
if (n.code === 'contamination') return t('skillsStudio.test.estimate.contamination', { provider: providerName(t, n.provider), paths: (n.paths || []).join(', ') });
|
|
276
|
+
if (n.code === 'grader-not-independent') return t('skillsStudio.test.estimate.notIndependent', { provider: providerName(t, n.provider) });
|
|
277
|
+
if (n.code === 'skipped') return t(`skillsStudio.test.estimate.skipped.${n.reason === 'needs-shell' ? 'shell' : 'other'}`, { caseId: n.caseId });
|
|
278
|
+
return null;
|
|
279
|
+
}).filter(Boolean);
|
|
280
|
+
return html`<p class="sx-test-estimate-line">${ICONS.info}<span>${tp('skillsStudio.test.estimate.runs', est.runs)} · ${money_}${est.mayExceedByOneStep && !est.free ? html` <span class="sx-dim">(${t('skillsStudio.test.estimate.oneStep')})</span>` : ''} · ${minutesText(t, est)}${est.unpriced.length ? html` · <span class="sx-dim">${t('skillsStudio.test.estimate.unpriced', { names: est.unpriced.join(', ') })}</span>` : ''}</span></p>
|
|
281
|
+
${e.ceiling ? html`<p class="sx-help sx-warn-text">${t(`skillsStudio.test.estimate.ceiling.${e.ceiling.code === 'time-below-estimate' ? 'time' : 'budget'}`)}</p>` : ''}
|
|
282
|
+
${e.grader ? html`<p class="sx-help">${t('skillsStudio.test.estimate.grader', { grader: runtimeLabel(t, e.grader) })}${e.grader.independent ? '' : html` <span class="sx-warn-text">${t('skillsStudio.test.estimate.graderSame')}</span>`}</p>` : ''}
|
|
283
|
+
${notes.length ? html`<ul class="sx-plain-list sx-test-notes">${notes.map((n) => html`<li>${n}</li>`)}</ul>` : ''}`;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
function barHTML(ctx, ws, st) {
|
|
287
|
+
const { t } = ctx;
|
|
288
|
+
if (ctx.guest) return html`<span class="sx-help">${t('skillsStudio.test.bar.guest')}</span>`;
|
|
289
|
+
const allows = suiteAllows(st.suite?.cases || []);
|
|
290
|
+
const running = st.run && isActiveEvalRun(st.run.state);
|
|
291
|
+
const blocked = !st.suite || !st.suite.ok || !allows.run;
|
|
292
|
+
const noRuntime = st.options && !(st.options.runtimes || []).some((r) => r.available);
|
|
293
|
+
const ceilingBlocked = !!st.estimate?.ceiling;
|
|
294
|
+
// Codex runs a version only once the owner allowed it (the line under the setup).
|
|
295
|
+
const codex = codexNow(st);
|
|
296
|
+
const codexBlocked = !!codex && !codex.confirmed;
|
|
297
|
+
const why = !st.suite ? t('skillsStudio.test.head.reading') : !st.suite.ok ? t('skillsStudio.test.bar.fixFirst') : !allows.run ? t('skillsStudio.test.bar.onlyHeld') : noRuntime ? t('skillsStudio.test.bar.noRuntime') : codexBlocked ? t('skillsStudio.test.bar.codexFirst') : ceilingBlocked ? t('skillsStudio.test.bar.raiseCeiling') : running ? t('skillsStudio.test.bar.running') : t('skillsStudio.test.bar.ready');
|
|
298
|
+
const off = blocked || noRuntime || st.starting || ceilingBlocked || codexBlocked;
|
|
299
|
+
return html`<span class="sx-test-bar-text sx-help" id="sx-test-bar-text">${why}</span>
|
|
300
|
+
${allows.optimise ? html`<button type="button" class="sx-btn" data-test="optimise"${off || running ? ' disabled' : ''} title="${t('skillsStudio.test.bar.optimiseHelp')}">${ICONS.spark}<span>${t('skillsStudio.test.bar.optimise')}</span></button>` : ''}
|
|
301
|
+
${allows.final ? html`<button type="button" class="sx-btn" data-test="final"${off || running ? ' disabled' : ''} title="${t('skillsStudio.test.bar.finalHelp')}">${ICONS.lock}<span>${t('skillsStudio.test.bar.final')}</span></button>` : ''}
|
|
302
|
+
<button type="button" class="sx-btn sx-btn--primary sx-btn--lg" id="sx-test-run-btn" data-test="run"${off ? ' disabled' : ''} aria-describedby="sx-test-bar-text" aria-keyshortcuts="${ctx.mac ? 'Meta+Enter' : 'Control+Enter'}">${ICONS.test}<span>${st.starting ? t('skillsStudio.test.bar.starting') : running ? t('skillsStudio.test.bar.runAgain') : t('skillsStudio.test.bar.run')}</span> <kbd>${ctx.mac ? '⌘↵' : 'Ctrl+↵'}</kbd></button>`;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
// ── The open run ──
|
|
306
|
+
|
|
307
|
+
function runHTML(ctx, ws, st) {
|
|
308
|
+
const { t } = ctx;
|
|
309
|
+
const run = st.run;
|
|
310
|
+
if (st.notice) return html`<div class="sx-outcome sx-outcome--error" role="alert"><p class="sx-outcome-line">${ICONS.warn}<span>${st.notice}</span></p></div>`;
|
|
311
|
+
if (!run) return html`<p class="sx-help">${st.history && !st.history.length ? t('skillsStudio.test.run.none') : ''}</p>`;
|
|
312
|
+
const active = isActiveEvalRun(run.state);
|
|
313
|
+
const purpose = run.request?.purpose || 'evals';
|
|
314
|
+
const spent = run.spend?.usd ?? 0;
|
|
315
|
+
const ceiling = run.request?.budgetUsd ?? 0;
|
|
316
|
+
const ratio = ceiling > 0 ? Math.min(1, spent / ceiling) : 0;
|
|
317
|
+
const p = run.progress || { trials: 0, done: 0 };
|
|
318
|
+
return html`<div class="sx-test-run" data-state="${run.state}" id="sx-test-run-card">
|
|
319
|
+
<div class="sx-test-run-head">
|
|
320
|
+
<h4 class="sx-section-title" id="sx-test-run-title" tabindex="-1">${t(`skillsStudio.test.run.title.${purpose}`)}</h4>
|
|
321
|
+
<span class="sx-test-state sx-test-state--${run.state}">${stateWord(t, run.state)}</span>
|
|
322
|
+
<span class="sx-spacer"></span>
|
|
323
|
+
${active ? html`<span class="sx-count" id="sx-test-clock">${clockText(elapsedSeconds(run))}</span>
|
|
324
|
+
<button type="button" class="sx-btn sx-btn--sm sx-btn--danger" data-test="cancel"${st.cancelling ? ' disabled' : ''}>${st.cancelling ? t('skillsStudio.test.run.cancelling') : t('skillsStudio.test.run.cancel')}</button>` : ''}
|
|
325
|
+
${!active && run.evalRunId ? exportHTML(ctx, st) : ''}
|
|
326
|
+
</div>
|
|
327
|
+
<p class="sx-help">${run.queuePosition ? ctx.tp('skillsStudio.test.run.queued', run.queuePosition) : t('skillsStudio.test.run.facts', { runtimes: (run.summary?.runtimes?.length ? run.summary.runtimes : run.request?.runtimes || []).map((r) => executionLabel(t, r, r.requestedRuntimes)).join(', '), when: relativeTime(run.createdAt, ctx.locale) })}
|
|
328
|
+
· <span class="sx-test-spend${ratio >= 0.9 ? ' is-over' : ''}" title="${t('skillsStudio.test.run.spendTitle')}">${t('skillsStudio.test.run.spend', { spent: money(spent), ceiling: money(ceiling) })} <span class="sx-meter" aria-hidden="true"><i style="width:${Math.round(ratio * 100)}%"></i></span></span>
|
|
329
|
+
${run.estimate ? html` · <span class="sx-dim">${t('skillsStudio.test.run.estimated', { low: money(run.estimate.usd.low), high: money(run.estimate.usd.high) })}</span>` : ''}</p>
|
|
330
|
+
${active ? progressHTML(ctx, st, p) : reportHTML(ctx, ws, st)}
|
|
331
|
+
${run.error ? html`<div class="sx-ai-error" role="alert"><p>${testErrorText(t, run.error)}</p></div>` : ''}
|
|
332
|
+
${run.stopReason ? html`<p class="sx-note sx-note--warn">${t(`skillsStudio.test.run.stopped.${run.stopReason === 'time' ? 'time' : 'budget'}`)}</p>` : ''}
|
|
333
|
+
</div>`;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
function progressHTML(ctx, st, p) {
|
|
337
|
+
const { t, tp } = ctx;
|
|
338
|
+
const groups = trialGroups(st.run);
|
|
339
|
+
return html`<p class="sx-test-progress-line">${tp('skillsStudio.test.run.progress', p.trials, { done: p.done })}</p>
|
|
340
|
+
<ul class="sx-test-live" aria-label="${t('skillsStudio.test.run.trials')}">${groups.map((g) => html`<li class="sx-test-live-case">
|
|
341
|
+
<span class="sx-test-case-id">${g.caseId}</span><span class="sx-tag sx-tag--${g.split}">${t(`skillsStudio.test.split.${g.split}`)}</span>
|
|
342
|
+
<span class="sx-test-dots">${g.trials.map((tr) => html`<span class="sx-test-dot sx-test-dot--${trialTone(tr)}" role="img" aria-label="${t(`skillsStudio.test.tone.${trialTone(tr)}`)} · ${t(`skillsStudio.test.config.${tr.configuration}`)}" title="${t(`skillsStudio.test.config.${tr.configuration}`)} · ${t(`skillsStudio.test.tone.${trialTone(tr)}`)}"></span>`)}</span>
|
|
343
|
+
</li>`)}</ul>`;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
function exportHTML(ctx, st) {
|
|
347
|
+
const { t } = ctx;
|
|
348
|
+
return html`<span class="sx-test-export">${raw(dropdownHTML('sx-test-export-format', EXPORT_FORMATS.map((f) => ({ value: f, label: t(`skillsStudio.test.export.${f}`), selected: st.exportFormat === f })), { label: t('skillsStudio.test.export.format'), classes: 'sx-dd' }))}
|
|
349
|
+
<button type="button" class="sx-btn sx-btn--sm" data-test="export">${ICONS.download}<span>${t('skillsStudio.test.export.button')}</span></button></span>`;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
function reportHTML(ctx, ws, st) {
|
|
353
|
+
const { t, tp } = ctx;
|
|
354
|
+
const run = st.run;
|
|
355
|
+
if (!st.runFull) return html`<p class="sx-help">${t('skillsStudio.test.report.loading')}</p>`;
|
|
356
|
+
const model = reportModel(run, { latestRevisionId: ws.skill?.pointers?.latest || null, latestSuiteDigest: st.suiteDigest || null });
|
|
357
|
+
const parts = [];
|
|
358
|
+
if (model.stale.length) parts.push(html`<p class="sx-note sx-note--warn" role="note">${ICONS.history} ${t(`skillsStudio.test.report.stale.${model.stale.includes('revision') ? 'revision' : 'suite'}`)}</p>`);
|
|
359
|
+
if (run.request?.purpose === 'optimise') parts.push(optimiseHTML(ctx, ws, st));
|
|
360
|
+
for (const r of model.runtimes) {
|
|
361
|
+
const head = [];
|
|
362
|
+
if (r.task) head.push(html`<div class="sx-test-stat"><span class="sx-test-stat-label">${t('skillsStudio.test.report.task')}</span>
|
|
363
|
+
<span class="sx-test-stat-value">${pct(r.task.candidate)}</span><span class="sx-dim">${t('skillsStudio.test.report.withSkill')}</span>
|
|
364
|
+
${model.baseline !== 'off' ? html`<span class="sx-test-stat-vs">${pct(r.task.baseline)} <span class="sx-dim">${t(model.baseline === 'revision' ? 'skillsStudio.test.report.withEarlier' : 'skillsStudio.test.report.withoutSkill')}</span></span>
|
|
365
|
+
<span class="sx-test-delta sx-test-delta--${r.task.delta === null ? 'none' : r.task.delta > 0.0001 ? 'up' : r.task.delta < -0.0001 ? 'down' : 'flat'}" title="${t('skillsStudio.test.report.deltaTitle')}">${pts(r.task.delta)} ${t('skillsStudio.test.report.points')}</span>
|
|
366
|
+
<span class="sx-dim">${tp('skillsStudio.test.report.pairs', r.task.pairs)}</span>` : ''}</div>`);
|
|
367
|
+
if (r.trigger) head.push(html`<div class="sx-test-stat"><span class="sx-test-stat-label">${t('skillsStudio.test.report.trigger')}</span>
|
|
368
|
+
<span class="sx-test-stat-value">${r.trigger.passed}/${r.trigger.queries}</span><span class="sx-dim">${t('skillsStudio.test.report.queriesRight')}</span>
|
|
369
|
+
<span class="sx-dim">${t('skillsStudio.test.report.polarity', { pos: `${r.trigger.positives.passed}/${r.trigger.positives.total}`, neg: `${r.trigger.nearMisses.passed}/${r.trigger.nearMisses.total}` })}</span>
|
|
370
|
+
${r.trigger.unread ? html`<span class="sx-warn-text">${tp('skillsStudio.test.report.unread', r.trigger.unread)}</span>` : ''}</div>`);
|
|
371
|
+
if (r.preference) head.push(html`<div class="sx-test-stat"><span class="sx-test-stat-label">${t('skillsStudio.test.report.preference')}</span><span class="sx-test-stat-value">${r.preference.passed}/${r.preference.total}</span></div>`);
|
|
372
|
+
parts.push(html`<article class="sx-test-runtime-report" aria-label="${executionLabel(t, r, r.requestedRuntimes)}">
|
|
373
|
+
<h5 class="sx-test-runtime-name">${executionLabel(t, r, r.requestedRuntimes)} <span class="sx-dim">· ${money(r.costUsd)}${r.durationMs ? ` · ${Math.round(r.durationMs / 1000)} s` : ''}${r.failedRuns ? ` · ${tp('skillsStudio.test.report.failedRuns', r.failedRuns)}` : ''}</span></h5>
|
|
374
|
+
<div class="sx-test-stats">${head}</div>
|
|
375
|
+
${r.task ? taskTableHTML(ctx, st, r, model) : ''}
|
|
376
|
+
${r.trigger ? triggerTableHTML(ctx, st, r) : ''}
|
|
377
|
+
</article>`);
|
|
378
|
+
}
|
|
379
|
+
if (model.notes.length) {
|
|
380
|
+
parts.push(html`<details class="sx-details sx-test-suite-notes"><summary>${tp('skillsStudio.test.report.notes', model.notes.length)}</summary>
|
|
381
|
+
<ul class="sx-plain-list">${model.notes.map((n) => html`<li>${t(`skillsStudio.test.note.${n.code}`, { caseId: n.caseId, assertionId: n.assertionId || '' })}</li>`)}</ul></details>`);
|
|
382
|
+
}
|
|
383
|
+
const again = rerunCases(run);
|
|
384
|
+
if (again.length && !ctx.guest && run.request?.purpose !== 'optimise') parts.push(html`<p class="sx-row"><button type="button" class="sx-btn sx-btn--sm" data-test="rerun">${ICONS.update}<span>${ctx.tp('skillsStudio.test.report.rerun', again.length)}</span></button></p>`);
|
|
385
|
+
if (!model.runtimes.length && run.request?.purpose !== 'optimise') parts.push(html`<p class="sx-help">${t('skillsStudio.test.report.empty')}</p>`);
|
|
386
|
+
return html`${parts}`;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
const runKey = (rt) => `${rt.provider}${rt.model ? `/${rt.model}` : ''}`;
|
|
390
|
+
|
|
391
|
+
function taskTableHTML(ctx, st, r, model) {
|
|
392
|
+
const { t } = ctx;
|
|
393
|
+
const trials = st.run.trials || [];
|
|
394
|
+
return html`<table class="sx-test-table"><caption class="sx-visually-hidden">${t('skillsStudio.test.report.taskCases')}</caption>
|
|
395
|
+
<thead><tr><th scope="col">${t('skillsStudio.test.report.case')}</th><th scope="col">${t('skillsStudio.test.report.colWith')}</th>${model.baseline !== 'off' ? html`<th scope="col">${t(model.baseline === 'revision' ? 'skillsStudio.test.report.colEarlier' : 'skillsStudio.test.report.colWithout')}</th><th scope="col">${t('skillsStudio.test.report.change')}</th>` : ''}<th scope="col"><span class="sx-visually-hidden">${t('skillsStudio.test.report.open')}</span></th></tr></thead>
|
|
396
|
+
<tbody>${r.task.cases.map((c) => {
|
|
397
|
+
// The report groups by the runtime that ran (executedRuntime), so a trial is found by the same key.
|
|
398
|
+
const first = trials.find((x) => x.caseId === c.caseId && x.configuration === 'candidate' && runKey(executedRuntime(x)) === r.key);
|
|
399
|
+
const verdict = c.comparison ? (c.comparison.candidate > c.comparison.baseline ? 'candidate' : c.comparison.baseline > c.comparison.candidate ? 'baseline' : c.comparison.tie ? 'tie' : 'inconclusive') : null;
|
|
400
|
+
return html`<tr>
|
|
401
|
+
<th scope="row"><span class="sx-test-case-id">${c.caseId}</span> <span class="sx-tag sx-tag--${c.split}">${t(`skillsStudio.test.split.${c.split}`)}</span>${verdict ? html` <span class="sx-tag" title="${t('skillsStudio.test.report.judgedTitle')}">${t(`skillsStudio.test.judged.${verdict}`)}</span>` : ''}</th>
|
|
402
|
+
<td>${pct(c.candidateRate)}</td>
|
|
403
|
+
${model.baseline !== 'off' ? html`<td>${pct(c.baselineRate)}</td><td><span class="sx-test-delta sx-test-delta--${c.outcome === 'better' ? 'up' : c.outcome === 'worse' ? 'down' : c.outcome === 'same' ? 'flat' : 'none'}">${c.outcome === 'n/a' ? '—' : `${pts(c.delta)} · ${t(`skillsStudio.test.outcome.${c.outcome}`)}`}</span></td>` : ''}
|
|
404
|
+
<td>${first ? html`<button type="button" class="sx-btn sx-btn--sm sx-btn--ghost" data-test="open-trial" data-trial="${first.trialId}">${t('skillsStudio.test.report.open')}</button>` : ''}</td>
|
|
405
|
+
</tr>`;
|
|
406
|
+
})}</tbody></table>`;
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
function triggerTableHTML(ctx, st, r) {
|
|
410
|
+
const { t } = ctx;
|
|
411
|
+
const trials = st.run.trials || [];
|
|
412
|
+
return html`<table class="sx-test-table"><caption class="sx-visually-hidden">${t('skillsStudio.test.report.triggerCases')}</caption>
|
|
413
|
+
<thead><tr><th scope="col">${t('skillsStudio.test.report.query')}</th><th scope="col">${t('skillsStudio.test.report.expected')}</th><th scope="col">${t('skillsStudio.test.report.reading')}</th><th scope="col">${t('skillsStudio.test.report.result')}</th><th scope="col"><span class="sx-visually-hidden">${t('skillsStudio.test.report.open')}</span></th></tr></thead>
|
|
414
|
+
<tbody>${r.trigger.cases.map((c) => {
|
|
415
|
+
const first = trials.find((x) => x.caseId === c.caseId && x.kind === 'trigger' && runKey(executedRuntime(x)) === r.key);
|
|
416
|
+
return html`<tr>
|
|
417
|
+
<th scope="row"><span class="sx-test-case-id">${c.caseId}</span> <span class="sx-tag sx-tag--${c.split}">${t(`skillsStudio.test.split.${c.split}`)}</span></th>
|
|
418
|
+
<td>${t(c.expected ? 'skillsStudio.test.report.shouldPick' : 'skillsStudio.test.report.nearMiss')}</td>
|
|
419
|
+
<td>${c.rate === null ? html`<span class="sx-warn-text">${t('skillsStudio.test.report.noReading')}</span>` : html`${t('skillsStudio.test.report.picked', { n: c.triggers, of: c.completed })}${c.method ? html` <span class="sx-dim" title="${t(`skillsStudio.test.method.${c.method}Help`)}">· ${t(`skillsStudio.test.method.${c.method}`)}</span>` : ''}`}</td>
|
|
420
|
+
<td>${c.passed === null ? '—' : html`<span class="sx-test-pass sx-test-pass--${c.passed ? 'yes' : 'no'}">${c.passed ? ICONS.ok : ICONS.err} ${t(c.passed ? 'skillsStudio.test.report.right' : 'skillsStudio.test.report.wrong')}</span>`}</td>
|
|
421
|
+
<td>${first ? html`<button type="button" class="sx-btn sx-btn--sm sx-btn--ghost" data-test="open-trial" data-trial="${first.trialId}">${t('skillsStudio.test.report.open')}</button>` : ''}</td>
|
|
422
|
+
</tr>`;
|
|
423
|
+
})}</tbody></table>`;
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
function optimiseHTML(ctx, ws, st) {
|
|
427
|
+
const { t } = ctx;
|
|
428
|
+
const o = st.run.optimise;
|
|
429
|
+
if (!o) return html`<p class="sx-help">${t('skillsStudio.test.optimise.none')}</p>`;
|
|
430
|
+
const score = (s) => (s && s.total ? `${s.passed}/${s.total}` : '—');
|
|
431
|
+
return html`<div class="sx-test-optimise">
|
|
432
|
+
<p class="sx-help">${t('skillsStudio.test.optimise.explain')}</p>
|
|
433
|
+
<table class="sx-test-table"><caption class="sx-visually-hidden">${t('skillsStudio.test.optimise.title')}</caption>
|
|
434
|
+
<thead><tr><th scope="col">#</th><th scope="col">${t('skillsStudio.test.optimise.description')}</th><th scope="col">${t('skillsStudio.test.optimise.train')}</th><th scope="col">${t('skillsStudio.test.optimise.validation')}</th></tr></thead>
|
|
435
|
+
<tbody>${(o.iterations || []).map((it) => html`<tr class="${o.best?.n === it.n ? 'is-best' : ''}"><th scope="row">${it.n}${o.best?.n === it.n ? html` <span class="sx-label sx-label--best">${t('skillsStudio.test.optimise.best')}</span>` : ''}</th><td class="sx-test-desc">${it.description}</td><td>${score(it.train)}</td><td>${score(it.validation)}</td></tr>`)}</tbody>
|
|
436
|
+
</table>
|
|
437
|
+
<p class="sx-help">${o.exitReason ? t(`skillsStudio.test.optimise.exit.${['train_pass', 'max_evaluations', 'budget', 'time', 'cancelled', 'proposal_refused', 'proposer_failed', 'unpatchable'].includes(o.exitReason) ? o.exitReason : 'other'}`) : ''}
|
|
438
|
+
${o.test ? html` · ${t('skillsStudio.test.optimise.heldOut', { score: score(o.test) })}` : ''}</p>
|
|
439
|
+
${o.proposal ? html`<div class="sx-outcome" role="status"><p class="sx-outcome-line">${ICONS.spark}<span>${t('skillsStudio.test.optimise.proposed')}</span></p>
|
|
440
|
+
<div class="sx-row"><button type="button" class="sx-btn sx-btn--sm sx-btn--primary" data-test="go-versions">${t('skillsStudio.test.optimise.openVersions')}</button></div></div>`
|
|
441
|
+
: o.best && !o.best.original ? '' : html`<p class="sx-help">${t('skillsStudio.test.optimise.keepOriginal')}</p>`}
|
|
442
|
+
</div>`;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
// ── A trial beside its request ──
|
|
446
|
+
|
|
447
|
+
function trialHTML(ctx, ws, st) {
|
|
448
|
+
const { t } = ctx;
|
|
449
|
+
if (!st.trialId) return '';
|
|
450
|
+
if (st.trialError) return html`<div class="sx-ai-error" role="alert"><p>${st.trialError}</p></div>`;
|
|
451
|
+
const tr = st.trial;
|
|
452
|
+
if (!tr) return html`<p class="sx-help">${t('skillsStudio.test.trial.loading')}</p>`;
|
|
453
|
+
const c = tr.case || {};
|
|
454
|
+
const fb = st.feedbackState[tr.trialId] || { text: tr.feedback?.text || '', reviewed: !!tr.feedback?.reviewed, saved: tr.feedback?.at || null, state: 'idle' };
|
|
455
|
+
st.feedbackState[tr.trialId] = fb;
|
|
456
|
+
return html`<div class="sx-test-trial" id="sx-test-trial-box" tabindex="-1" aria-labelledby="sx-test-trial-title">
|
|
457
|
+
<div class="sx-test-run-head">
|
|
458
|
+
<h4 class="sx-section-title" id="sx-test-trial-title">${t('skillsStudio.test.trial.title', { caseId: tr.caseId })}</h4>
|
|
459
|
+
<span class="sx-tag">${t(`skillsStudio.test.config.${tr.configuration}`)}</span><span class="sx-tag">${executionLabel(t, tr.actualRuntime, tr.runtime)}${tr.repetition > 1 ? ` · #${tr.repetition}` : ''}</span>
|
|
460
|
+
<span class="sx-spacer"></span>
|
|
461
|
+
<button type="button" class="sx-icon-btn sx-icon-btn--sm" data-test="trial-prev" aria-label="${t('skillsStudio.test.trial.prev')}" title="${t('skillsStudio.test.trial.prev')}" aria-keyshortcuts="ArrowLeft">${ICONS.back}</button>
|
|
462
|
+
<button type="button" class="sx-icon-btn sx-icon-btn--sm sx-test-next" data-test="trial-next" aria-label="${t('skillsStudio.test.trial.next')}" title="${t('skillsStudio.test.trial.next')}" aria-keyshortcuts="ArrowRight">${ICONS.back}</button>
|
|
463
|
+
<button type="button" class="sx-icon-btn sx-icon-btn--sm" data-test="trial-close" aria-label="${t('skillsStudio.test.trial.close')}" title="${t('skillsStudio.test.trial.close')}">${ICONS.close}</button>
|
|
464
|
+
</div>
|
|
465
|
+
${tr.detailMissing ? html`<p class="sx-note sx-note--warn" role="note">${ICONS.warn} ${t('skillsStudio.test.trial.detailMissing')}</p>` : ''}
|
|
466
|
+
<p class="sx-help">${t(`skillsStudio.test.trialState.${tr.status}`)}${tr.costUsd !== null ? ` · ${money(tr.costUsd)}` : ''}${tr.durationMs ? ` · ${Math.round(tr.durationMs / 1000)} s` : ''}${tr.model ? ` · ${tr.model}` : ''}${tr.error ? html` · <span class="sx-err-text">${tr.error}</span>` : ''}${tr.contaminationRisk ? html` · <span class="sx-warn-text">${t('skillsStudio.test.trial.contamination')}</span>` : ''}</p>
|
|
467
|
+
<div class="sx-test-side">
|
|
468
|
+
<div class="sx-test-pane"><h5 class="sx-test-pane-title">${t(c.kind === 'trigger' ? 'skillsStudio.test.trial.query' : 'skillsStudio.test.trial.request')}</h5>
|
|
469
|
+
<pre class="sx-pre">${c.prompt || c.query || ''}</pre>
|
|
470
|
+
${c.expected ? html`<h5 class="sx-test-pane-title">${t('skillsStudio.test.trial.expected')}</h5><pre class="sx-pre sx-pre--sm">${c.expected}</pre>` : ''}
|
|
471
|
+
${c.kind === 'trigger' ? html`<p class="sx-help">${t(c.shouldTrigger ? 'skillsStudio.test.report.shouldPick' : 'skillsStudio.test.report.nearMiss')}</p>` : ''}</div>
|
|
472
|
+
<div class="sx-test-pane"><h5 class="sx-test-pane-title">${t('skillsStudio.test.trial.answer')}</h5>
|
|
473
|
+
<pre class="sx-pre">${tr.answer || t('skillsStudio.test.trial.noAnswer')}</pre>
|
|
474
|
+
${tr.answerTruncated ? html`<p class="sx-help">${t('skillsStudio.test.trial.truncated')}</p>` : ''}
|
|
475
|
+
${(tr.outputs || []).length ? html`<h5 class="sx-test-pane-title">${t('skillsStudio.test.trial.files')}</h5>
|
|
476
|
+
<ul class="sx-plain-list sx-test-files">${tr.outputs.map((o) => html`<li>${o.kind === 'link' ? html`<span class="sx-warn-text">${o.path} · ${t('skillsStudio.test.trial.link')}</span>` : html`<button type="button" class="sx-link" data-test="trial-file" data-path="${o.path}">${ICONS.file}<span>${o.path}</span></button> <span class="sx-dim">${formatBytes(o.bytes, ctx.locale)}</span>${o.kind === 'config' ? html` <span class="sx-warn-text">· ${t('skillsStudio.test.trial.config')}</span>` : ''}`}</li>`)}</ul>
|
|
477
|
+
<div id="sx-test-file-view"></div>` : ''}</div>
|
|
478
|
+
</div>
|
|
479
|
+
${tr.trigger ? html`<p class="sx-help">${tr.trigger.observed === true ? t('skillsStudio.test.trial.picked', { method: t(`skillsStudio.test.method.${tr.trigger.method || 'native_skill_tool'}`) }) : tr.trigger.observed === false ? t('skillsStudio.test.trial.notPicked') : t('skillsStudio.test.report.noReading')}${(tr.trigger.others || []).length ? ` · ${t('skillsStudio.test.trial.others', { names: tr.trigger.others.join(', ') })}` : ''}</p>` : ''}
|
|
480
|
+
${tr.grade ? html`<details class="sx-details sx-test-grade"><summary>${t('skillsStudio.test.trial.grades', { passed: tr.grade.counts.passed, total: tr.grade.graded })}${tr.grade.grader ? html` · <span class="sx-dim">${t('skillsStudio.test.trial.gradedBy', { grader: executionLabel(t, tr.grade.grader, tr.grade.grader.requestedRuntime) })}${tr.grade.grader.independent === false ? ` (${t('skillsStudio.test.estimate.graderSame')})` : ''}</span>` : ''}</summary>
|
|
481
|
+
<ul class="sx-test-assertions">${tr.grade.assertions.map((a) => html`<li class="sx-test-assertion sx-test-assertion--${a.status}">
|
|
482
|
+
<span class="sx-test-pass sx-test-pass--${a.status === 'passed' ? 'yes' : a.status === 'failed' ? 'no' : 'maybe'}">${a.status === 'passed' ? ICONS.ok : a.status === 'failed' ? ICONS.err : ICONS.warn} ${t(`skillsStudio.test.assertion.${a.status}`)}</span>
|
|
483
|
+
<span class="sx-test-assertion-text">${a.text}</span> <span class="sx-dim">· ${t(a.by === 'check' ? 'skillsStudio.test.assertion.byCheck' : 'skillsStudio.test.assertion.byModel')}${a.kind === 'preference' ? ` · ${t('skillsStudio.test.assertion.preference')}` : ''}</span>
|
|
484
|
+
${a.evidence ? html`<p class="sx-test-evidence">${a.evidence}</p>` : ''}</li>`)}</ul></details>` : ''}
|
|
485
|
+
${st.previous ? html`<details class="sx-details sx-test-previous"><summary>${t('skillsStudio.test.trial.previous')}</summary><pre class="sx-pre sx-pre--sm">${st.previous.answer || t('skillsStudio.test.trial.noAnswer')}</pre>
|
|
486
|
+
${st.previous.feedback?.text ? html`<p class="sx-help">${t('skillsStudio.test.trial.previousNote', { note: st.previous.feedback.text })}</p>` : ''}</details>` : ''}
|
|
487
|
+
${ctx.guest ? '' : html`<div class="sx-field sx-test-feedback"><label class="sx-field-label" for="sx-test-note"><span>${t('skillsStudio.test.trial.note')}</span><span class="sx-count" id="sx-test-note-state" role="status">${feedbackWord(ctx, fb)}</span></label>
|
|
488
|
+
<textarea class="sx-input sx-textarea" id="sx-test-note" rows="3" maxlength="4000" placeholder="${t('skillsStudio.test.trial.notePlaceholder')}">${fb.text}</textarea>
|
|
489
|
+
<label class="sx-choice"><input type="checkbox" class="sx-check" id="sx-test-reviewed"${fb.reviewed ? ' checked' : ''}> <span>${t('skillsStudio.test.trial.reviewed')}</span></label></div>`}
|
|
490
|
+
</div>`;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
function feedbackWord(ctx, fb) {
|
|
494
|
+
const { t } = ctx;
|
|
495
|
+
if (fb.state === 'saving') return t('skillsStudio.test.trial.saving');
|
|
496
|
+
if (fb.state === 'error') return t('skillsStudio.test.trial.saveFailed');
|
|
497
|
+
if (fb.state === 'dirty') return t('skillsStudio.test.trial.unsaved');
|
|
498
|
+
return fb.saved ? t('skillsStudio.test.trial.saved', { when: relativeTime(fb.saved, ctx.locale) }) : '';
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
// ── History ──
|
|
502
|
+
|
|
503
|
+
function historyHTML(ctx, ws, st) {
|
|
504
|
+
const { t, tp } = ctx;
|
|
505
|
+
if (st.historyError) return html`<p class="sx-help sx-err-text">${t('skillsStudio.test.history.failed', { error: st.historyError })}</p>`;
|
|
506
|
+
if (!st.history) return html`<p class="sx-help">${t('skillsStudio.test.history.loading')}</p>`;
|
|
507
|
+
if (!st.history.length) return html`<h4 class="sx-section-title" id="sx-test-history-title">${t('skillsStudio.test.history.title')}</h4><p class="sx-help">${t('skillsStudio.test.history.empty')}</p>`;
|
|
508
|
+
const rows = historyRows(st.history, { latestRevisionId: ws.skill?.pointers?.latest || null, latestSuiteDigest: st.suiteDigest || null });
|
|
509
|
+
return html`<div class="sx-test-run-head"><h4 class="sx-section-title" id="sx-test-history-title">${tp('skillsStudio.test.history.count', rows.length)}</h4><span class="sx-spacer"></span>
|
|
510
|
+
<button type="button" class="sx-btn sx-btn--sm" data-test="compare"${st.compare.size === 2 ? '' : ' disabled'}>${t('skillsStudio.test.history.compare')}</button></div>
|
|
511
|
+
<p class="sx-help">${t('skillsStudio.test.history.help')}</p>
|
|
512
|
+
<ul class="sx-test-history">${rows.map((r) => html`<li class="sx-test-history-row${st.openId === r.evalRunId ? ' is-on' : ''}">
|
|
513
|
+
<input type="checkbox" class="sx-check" data-test="pick" data-id="${r.evalRunId}" aria-label="${t('skillsStudio.test.history.pick')}"${st.compare.has(r.evalRunId) ? ' checked' : ''}>
|
|
514
|
+
<button type="button" class="sx-test-history-open" data-test="open-run" data-id="${r.evalRunId}">
|
|
515
|
+
<span class="sx-test-state sx-test-state--${r.state}">${stateWord(t, r.state)}</span>
|
|
516
|
+
<span class="sx-test-history-what">${t(`skillsStudio.test.run.title.${r.purpose}`)} · ${r.runtimeDetails.map(r => executionLabel(t, r, r.requestedRuntimes)).join(', ')}</span>
|
|
517
|
+
<span class="sx-test-history-numbers">${r.task ? html`${pct(r.task.candidate)}${r.task.delta !== null ? html` (${pts(r.task.delta)})` : ''}` : ''}${r.task && r.trigger ? ' · ' : ''}${r.trigger ? `${r.trigger.passed}/${r.trigger.total}` : ''}</span>
|
|
518
|
+
${r.stale.length ? html`<span class="sx-tag" title="${t('skillsStudio.test.history.staleTitle')}">${t('skillsStudio.test.history.stale')}</span>` : ''}
|
|
519
|
+
<span class="sx-dim">${money(r.costUsd)}</span>
|
|
520
|
+
<time class="sx-dim" datetime="${r.createdAt}" title="${fullTime(r.createdAt, ctx.locale)}">${relativeTime(r.createdAt, ctx.locale)}</time>
|
|
521
|
+
</button></li>`)}</ul>
|
|
522
|
+
<div id="sx-test-compare">${st.comparison ? comparisonHTML(ctx, st) : ''}</div>`;
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
function comparisonHTML(ctx, st) {
|
|
526
|
+
const { t } = ctx;
|
|
527
|
+
const c = st.comparison;
|
|
528
|
+
const cell = (row, side) => (row.kind === 'task' ? pct(row[side]) : row[side] === null || row[side] === undefined ? '—' : t(row[side] ? 'skillsStudio.test.report.right' : 'skillsStudio.test.report.wrong'));
|
|
529
|
+
return html`<div class="sx-test-comparison" role="region" aria-label="${t('skillsStudio.test.history.comparison')}">
|
|
530
|
+
<h5 class="sx-test-pane-title">${t('skillsStudio.test.history.comparison')}</h5>
|
|
531
|
+
${!c.sameSuite ? html`<p class="sx-note sx-note--warn">${t('skillsStudio.test.history.differentSuite')}</p>` : ''}
|
|
532
|
+
${c.sameRevision ? html`<p class="sx-help">${t('skillsStudio.test.history.sameRevision')}</p>` : ''}
|
|
533
|
+
<table class="sx-test-table"><thead><tr><th scope="col">${t('skillsStudio.test.report.case')}</th><th scope="col">${t('skillsStudio.test.history.runtime')}</th><th scope="col">${t('skillsStudio.test.history.older')}</th><th scope="col">${t('skillsStudio.test.history.newer')}</th><th scope="col">${t('skillsStudio.test.report.change')}</th></tr></thead>
|
|
534
|
+
<tbody>${c.rows.map((row) => html`<tr><th scope="row">${row.caseId}</th><td>${row.key}</td><td>${cell(row, 'a')}</td><td>${cell(row, 'b')}</td><td><span class="sx-test-delta sx-test-delta--${row.change === null ? 'none' : row.change > 0 ? 'up' : row.change < 0 ? 'down' : 'flat'}">${row.change === null ? '—' : row.kind === 'task' ? pts(row.change) : t(row.change > 0 ? 'skillsStudio.test.outcome.better' : row.change < 0 ? 'skillsStudio.test.outcome.worse' : 'skillsStudio.test.outcome.same')}</span></td></tr>`)}</tbody></table>
|
|
535
|
+
</div>`;
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
// ── Import ──
|
|
539
|
+
|
|
540
|
+
function importHTML(ctx, st) {
|
|
541
|
+
const { t } = ctx;
|
|
542
|
+
const p = st.importPreview;
|
|
543
|
+
if (!p) return '';
|
|
544
|
+
if (!p.ok) return html`<div class="sx-ai-error" role="alert"><p>${t('skillsStudio.test.import.failed')}</p><ul class="sx-plain-list">${p.problems.slice(0, 5).map((x) => html`<li>${x.message}</li>`)}</ul></div>`;
|
|
545
|
+
const counts = { tasks: p.evals.evals.length, triggers: p.evals.triggers.length };
|
|
546
|
+
return html`<div class="sx-outcome" role="status"><p class="sx-outcome-line">${ICONS.ok}<span>${t('skillsStudio.test.import.ready', { tasks: counts.tasks, triggers: counts.triggers, dialects: p.dialects.join(', ') })}</span></p>
|
|
547
|
+
${p.notes.length ? html`<ul class="sx-plain-list">${p.notes.slice(0, 5).map((n) => html`<li>${n}</li>`)}</ul>` : ''}
|
|
548
|
+
<div class="sx-row"><button type="button" class="sx-btn sx-btn--primary sx-btn--sm" data-test="import-save">${t('skillsStudio.test.import.save', { path: EVALS_PATH })}</button><button type="button" class="sx-btn sx-btn--ghost sx-btn--sm" data-test="import-drop">${t('skillsStudio.dialog.cancel')}</button></div></div>`;
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
/** Pick one or two files (a task file, a trigger set) and convert them here, with the same reader the server uses. */
|
|
552
|
+
function pickImport(panel, ctx, ws, signal) {
|
|
553
|
+
const st = stateOf(ws);
|
|
554
|
+
const input = document.createElement('input');
|
|
555
|
+
input.type = 'file';
|
|
556
|
+
input.accept = '.json,application/json';
|
|
557
|
+
input.multiple = true;
|
|
558
|
+
input.addEventListener('change', async () => {
|
|
559
|
+
const files = [...(input.files || [])].slice(0, 2);
|
|
560
|
+
if (!files.length) return;
|
|
561
|
+
let tasks = null; let triggers = null;
|
|
562
|
+
for (const f of files) {
|
|
563
|
+
if (f.size > 1024 * 1024) { ctx.toast(ctx.t('skillsStudio.test.import.tooLarge'), 'error'); return; }
|
|
564
|
+
const text = await f.text();
|
|
565
|
+
let parsed = null;
|
|
566
|
+
try { parsed = JSON.parse(text); } catch { parsed = null; }
|
|
567
|
+
if (Array.isArray(parsed)) triggers = text; else tasks = text;
|
|
568
|
+
}
|
|
569
|
+
if (!live(panel, signal)) return;
|
|
570
|
+
st.importPreview = importEvals({ tasks, triggers, skillName: ws.row?.folderName || ws.label || '' });
|
|
571
|
+
const box = panel.querySelector('#sx-test-import');
|
|
572
|
+
if (box) setHTML(box, importHTML(ctx, st));
|
|
573
|
+
}, { once: true });
|
|
574
|
+
input.click();
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
/** The imported suite into the skill's draft as evals/evals.json (the owner then saves a version to test it). */
|
|
578
|
+
async function saveImport(panel, ctx, ws, signal) {
|
|
579
|
+
const st = stateOf(ws);
|
|
580
|
+
const p = st.importPreview;
|
|
581
|
+
if (!p?.ok) return;
|
|
582
|
+
const gen = ctx.gen();
|
|
583
|
+
try {
|
|
584
|
+
let draft = ws.draft;
|
|
585
|
+
if (!draft) draft = (await ctx.api.createSkillsDraft({ skillId: ws.skill.skillId }, { idempotencyKey: ctx.key('sx-draft') })).draft;
|
|
586
|
+
const out = await ctx.api.patchSkillsDraft(draft.draftId, { expectedEtag: draft.etag, ops: [{ op: 'put', path: EVALS_PATH, content: `${JSON.stringify(p.evals, null, 2)}\n` }] });
|
|
587
|
+
if (!ctx.alive(gen)) return;
|
|
588
|
+
ws.draft = out.draft;
|
|
589
|
+
st.importPreview = null;
|
|
590
|
+
ctx.toast(ctx.t('skillsStudio.test.import.saved', { path: EVALS_PATH }));
|
|
591
|
+
ctx.announce(ctx.t('skillsStudio.test.import.saved', { path: EVALS_PATH }));
|
|
592
|
+
await ctx.reload();
|
|
593
|
+
} catch (err) { if (ctx.alive(gen) && live(panel, signal)) ctx.toast(ctx.t('skillsStudio.test.import.saveFailed', { error: testErrorText(ctx.t, err) }), 'error'); }
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
// ── Loading and following ──
|
|
597
|
+
|
|
598
|
+
async function load(panel, ctx, ws, signal) {
|
|
599
|
+
const st = stateOf(ws);
|
|
600
|
+
const gen = ctx.gen();
|
|
601
|
+
const ok = () => ctx.alive(gen) && live(panel, signal) && ws.tab === 'test';
|
|
602
|
+
const jobs = [];
|
|
603
|
+
if (!st.options) {
|
|
604
|
+
jobs.push(ctx.api.fetchSkillEvalOptions().then((res) => {
|
|
605
|
+
st.options = res; st.optionsError = null;
|
|
606
|
+
if (!st.formFromOptions) { st.form = defaultForm(res, readStored(ctx)); st.formFromOptions = true; }
|
|
607
|
+
if (ok()) { redraw(panel, ctx, ws, ['setup', 'bar', 'head']); scheduleEstimate(panel, ctx, ws, signal); }
|
|
608
|
+
}).catch((err) => { st.optionsError = testErrorText(ctx.t, err); if (ok()) redraw(panel, ctx, ws, ['bar']); }));
|
|
609
|
+
} else scheduleEstimate(panel, ctx, ws, signal);
|
|
610
|
+
if (st.suiteFor !== st.revisionId) {
|
|
611
|
+
jobs.push(ctx.api.fetchSkillsRevisionFile(st.revisionId, EVALS_PATH).then((res) => {
|
|
612
|
+
const head = (ws.revision?.manifest?.files || []).find((f) => f.path === EVALS_PATH);
|
|
613
|
+
st.suiteDigest = head?.digest || res.digest || null;
|
|
614
|
+
st.suite = readSuiteText(res.content || '', { skillName: ws.row?.folderName || '' });
|
|
615
|
+
st.suiteFor = st.revisionId; st.suiteError = null;
|
|
616
|
+
if (ok()) { redraw(panel, ctx, ws, ['head', 'setup', 'bar']); scheduleEstimate(panel, ctx, ws, signal); }
|
|
617
|
+
}).catch((err) => { st.suiteError = testErrorText(ctx.t, err); if (ok()) redraw(panel, ctx, ws, ['head', 'bar']); }));
|
|
618
|
+
}
|
|
619
|
+
jobs.push(reloadHistory(panel, ctx, ws, signal));
|
|
620
|
+
if (!st.revisions) {
|
|
621
|
+
jobs.push(ctx.api.fetchSkillRevisions(ws.skill.skillId).then((res) => {
|
|
622
|
+
st.revisions = (res.revisions || []).slice().sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)));
|
|
623
|
+
// A baseline revision kept from before that is not one of these goes back to the first earlier one.
|
|
624
|
+
if (st.form?.baseline === 'revision' && !st.revisions.some((r) => r.revisionId === st.form.baselineRevisionId)) st.form.baselineRevisionId = st.revisions.find((r) => r.revisionId !== st.revisionId)?.revisionId || '';
|
|
625
|
+
if (ok()) redraw(panel, ctx, ws, ['setup']);
|
|
626
|
+
}).catch(() => { st.revisions = []; }));
|
|
627
|
+
}
|
|
628
|
+
if (!st.catalog) {
|
|
629
|
+
jobs.push(import('../../assistant/asst-brain-picker.js').then((m) => m.loadModelCatalog()).then((catalog) => { st.catalog = catalog; if (ok()) redraw(panel, ctx, ws, ['setup']); }).catch(() => { /* the providers' defaults are still offered */ }));
|
|
630
|
+
}
|
|
631
|
+
await Promise.allSettled(jobs);
|
|
632
|
+
if (st.openId && st.run && isActiveEvalRun(st.run.state) && ok()) follow(panel, ctx, ws, signal);
|
|
633
|
+
else if (st.openId && st.run && !st.runFull && ok()) openRun(panel, ctx, ws, signal, st.openId);
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
async function reloadHistory(panel, ctx, ws, signal) {
|
|
637
|
+
const st = stateOf(ws);
|
|
638
|
+
const gen = ctx.gen();
|
|
639
|
+
try {
|
|
640
|
+
const res = await ctx.api.fetchSkillEvals({ skillId: ws.skill.skillId, limit: 30 });
|
|
641
|
+
if (!ctx.alive(gen) || !live(panel, signal)) { st.history = res.evals || []; return; }
|
|
642
|
+
st.history = res.evals || [];
|
|
643
|
+
if (res.queue && st.options) st.options = { ...st.options, queue: res.queue };
|
|
644
|
+
st.historyError = null;
|
|
645
|
+
// The newest run opens by itself the first time (a running one is followed).
|
|
646
|
+
if (!st.openId && st.history.length) {
|
|
647
|
+
const first = st.history[0];
|
|
648
|
+
st.openId = first.evalRunId;
|
|
649
|
+
st.run = first; st.runFull = false;
|
|
650
|
+
if (ws.tab === 'test') { redraw(panel, ctx, ws, ['run', 'bar']); if (!isActiveEvalRun(first.state)) openRun(panel, ctx, ws, signal, first.evalRunId); }
|
|
651
|
+
}
|
|
652
|
+
if (ws.tab === 'test') redraw(panel, ctx, ws, ['history', 'head', ...(st.run ? [] : ['run', 'setup'])]);
|
|
653
|
+
} catch (err) { st.historyError = testErrorText(ctx.t, err); if (ws.tab === 'test' && live(panel, signal)) redraw(panel, ctx, ws, ['history']); }
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
/** The estimate of the form as it is, a moment after the last change (one request at a time, the last one wins). */
|
|
657
|
+
function scheduleEstimate(panel, ctx, ws, signal) {
|
|
658
|
+
const st = stateOf(ws);
|
|
659
|
+
clearTimeout(st.estimateTimer);
|
|
660
|
+
if (!st.options || !st.suite?.ok || ctx.guest) return;
|
|
661
|
+
st.estimateTimer = setTimeout(async () => {
|
|
662
|
+
if (!live(panel, signal)) return;
|
|
663
|
+
const seq = ++st.estimateSeq;
|
|
664
|
+
st.estimating = true;
|
|
665
|
+
redraw(panel, ctx, ws, ['estimate']);
|
|
666
|
+
try {
|
|
667
|
+
const res = await ctx.api.estimateSkillEval(requestOf(st.form, { revisionId: st.revisionId }));
|
|
668
|
+
if (seq !== st.estimateSeq || !live(panel, signal)) return;
|
|
669
|
+
st.estimate = res; st.estimateError = null;
|
|
670
|
+
st.codexSeen = res.codex || null;
|
|
671
|
+
} catch (err) {
|
|
672
|
+
if (seq !== st.estimateSeq || !live(panel, signal)) return;
|
|
673
|
+
st.estimate = null;
|
|
674
|
+
st.estimateError = err.body?.problems?.[0]?.code === 'test-sealed' ? ctx.t('skillsStudio.test.errors.sealed') : testErrorText(ctx.t, err);
|
|
675
|
+
}
|
|
676
|
+
st.estimating = false;
|
|
677
|
+
redraw(panel, ctx, ws, ['estimate', 'bar']);
|
|
678
|
+
}, 350);
|
|
679
|
+
signal.addEventListener('abort', () => clearTimeout(st.estimateTimer), { once: true });
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
async function start(panel, ctx, ws, signal, purpose, caseIds = null) {
|
|
683
|
+
const st = stateOf(ws);
|
|
684
|
+
if (st.starting || ctx.guest) return;
|
|
685
|
+
const gen = ctx.gen();
|
|
686
|
+
st.starting = true; st.notice = null;
|
|
687
|
+
redraw(panel, ctx, ws, ['bar']);
|
|
688
|
+
const form = purpose === 'optimise' ? { ...st.form, budgetUsd: Math.max(st.form.budgetUsd, 5), maxMinutes: Math.max(st.form.maxMinutes, 60) } : st.form;
|
|
689
|
+
try {
|
|
690
|
+
const res = await ctx.api.startSkillEval(requestOf(form, { revisionId: st.revisionId, purpose, caseIds }), { idempotencyKey: idempotencyKey('sx-eval') });
|
|
691
|
+
if (!ctx.alive(gen)) return;
|
|
692
|
+
st.openId = res.eval.evalRunId; st.run = res.eval; st.runFull = true; st.trialId = null; st.trial = null; st.comparison = null;
|
|
693
|
+
ctx.announce(ctx.t('skillsStudio.test.run.started'));
|
|
694
|
+
st.starting = false;
|
|
695
|
+
if (live(panel, signal)) {
|
|
696
|
+
redraw(panel, ctx, ws, ['setup', 'run', 'bar', 'trial']);
|
|
697
|
+
panel.querySelector('#sx-test-run-title')?.focus({ preventScroll: false });
|
|
698
|
+
follow(panel, ctx, ws, signal);
|
|
699
|
+
reloadHistory(panel, ctx, ws, signal);
|
|
700
|
+
}
|
|
701
|
+
} catch (err) {
|
|
702
|
+
st.starting = false;
|
|
703
|
+
if (!ctx.alive(gen)) return;
|
|
704
|
+
st.notice = startError(ctx, err);
|
|
705
|
+
ctx.toast(st.notice, 'error');
|
|
706
|
+
if (live(panel, signal)) redraw(panel, ctx, ws, ['run', 'bar']);
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
function startError(ctx, err) {
|
|
711
|
+
const { t } = ctx;
|
|
712
|
+
const code = err?.code || err?.body?.code || '';
|
|
713
|
+
if (code === 'budget_below_estimate') return t('skillsStudio.test.errors.budget', { low: money(err.body?.estimate?.usd?.low), high: money(err.body?.estimate?.usd?.high) });
|
|
714
|
+
if (code === 'time_below_estimate') return t('skillsStudio.test.errors.time');
|
|
715
|
+
if (code === 'queue_full') return t('skillsStudio.test.errors.queueFull');
|
|
716
|
+
if (code === 'evals_unavailable') return t('skillsStudio.test.errors.notReady');
|
|
717
|
+
if (code === 'suite_invalid') return t('skillsStudio.test.errors.suite');
|
|
718
|
+
if (code === 'runtime_unavailable') return t('skillsStudio.test.errors.runtime', { error: testErrorText(t, err) });
|
|
719
|
+
if (code === 'optimise_unavailable') return t('skillsStudio.test.errors.optimise');
|
|
720
|
+
if (code === 'codex_not_confirmed') return t('skillsStudio.test.errors.codex');
|
|
721
|
+
return t('skillsStudio.test.errors.start', { error: testErrorText(t, err) });
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
/** Follow the open run while it runs: a long poll, one at a time, ended by the tab's signal. */
|
|
725
|
+
async function follow(panel, ctx, ws, signal) {
|
|
726
|
+
const st = stateOf(ws);
|
|
727
|
+
const id = st.openId;
|
|
728
|
+
if (!id || st.following === id) return;
|
|
729
|
+
st.following = id;
|
|
730
|
+
const gen = ctx.gen();
|
|
731
|
+
const tick = setInterval(() => {
|
|
732
|
+
const el = panel.querySelector('#sx-test-clock');
|
|
733
|
+
if (!live(panel, signal) || !el || !isActiveEvalRun(st.run?.state)) { clearInterval(tick); return; }
|
|
734
|
+
el.textContent = clockText(elapsedSeconds(st.run));
|
|
735
|
+
}, 1000);
|
|
736
|
+
signal.addEventListener('abort', () => clearInterval(tick), { once: true });
|
|
737
|
+
try {
|
|
738
|
+
while (live(panel, signal) && ctx.alive(gen) && st.openId === id && isActiveEvalRun(st.run?.state)) {
|
|
739
|
+
let next = null;
|
|
740
|
+
try { next = (await ctx.api.waitSkillEval(id, { since: st.run.version ?? -1, seconds: 20 })).eval; } catch { await new Promise((r) => setTimeout(r, 3000)); continue; }
|
|
741
|
+
if (!live(panel, signal) || !ctx.alive(gen) || st.openId !== id) return;
|
|
742
|
+
const was = st.run?.state;
|
|
743
|
+
st.run = next; st.runFull = false;
|
|
744
|
+
if (was !== next.state) ctx.announce(ctx.t('skillsStudio.test.run.nowState', { state: stateWord(ctx.t, next.state) }));
|
|
745
|
+
if (!isActiveEvalRun(next.state)) { st.cancelling = false; await openRun(panel, ctx, ws, signal, id); reloadHistory(panel, ctx, ws, signal); return; }
|
|
746
|
+
redraw(panel, ctx, ws, ['run', 'bar']);
|
|
747
|
+
}
|
|
748
|
+
} finally { if (st.following === id) st.following = null; clearInterval(tick); }
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
/** A run in full (its report), and the trial viewer's previous runs. */
|
|
752
|
+
async function openRun(panel, ctx, ws, signal, id) {
|
|
753
|
+
const st = stateOf(ws);
|
|
754
|
+
const gen = ctx.gen();
|
|
755
|
+
st.openId = id;
|
|
756
|
+
try {
|
|
757
|
+
const res = await ctx.api.fetchSkillEval(id);
|
|
758
|
+
if (!ctx.alive(gen) || !live(panel, signal) || st.openId !== id) return;
|
|
759
|
+
st.run = res.eval; st.runFull = !isActiveEvalRun(res.eval.state);
|
|
760
|
+
redraw(panel, ctx, ws, ['run', 'bar', 'history']);
|
|
761
|
+
if (isActiveEvalRun(res.eval.state)) follow(panel, ctx, ws, signal);
|
|
762
|
+
} catch (err) { if (ctx.alive(gen) && live(panel, signal)) { st.notice = ctx.t('skillsStudio.test.errors.open', { error: testErrorText(ctx.t, err) }); redraw(panel, ctx, ws, ['run']); } }
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
async function openTrial(panel, ctx, ws, signal, trialId) {
|
|
766
|
+
const st = stateOf(ws);
|
|
767
|
+
const gen = ctx.gen();
|
|
768
|
+
st.trialId = trialId; st.trial = null; st.trialError = null; st.previous = null;
|
|
769
|
+
redraw(panel, ctx, ws, ['trial']);
|
|
770
|
+
try {
|
|
771
|
+
const res = await ctx.api.fetchSkillEvalTrial(st.openId, trialId);
|
|
772
|
+
if (!ctx.alive(gen) || !live(panel, signal) || st.trialId !== trialId) return;
|
|
773
|
+
st.trial = res.trial;
|
|
774
|
+
// The same case's answer in the skill's previous run, beside this one.
|
|
775
|
+
const prevRef = previousTrial((st.history || []).filter((r) => r.evalRunId !== st.openId).map((r) => ({ ...r, trials: st.fullRuns?.[r.evalRunId]?.trials || null })).filter((r) => r.trials), st.run, res.trial);
|
|
776
|
+
if (prevRef) {
|
|
777
|
+
try { const prev = await ctx.api.fetchSkillEvalTrial(prevRef.evalRunId, prevRef.trialId); if (st.trialId === trialId) st.previous = prev.trial; } catch { /* no previous answer */ }
|
|
778
|
+
} else {
|
|
779
|
+
const older = (st.history || []).find((r) => r.evalRunId !== st.openId && Date.parse(r.createdAt) < Date.parse(st.run.createdAt) && !isActiveEvalRun(r.state));
|
|
780
|
+
if (older) {
|
|
781
|
+
try {
|
|
782
|
+
const full = (await ctx.api.fetchSkillEval(older.evalRunId)).eval;
|
|
783
|
+
st.fullRuns = { ...(st.fullRuns || {}), [older.evalRunId]: full };
|
|
784
|
+
const ref = previousTrial([full], st.run, res.trial);
|
|
785
|
+
if (ref && st.trialId === trialId) st.previous = (await ctx.api.fetchSkillEvalTrial(ref.evalRunId, ref.trialId)).trial;
|
|
786
|
+
} catch { /* no previous answer */ }
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
} catch (err) { if (ctx.alive(gen) && st.trialId === trialId) st.trialError = ctx.t('skillsStudio.test.trial.failed', { error: testErrorText(ctx.t, err) }); }
|
|
790
|
+
if (!live(panel, signal) || st.trialId !== trialId) return;
|
|
791
|
+
redraw(panel, ctx, ws, ['trial']);
|
|
792
|
+
panel.querySelector('#sx-test-trial-box')?.focus({ preventScroll: false });
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
/** The owner's note on a trial, saved a moment after typing stops; the answer says it was kept. */
|
|
796
|
+
function queueFeedback(panel, ctx, ws, signal, { immediate = false } = {}) {
|
|
797
|
+
const st = stateOf(ws);
|
|
798
|
+
const tr = st.trial;
|
|
799
|
+
if (!tr) return;
|
|
800
|
+
const fb = st.feedbackState[tr.trialId];
|
|
801
|
+
clearTimeout(fb.timer);
|
|
802
|
+
fb.state = 'dirty';
|
|
803
|
+
paintFeedback(panel, ctx, fb);
|
|
804
|
+
const save = async () => {
|
|
805
|
+
const runId = st.openId;
|
|
806
|
+
const trialId = tr.trialId;
|
|
807
|
+
const seq = (fb.seq = (fb.seq || 0) + 1);
|
|
808
|
+
fb.state = 'saving'; paintFeedback(panel, ctx, fb);
|
|
809
|
+
try {
|
|
810
|
+
const res = await ctx.api.saveSkillEvalFeedback(runId, { trialId, text: fb.text, reviewed: fb.reviewed });
|
|
811
|
+
if (seq !== fb.seq) return;
|
|
812
|
+
fb.state = 'idle'; fb.saved = res.feedback.at;
|
|
813
|
+
} catch { if (seq === fb.seq) fb.state = 'error'; }
|
|
814
|
+
if (live(panel, signal) && st.trial?.trialId === trialId) paintFeedback(panel, ctx, fb);
|
|
815
|
+
};
|
|
816
|
+
if (immediate) save(); else fb.timer = setTimeout(save, 800);
|
|
817
|
+
}
|
|
818
|
+
function paintFeedback(panel, ctx, fb) { const el = panel.querySelector('#sx-test-note-state'); if (el) el.textContent = feedbackWord(ctx, fb); }
|
|
819
|
+
|
|
820
|
+
// ── Redrawing parts ──
|
|
821
|
+
|
|
822
|
+
function redraw(panel, ctx, ws, parts) {
|
|
823
|
+
const st = stateOf(ws);
|
|
824
|
+
if (!panel.isConnected) return;
|
|
825
|
+
const set = (sel, markup) => { const el = panel.querySelector(sel); if (el) setHTML(el, markup); };
|
|
826
|
+
for (const part of parts) {
|
|
827
|
+
if (part === 'head') { st.fixturesOpen = !!panel.querySelector('#sx-test-fixtures')?.open; set('#sx-test-head', headHTML(ctx, ws, st)); }
|
|
828
|
+
// The setup is drawn again only while nothing in it has the focus (a dropdown being used keeps its state).
|
|
829
|
+
else if (part === 'setup') { const box = panel.querySelector('#sx-test-setup'); if (box && !box.contains(document.activeElement)) { st.moreOpen = !!box.querySelector('.sx-test-more')?.open; setHTML(box, setupHTML(ctx, ws, st)); } }
|
|
830
|
+
else if (part === 'estimate') { set('#sx-test-estimate', estimateHTML(ctx, st)); set('#sx-test-codex-box', codexHTML(ctx, st)); set('#sx-test-safety', safetyHTML(ctx, st)); }
|
|
831
|
+
else if (part === 'codex') set('#sx-test-codex-box', codexHTML(ctx, st));
|
|
832
|
+
else if (part === 'safety') set('#sx-test-safety', safetyHTML(ctx, st));
|
|
833
|
+
else if (part === 'bar') set('#sx-test-bar', barHTML(ctx, ws, st));
|
|
834
|
+
else if (part === 'run') set('#sx-test-run', runHTML(ctx, ws, st));
|
|
835
|
+
else if (part === 'trial') { const box = panel.querySelector('#sx-test-trial'); if (box && !(box.contains(document.activeElement) && document.activeElement?.id === 'sx-test-note')) setHTML(box, trialHTML(ctx, ws, st)); }
|
|
836
|
+
else if (part === 'history') { const box = panel.querySelector('#sx-test-history'); if (box && !box.contains(document.activeElement)) setHTML(box, historyHTML(ctx, ws, st)); }
|
|
837
|
+
}
|
|
838
|
+
panel.querySelector('.sx-test')?.setAttribute('data-test-state', st.run?.state || 'idle');
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
// ── Listening ──
|
|
842
|
+
|
|
843
|
+
/** ⌘/Ctrl+Enter is Run evals while the Test tab is shown (the workspace's shortcut). */
|
|
844
|
+
export function testShortcut(e, ctx, ws) {
|
|
845
|
+
if (ws.tab !== 'test') return false;
|
|
846
|
+
const mod = ctx.mac ? e.metaKey : e.ctrlKey;
|
|
847
|
+
if (mod && e.key === 'Enter' && !e.altKey && !e.shiftKey) {
|
|
848
|
+
const btn = ctx.panel?.querySelector('#sx-test-run-btn');
|
|
849
|
+
if (btn && !btn.disabled) { btn.click(); return true; }
|
|
850
|
+
}
|
|
851
|
+
return false;
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
function wire(panel, ctx, ws, signal) {
|
|
855
|
+
const st = stateOf(ws);
|
|
856
|
+
const formChanged = () => { keepForm(ctx, st.form); st.estimate = null; scheduleEstimate(panel, ctx, ws, signal); redraw(panel, ctx, ws, ['codex', 'safety', 'bar']); };
|
|
857
|
+
panel.addEventListener('change', (e) => {
|
|
858
|
+
const el = e.target;
|
|
859
|
+
const id = el?.id || '';
|
|
860
|
+
if (/^sx-test-runtime-\d$/.test(id)) { const i = Number(id.slice(-1)); st.form.runtimes[i] = el.value; st.form.runtimes = [...new Set(st.form.runtimes)]; formChanged(); if (st.form.runtimes.length !== i + 1 && i >= st.form.runtimes.length) redraw(panel, ctx, ws, ['setup']); return; }
|
|
861
|
+
if (id === 'sx-test-baseline') { st.form.baseline = el.value; formChanged(); redraw(panel, ctx, ws, ['setup']); return; }
|
|
862
|
+
if (id === 'sx-test-baseline-rev') { st.form.baselineRevisionId = el.value; formChanged(); return; }
|
|
863
|
+
if (id === 'sx-test-budget') { st.form.budgetUsd = Number(el.value); formChanged(); return; }
|
|
864
|
+
if (id === 'sx-test-minutes') { st.form.maxMinutes = Number(el.value); formChanged(); return; }
|
|
865
|
+
if (id === 'sx-test-task-reps') { st.form.taskReps = Number(el.value); formChanged(); return; }
|
|
866
|
+
if (id === 'sx-test-trigger-reps') { st.form.triggerReps = Number(el.value); formChanged(); return; }
|
|
867
|
+
if (id === 'sx-test-grader') { st.form.grader = el.value; formChanged(); return; }
|
|
868
|
+
if (id === 'sx-test-judge') { st.form.judge = el.checked; formChanged(); return; }
|
|
869
|
+
if (id === 'sx-test-export-format') { st.exportFormat = el.value; return; }
|
|
870
|
+
if (id === 'sx-test-reviewed') { const fb = st.feedbackState[st.trial?.trialId]; if (fb) { fb.reviewed = el.checked; queueFeedback(panel, ctx, ws, signal, { immediate: true }); } return; }
|
|
871
|
+
if (el?.dataset?.test === 'pick') {
|
|
872
|
+
if (el.checked) { if (st.compare.size >= 2) { const first = st.compare.values().next().value; st.compare.delete(first); } st.compare.add(el.dataset.id); } else st.compare.delete(el.dataset.id);
|
|
873
|
+
st.comparison = null;
|
|
874
|
+
redrawHistorySoon(panel, ctx, ws);
|
|
875
|
+
}
|
|
876
|
+
}, { signal });
|
|
877
|
+
panel.addEventListener('input', (e) => {
|
|
878
|
+
if (e.target?.id !== 'sx-test-note') return;
|
|
879
|
+
const fb = st.feedbackState[st.trial?.trialId];
|
|
880
|
+
if (!fb) return;
|
|
881
|
+
fb.text = e.target.value;
|
|
882
|
+
queueFeedback(panel, ctx, ws, signal);
|
|
883
|
+
}, { signal });
|
|
884
|
+
panel.addEventListener('focusout', (e) => {
|
|
885
|
+
if (e.target?.id === 'sx-test-note') { const fb = st.feedbackState[st.trial?.trialId]; if (fb?.state === 'dirty') queueFeedback(panel, ctx, ws, signal, { immediate: true }); }
|
|
886
|
+
}, { signal });
|
|
887
|
+
panel.addEventListener('keydown', (e) => {
|
|
888
|
+
const box = e.target.closest?.('#sx-test-trial-box');
|
|
889
|
+
if (!box || e.target.closest?.('textarea, input, .cc-dropdown')) return;
|
|
890
|
+
if (e.key === 'ArrowRight' || e.key === 'ArrowLeft') {
|
|
891
|
+
e.preventDefault();
|
|
892
|
+
const next = trialStep(st.run, st.trialId, e.key === 'ArrowRight' ? 1 : -1);
|
|
893
|
+
if (next) openTrial(panel, ctx, ws, signal, next);
|
|
894
|
+
} else if (e.key === 'Escape') { e.preventDefault(); st.trialId = null; st.trial = null; redraw(panel, ctx, ws, ['trial']); }
|
|
895
|
+
}, { signal });
|
|
896
|
+
panel.addEventListener('click', async (e) => {
|
|
897
|
+
// The owner opening or folding the setup (a click or Enter on its summary, before the browser toggles it) is kept.
|
|
898
|
+
const summary = e.target.closest?.('.sx-test-setup-summary');
|
|
899
|
+
if (summary) { const box = summary.parentElement; if (box) st.setupOpen = !box.open; return; }
|
|
900
|
+
const btn = e.target.closest?.('[data-test]');
|
|
901
|
+
if (!btn || btn.disabled || btn.tagName === 'INPUT') return;
|
|
902
|
+
const act = btn.dataset.test;
|
|
903
|
+
switch (act) {
|
|
904
|
+
case 'run': start(panel, ctx, ws, signal, 'evals'); return;
|
|
905
|
+
case 'final': start(panel, ctx, ws, signal, 'final'); return;
|
|
906
|
+
case 'optimise': start(panel, ctx, ws, signal, 'optimise'); return;
|
|
907
|
+
case 'rerun': start(panel, ctx, ws, signal, 'evals', rerunCases(st.run)); return;
|
|
908
|
+
case 'cancel': {
|
|
909
|
+
st.cancelling = true; redraw(panel, ctx, ws, ['run']);
|
|
910
|
+
try { st.run = (await ctx.api.cancelSkillEval(st.openId)).eval; } catch (err) { ctx.toast(ctx.t('skillsStudio.test.errors.cancel', { error: testErrorText(ctx.t, err) }), 'error'); }
|
|
911
|
+
st.cancelling = false;
|
|
912
|
+
if (live(panel, signal)) { if (!isActiveEvalRun(st.run?.state)) openRun(panel, ctx, ws, signal, st.openId); else redraw(panel, ctx, ws, ['run']); reloadHistory(panel, ctx, ws, signal); }
|
|
913
|
+
return;
|
|
914
|
+
}
|
|
915
|
+
case 'kinds': st.form.kinds = btn.dataset.value; formChanged(); redraw(panel, ctx, ws, ['setup']); panel.querySelector(`[data-test="kinds"][data-value="${btn.dataset.value}"]`)?.focus(); return;
|
|
916
|
+
case 'add-runtime': {
|
|
917
|
+
const used = new Set(st.form.runtimes.map((v) => v.split('|')[0]));
|
|
918
|
+
// Another runtime that is not an opt-in one first (Codex is picked by the owner in a row's list, never by this).
|
|
919
|
+
const rows = (st.options?.runtimes || []).filter((r) => r.available && !used.has(r.provider));
|
|
920
|
+
const next = rows.find((r) => !isOptInRuntime(r.provider, st.options)) || rows[0] || (st.options?.runtimes || []).find((r) => r.available);
|
|
921
|
+
if (next) { st.form.runtimes = [...st.form.runtimes, next.provider].slice(0, 3); st.form.runtimes = [...new Set(st.form.runtimes)]; formChanged(); redraw(panel, ctx, ws, ['setup']); }
|
|
922
|
+
return;
|
|
923
|
+
}
|
|
924
|
+
case 'remove-runtime': st.form.runtimes.splice(Number(btn.dataset.index), 1); formChanged(); redraw(panel, ctx, ws, ['setup']); return;
|
|
925
|
+
case 'open-run': st.trialId = null; st.trial = null; st.comparison = null; redraw(panel, ctx, ws, ['trial']); openRun(panel, ctx, ws, signal, btn.dataset.id); return;
|
|
926
|
+
case 'open-trial': openTrial(panel, ctx, ws, signal, btn.dataset.trial); return;
|
|
927
|
+
case 'trial-prev': case 'trial-next': { const next = trialStep(st.run, st.trialId, act === 'trial-next' ? 1 : -1); if (next) openTrial(panel, ctx, ws, signal, next); return; }
|
|
928
|
+
case 'trial-close': st.trialId = null; st.trial = null; redraw(panel, ctx, ws, ['trial']); panel.querySelector('#sx-test-run-title')?.focus(); return;
|
|
929
|
+
case 'trial-file': {
|
|
930
|
+
const gen = ctx.gen();
|
|
931
|
+
const view = panel.querySelector('#sx-test-file-view');
|
|
932
|
+
try {
|
|
933
|
+
const res = await ctx.api.fetchSkillEvalTrialFile(st.openId, st.trialId, btn.dataset.path);
|
|
934
|
+
if (!ctx.alive(gen) || !view?.isConnected) return;
|
|
935
|
+
setHTML(view, res.file.text === null ? html`<p class="sx-help">${ctx.t('skillsStudio.test.trial.notText')}</p>` : html`${res.file.kept ? html`<p class="sx-help">${ctx.t(res.file.clipped ? 'skillsStudio.test.trial.keptClipped' : 'skillsStudio.test.trial.kept')}</p>` : ''}<pre class="sx-pre">${res.file.text}</pre>`);
|
|
936
|
+
} catch (err) { if (view?.isConnected) setHTML(view, html`<p class="sx-help sx-err-text">${testErrorText(ctx.t, err)}</p>`); }
|
|
937
|
+
return;
|
|
938
|
+
}
|
|
939
|
+
case 'compare': {
|
|
940
|
+
const [a, b] = [...st.compare].map((id) => st.history.find((r) => r.evalRunId === id)).filter(Boolean).sort((x, y) => String(x.createdAt).localeCompare(String(y.createdAt)));
|
|
941
|
+
if (!a || !b) return;
|
|
942
|
+
try { st.comparison = (await ctx.api.compareSkillEvals(a.evalRunId, b.evalRunId)).comparison; } catch (err) { ctx.toast(ctx.t('skillsStudio.test.errors.compare', { error: testErrorText(ctx.t, err) }), 'error'); return; }
|
|
943
|
+
if (live(panel, signal)) { const box = panel.querySelector('#sx-test-compare'); if (box) setHTML(box, comparisonHTML(ctx, st)); }
|
|
944
|
+
return;
|
|
945
|
+
}
|
|
946
|
+
case 'export': {
|
|
947
|
+
const a = document.createElement('a');
|
|
948
|
+
a.href = ctx.api.getSkillEvalExportUrl(st.openId, st.exportFormat);
|
|
949
|
+
a.download = '';
|
|
950
|
+
a.rel = 'noopener';
|
|
951
|
+
document.body.appendChild(a); a.click(); a.remove();
|
|
952
|
+
return;
|
|
953
|
+
}
|
|
954
|
+
case 'codex-confirm': case 'codex-undo': {
|
|
955
|
+
const c = codexNow(st);
|
|
956
|
+
if (!c) return;
|
|
957
|
+
const confirm = act === 'codex-confirm';
|
|
958
|
+
const ids = confirm ? (c.pending.length ? c.pending : c.revisionIds) : c.revisionIds;
|
|
959
|
+
if (!ids.length) return;
|
|
960
|
+
st.codexBusy = true; redraw(panel, ctx, ws, ['codex']);
|
|
961
|
+
try {
|
|
962
|
+
await ctx.api.setSkillEvalCodexTrust(ids, confirm);
|
|
963
|
+
ctx.announce(ctx.t(confirm ? 'skillsStudio.test.codex.announceOn' : 'skillsStudio.test.codex.announceOff'));
|
|
964
|
+
} catch (err) { ctx.toast(ctx.t('skillsStudio.test.codex.failed', { error: testErrorText(ctx.t, err) }), 'error'); }
|
|
965
|
+
st.codexBusy = false;
|
|
966
|
+
if (!live(panel, signal)) return;
|
|
967
|
+
// The estimate says again what Codex may do now.
|
|
968
|
+
st.estimate = null; scheduleEstimate(panel, ctx, ws, signal); redraw(panel, ctx, ws, ['codex', 'bar']);
|
|
969
|
+
return;
|
|
970
|
+
}
|
|
971
|
+
case 'go-edit': ws.tab = 'edit'; ctx.render(); return;
|
|
972
|
+
case 'go-versions': ws.tab = 'versions'; ctx.render(); return;
|
|
973
|
+
case 'save-version': {
|
|
974
|
+
const { saveVersion, dirtyTabs } = await import('./sx-docs.js');
|
|
975
|
+
if (dirtyTabs(ws).length) { ctx.toast(ctx.t('skillsStudio.toast.saveFirst'), 'error'); return; }
|
|
976
|
+
try { await saveVersion(ws, ctx.api, { key: ctx.key }); ctx.toast(ctx.t('skillsStudio.toast.versionSaved')); st.suiteFor = null; await ctx.reload(); } catch (err) { ctx.toast(ctx.t('skillsStudio.toast.versionFailed', { error: testErrorText(ctx.t, err) }), 'error'); }
|
|
977
|
+
return;
|
|
978
|
+
}
|
|
979
|
+
case 'more': return;
|
|
980
|
+
default: return;
|
|
981
|
+
}
|
|
982
|
+
}, { signal });
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
function redrawHistorySoon(panel, ctx, ws) {
|
|
986
|
+
requestAnimationFrame(() => {
|
|
987
|
+
const st = stateOf(ws);
|
|
988
|
+
const btn = panel.querySelector('[data-test="compare"]');
|
|
989
|
+
if (btn) btn.disabled = st.compare.size !== 2;
|
|
990
|
+
for (const box of panel.querySelectorAll('[data-test="pick"]')) box.checked = st.compare.has(box.dataset.id);
|
|
991
|
+
const cmp = panel.querySelector('#sx-test-compare');
|
|
992
|
+
if (cmp && !st.comparison) setHTML(cmp, '');
|
|
993
|
+
});
|
|
994
|
+
}
|