session-orchestrator 3.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +29 -0
- package/.claude-plugin/plugin.json +18 -0
- package/.codex-plugin/agents/explorer.toml +14 -0
- package/.codex-plugin/agents/session-reviewer.toml +23 -0
- package/.codex-plugin/agents/wave-worker.toml +15 -0
- package/.codex-plugin/config.toml +20 -0
- package/.codex-plugin/plugin.json +37 -0
- package/.cursor/rules/000-session-orchestrator.mdc +73 -0
- package/.cursor/rules/010-session-workflow.mdc +170 -0
- package/.cursor/rules/020-quality-gates.mdc +128 -0
- package/.cursor/rules/030-wave-execution.mdc +216 -0
- package/.cursor/rules/040-discovery.mdc +242 -0
- package/.cursor/rules/050-plan.mdc +235 -0
- package/.cursor/rules/060-evolve.mdc +232 -0
- package/.cursor/rules/070-gitlab-ops.mdc +246 -0
- package/.cursor/rules/080-ecosystem-health.mdc +145 -0
- package/.mcp.json +8 -0
- package/CHANGELOG.md +1544 -0
- package/LICENSE +21 -0
- package/NOTICE +64 -0
- package/README.md +242 -0
- package/SECURITY.md +90 -0
- package/agents/AGENTS.md +136 -0
- package/agents/analyst.md +99 -0
- package/agents/architect-reviewer.md +93 -0
- package/agents/code-implementer.md +106 -0
- package/agents/db-specialist.md +104 -0
- package/agents/dialectic-deriver.md +139 -0
- package/agents/docs-writer.md +113 -0
- package/agents/eval-judge.md +146 -0
- package/agents/memory-proposal-collector.md +297 -0
- package/agents/qa-strategist.md +102 -0
- package/agents/schemas/analyst.schema.json +46 -0
- package/agents/schemas/architect-reviewer.schema.json +50 -0
- package/agents/schemas/code-implementer.schema.json +61 -0
- package/agents/schemas/db-specialist.schema.json +80 -0
- package/agents/schemas/docs-writer.schema.json +56 -0
- package/agents/schemas/persona-panel-sidecar.schema.json +245 -0
- package/agents/schemas/qa-strategist.schema.json +46 -0
- package/agents/schemas/security-reviewer.schema.json +86 -0
- package/agents/schemas/session-reviewer.schema.json +69 -0
- package/agents/schemas/test-writer.schema.json +69 -0
- package/agents/schemas/ui-developer.schema.json +90 -0
- package/agents/schemas/ux-evaluator.schema.json +51 -0
- package/agents/security-reviewer.md +236 -0
- package/agents/session-reviewer.md +201 -0
- package/agents/skill-applied-judge.md +122 -0
- package/agents/test-writer.md +123 -0
- package/agents/ui-developer.md +109 -0
- package/agents/ux-evaluator.md +161 -0
- package/assets/icon.svg +11 -0
- package/assets/og-card.png +0 -0
- package/assets/og-card.svg +47 -0
- package/commands/autopilot-multi.md +74 -0
- package/commands/autopilot.md +80 -0
- package/commands/bootstrap.md +56 -0
- package/commands/brainstorm.md +48 -0
- package/commands/close.md +24 -0
- package/commands/debug.md +36 -0
- package/commands/discovery.md +32 -0
- package/commands/dispatcher.md +59 -0
- package/commands/eval.md +28 -0
- package/commands/evolve.md +10 -0
- package/commands/go.md +41 -0
- package/commands/grill.md +45 -0
- package/commands/harness-audit.md +26 -0
- package/commands/memory-cleanup.md +25 -0
- package/commands/persona-panel.md +121 -0
- package/commands/plan.md +15 -0
- package/commands/portfolio.md +97 -0
- package/commands/reconcile.md +23 -0
- package/commands/repo-audit.md +24 -0
- package/commands/session.md +30 -0
- package/commands/spinout.md +15 -0
- package/commands/sunset-review.md +27 -0
- package/commands/templates-ack.md +96 -0
- package/commands/test.md +97 -0
- package/docs/README.md +105 -0
- package/docs/USER-GUIDE.md +1403 -0
- package/docs/ci-setup.md +81 -0
- package/docs/codex-setup.md +142 -0
- package/docs/components.md +74 -0
- package/docs/cursor-setup.md +104 -0
- package/docs/events-schema.md +81 -0
- package/docs/migration-v3.md +148 -0
- package/docs/owner-config-schema.md +154 -0
- package/docs/persona-panel.md +433 -0
- package/docs/pi-setup.md +115 -0
- package/docs/plugin-architecture-v3.md +296 -0
- package/docs/pm-skills-marketplace.md +114 -0
- package/docs/policy-cache-validation-2026-04-28.md +118 -0
- package/docs/rule-authoring.md +316 -0
- package/docs/session-config-reference.md +1439 -0
- package/docs/session-config-template.md +961 -0
- package/docs/vault-docs-architecture.md +297 -0
- package/hooks/_lib/lock-bootstrap.mjs +272 -0
- package/hooks/_lib/lock-reconcile.mjs +93 -0
- package/hooks/_lib/profile-gate.mjs +95 -0
- package/hooks/_lib/transcript-history.mjs +211 -0
- package/hooks/agent-teams-h3-test.sh +362 -0
- package/hooks/config-protection.mjs +0 -0
- package/hooks/cwd-change-restore.mjs +131 -0
- package/hooks/enforce-commands.mjs +179 -0
- package/hooks/enforce-scope.mjs +273 -0
- package/hooks/hooks-codex.json +60 -0
- package/hooks/hooks-cursor.json +15 -0
- package/hooks/hooks-pi.json +115 -0
- package/hooks/hooks.json +215 -0
- package/hooks/loop-guard.mjs +260 -0
- package/hooks/on-session-end.mjs +217 -0
- package/hooks/on-session-start.mjs +660 -0
- package/hooks/on-stop.mjs +294 -0
- package/hooks/operator-steer.mjs +64 -0
- package/hooks/post-edit-validate.mjs +225 -0
- package/hooks/post-subagent-discovery-validator.mjs +398 -0
- package/hooks/post-tool-batch-wave-signal.mjs +328 -0
- package/hooks/post-tool-failure-corrective-context.mjs +248 -0
- package/hooks/post-tooluse-frontend-slop.mjs +184 -0
- package/hooks/pre-bash-destructive-guard.mjs +515 -0
- package/hooks/pre-bash-memory-propose-audit.mjs +206 -0
- package/hooks/pre-bash-staging-fence.mjs +223 -0
- package/hooks/pre-bash-templates-first.mjs +404 -0
- package/hooks/run-node.sh +72 -0
- package/hooks/skill-invocation-telemetry.mjs +99 -0
- package/hooks/subagent-telemetry.mjs +249 -0
- package/hooks/wave-scope-commit-guard.mjs +191 -0
- package/monitors/monitors.json +14 -0
- package/output-styles/finding-report.md +48 -0
- package/output-styles/session-report.md +53 -0
- package/output-styles/wave-summary.md +38 -0
- package/package.json +94 -0
- package/pi/extensions/session-orchestrator.ts +25 -0
- package/pi/prompts/autopilot-multi.md +12 -0
- package/pi/prompts/autopilot.md +12 -0
- package/pi/prompts/bootstrap.md +12 -0
- package/pi/prompts/brainstorm.md +12 -0
- package/pi/prompts/close.md +11 -0
- package/pi/prompts/debug.md +12 -0
- package/pi/prompts/discovery.md +12 -0
- package/pi/prompts/dispatcher.md +12 -0
- package/pi/prompts/eval.md +12 -0
- package/pi/prompts/evolve.md +12 -0
- package/pi/prompts/go.md +12 -0
- package/pi/prompts/grill.md +12 -0
- package/pi/prompts/harness-audit.md +12 -0
- package/pi/prompts/memory-cleanup.md +12 -0
- package/pi/prompts/persona-panel.md +12 -0
- package/pi/prompts/plan.md +12 -0
- package/pi/prompts/portfolio.md +12 -0
- package/pi/prompts/reconcile.md +12 -0
- package/pi/prompts/repo-audit.md +12 -0
- package/pi/prompts/session.md +12 -0
- package/pi/prompts/spinout.md +12 -0
- package/pi/prompts/sunset-review.md +12 -0
- package/pi/prompts/templates-ack.md +12 -0
- package/pi/prompts/test.md +12 -0
- package/rules/_index.md +51 -0
- package/rules/always-on/commit-discipline.md +26 -0
- package/rules/always-on/npm-quality-gates.md +26 -0
- package/rules/always-on/parallel-sessions.md +43 -0
- package/rules/opt-in-domain/prompt-caching.md +270 -0
- package/rules/opt-in-stack/backend-data.md +188 -0
- package/rules/opt-in-stack/backend.md +390 -0
- package/rules/opt-in-stack/frontend.md +98 -0
- package/rules/opt-in-stack/security-web.md +194 -0
- package/rules/opt-in-stack/swift.md +65 -0
- package/scripts/archive-closed-prds.mjs +416 -0
- package/scripts/autopilot-multi.mjs +802 -0
- package/scripts/autopilot.mjs +383 -0
- package/scripts/backfill-abandoned-sessions.mjs +265 -0
- package/scripts/backfill-learnings-expires.mjs +196 -0
- package/scripts/backfill-learnings.mjs +203 -0
- package/scripts/backfill-sessions.mjs +282 -0
- package/scripts/check-doc-consistency.sh +279 -0
- package/scripts/check-package-manager.mjs +445 -0
- package/scripts/ci/assert-vitest-green.mjs +267 -0
- package/scripts/codex-install.mjs +435 -0
- package/scripts/compute-grounding-injection.sh +186 -0
- package/scripts/cursor-install.mjs +113 -0
- package/scripts/dialectic-deriver.mjs +573 -0
- package/scripts/emit-event.mjs +160 -0
- package/scripts/emit-session.mjs +212 -0
- package/scripts/eval-session.mjs +262 -0
- package/scripts/export-hw-learnings.mjs +437 -0
- package/scripts/gc-stale-worktrees.mjs +666 -0
- package/scripts/generate-pi-prompts.mjs +127 -0
- package/scripts/harness-audit.mjs +287 -0
- package/scripts/lib/agent-frontmatter.mjs +266 -0
- package/scripts/lib/agent-output-schema.mjs +166 -0
- package/scripts/lib/agent-status.mjs +303 -0
- package/scripts/lib/ajv-loader.mjs +34 -0
- package/scripts/lib/auto-dialectic.mjs +382 -0
- package/scripts/lib/auto-dream.mjs +471 -0
- package/scripts/lib/autonomy/suitability.mjs +212 -0
- package/scripts/lib/autopilot/dep-graph.mjs +417 -0
- package/scripts/lib/autopilot/durable-telemetry.mjs +121 -0
- package/scripts/lib/autopilot/flags.mjs +104 -0
- package/scripts/lib/autopilot/kill-switches.mjs +174 -0
- package/scripts/lib/autopilot/loop.mjs +320 -0
- package/scripts/lib/autopilot/mr-draft.mjs +520 -0
- package/scripts/lib/autopilot/multi-killswitch.mjs +184 -0
- package/scripts/lib/autopilot/recent-runs.mjs +106 -0
- package/scripts/lib/autopilot/stall-sampler.mjs +97 -0
- package/scripts/lib/autopilot/telemetry.mjs +224 -0
- package/scripts/lib/autopilot/worktree-pipeline.mjs +605 -0
- package/scripts/lib/autopilot-telemetry.mjs +11 -0
- package/scripts/lib/autopilot.mjs +39 -0
- package/scripts/lib/backlog-scan.mjs +179 -0
- package/scripts/lib/bootstrap-lock-freshness.mjs +260 -0
- package/scripts/lib/bootstrap-lock-refresh.mjs +186 -0
- package/scripts/lib/build-live-signals.mjs +150 -0
- package/scripts/lib/ci-status-banner.mjs +425 -0
- package/scripts/lib/claude-md-budget-lint.mjs +246 -0
- package/scripts/lib/cli-flags.mjs +158 -0
- package/scripts/lib/codex/plugin-contract.mjs +610 -0
- package/scripts/lib/cold-start-detector.mjs +240 -0
- package/scripts/lib/command-blocker.mjs +458 -0
- package/scripts/lib/common.mjs +333 -0
- package/scripts/lib/config/auto-dream.mjs +77 -0
- package/scripts/lib/config/block-header.mjs +94 -0
- package/scripts/lib/config/broken-window.mjs +114 -0
- package/scripts/lib/config/coercers.mjs +248 -0
- package/scripts/lib/config/cold-start.mjs +92 -0
- package/scripts/lib/config/config-protection.mjs +120 -0
- package/scripts/lib/config/cross-repo.mjs +104 -0
- package/scripts/lib/config/custom-phases.mjs +213 -0
- package/scripts/lib/config/dialectic.mjs +92 -0
- package/scripts/lib/config/discovery-validator.mjs +75 -0
- package/scripts/lib/config/dispatcher-autonomy-capture.mjs +240 -0
- package/scripts/lib/config/dispatcher-autonomy.mjs +152 -0
- package/scripts/lib/config/docs-orchestrator.mjs +90 -0
- package/scripts/lib/config/docs-staleness.mjs +96 -0
- package/scripts/lib/config/drift-check.mjs +155 -0
- package/scripts/lib/config/eval.mjs +130 -0
- package/scripts/lib/config/events-rotation.mjs +74 -0
- package/scripts/lib/config/evolve.mjs +308 -0
- package/scripts/lib/config/frontend-slop-hook.mjs +104 -0
- package/scripts/lib/config/gitlab-portfolio.mjs +150 -0
- package/scripts/lib/config/handover-gate.mjs +106 -0
- package/scripts/lib/config/host-paths.mjs +76 -0
- package/scripts/lib/config/io.mjs +54 -0
- package/scripts/lib/config/loop-guard.mjs +117 -0
- package/scripts/lib/config/memory.mjs +150 -0
- package/scripts/lib/config/persona-gate-wave.mjs +258 -0
- package/scripts/lib/config/reconcile.mjs +205 -0
- package/scripts/lib/config/section-extractor.mjs +100 -0
- package/scripts/lib/config/skill-evolution.mjs +112 -0
- package/scripts/lib/config/slopcheck.mjs +99 -0
- package/scripts/lib/config/state-md-lock.mjs +83 -0
- package/scripts/lib/config/templates-first.mjs +94 -0
- package/scripts/lib/config/test.mjs +113 -0
- package/scripts/lib/config/vault-integration.mjs +201 -0
- package/scripts/lib/config/vault-mirror-quality.mjs +99 -0
- package/scripts/lib/config/vault-staleness.mjs +84 -0
- package/scripts/lib/config/vault-sync.mjs +96 -0
- package/scripts/lib/config/verification-auto-fix.mjs +84 -0
- package/scripts/lib/config/wave-reviewers.mjs +133 -0
- package/scripts/lib/config-schema.mjs +345 -0
- package/scripts/lib/config.mjs +474 -0
- package/scripts/lib/convergence-monitor.mjs +389 -0
- package/scripts/lib/coordinator-snapshot.mjs +371 -0
- package/scripts/lib/crypto-digest-utils.mjs +91 -0
- package/scripts/lib/discovery/helpers.mjs +127 -0
- package/scripts/lib/discovery/triage-state.mjs +279 -0
- package/scripts/lib/dispatcher/cli.mjs +257 -0
- package/scripts/lib/dispatcher/enumerate.mjs +243 -0
- package/scripts/lib/dispatcher/rank.mjs +363 -0
- package/scripts/lib/ecosystem-health.mjs +224 -0
- package/scripts/lib/ecosystem-wizard/ci-detector.mjs +18 -0
- package/scripts/lib/ecosystem-wizard/config-parser.mjs +54 -0
- package/scripts/lib/ecosystem-wizard/config-writer.mjs +287 -0
- package/scripts/lib/ecosystem-wizard/package-manager-detector.mjs +42 -0
- package/scripts/lib/ecosystem-wizard/wizard-prompt.mjs +246 -0
- package/scripts/lib/ecosystem-wizard.mjs +48 -0
- package/scripts/lib/env-check.mjs +89 -0
- package/scripts/lib/eval/engine.mjs +605 -0
- package/scripts/lib/eval/judge.mjs +433 -0
- package/scripts/lib/eval/report.mjs +367 -0
- package/scripts/lib/eval/schema.mjs +618 -0
- package/scripts/lib/eval/session-resolve.mjs +137 -0
- package/scripts/lib/eval/sink.mjs +77 -0
- package/scripts/lib/events-rotation.mjs +86 -0
- package/scripts/lib/events-schema.mjs +81 -0
- package/scripts/lib/events.mjs +80 -0
- package/scripts/lib/evolve/autonomy-verdict.mjs +461 -0
- package/scripts/lib/evolve/autopilot-effectiveness.mjs +293 -0
- package/scripts/lib/exclusivity-matrix.mjs +68 -0
- package/scripts/lib/fetch-baseline.mjs +311 -0
- package/scripts/lib/file-lock.mjs +512 -0
- package/scripts/lib/frontend-detect/detect.mjs +138 -0
- package/scripts/lib/frontend-detect/rules.mjs +295 -0
- package/scripts/lib/frontmatter-guard.mjs +241 -0
- package/scripts/lib/gates/echo-stub-detect.mjs +39 -0
- package/scripts/lib/gates/gate-baseline.mjs +42 -0
- package/scripts/lib/gates/gate-full.mjs +85 -0
- package/scripts/lib/gates/gate-helpers.mjs +231 -0
- package/scripts/lib/gates/gate-incremental.mjs +76 -0
- package/scripts/lib/gates/gate-per-file.mjs +55 -0
- package/scripts/lib/gitlab-ops/stale-mr-sweep.mjs +447 -0
- package/scripts/lib/gitlab-portfolio/aggregator.mjs +383 -0
- package/scripts/lib/gitlab-portfolio/cli.mjs +428 -0
- package/scripts/lib/gitlab-portfolio/markdown-writer.mjs +289 -0
- package/scripts/lib/gitlab-portfolio/vcs-detect.mjs +182 -0
- package/scripts/lib/handover-gate.mjs +222 -0
- package/scripts/lib/hardening.mjs +43 -0
- package/scripts/lib/hardware-pattern-detector.mjs +238 -0
- package/scripts/lib/harness-audit/categories/category1.mjs +123 -0
- package/scripts/lib/harness-audit/categories/category2.mjs +145 -0
- package/scripts/lib/harness-audit/categories/category3.mjs +143 -0
- package/scripts/lib/harness-audit/categories/category4.mjs +202 -0
- package/scripts/lib/harness-audit/categories/category5.mjs +152 -0
- package/scripts/lib/harness-audit/categories/category6.mjs +211 -0
- package/scripts/lib/harness-audit/categories/category7.mjs +125 -0
- package/scripts/lib/harness-audit/categories/category8.mjs +328 -0
- package/scripts/lib/harness-audit/categories/category9.mjs +294 -0
- package/scripts/lib/harness-audit/categories/helpers.mjs +165 -0
- package/scripts/lib/harness-audit/categories.mjs +19 -0
- package/scripts/lib/historical-guard.mjs +15 -0
- package/scripts/lib/host-identity.mjs +262 -0
- package/scripts/lib/instruction-budget-guard.mjs +332 -0
- package/scripts/lib/io.mjs +304 -0
- package/scripts/lib/issue-close-strip-labels.mjs +161 -0
- package/scripts/lib/language-mappers/README.md +57 -0
- package/scripts/lib/language-mappers/index.mjs +165 -0
- package/scripts/lib/language-mappers/markdown.mjs +149 -0
- package/scripts/lib/language-mappers/python.mjs +249 -0
- package/scripts/lib/language-mappers/swift.mjs +201 -0
- package/scripts/lib/language-mappers/typescript.mjs +433 -0
- package/scripts/lib/learnings/expiry-sweep.mjs +164 -0
- package/scripts/lib/learnings/filters.mjs +43 -0
- package/scripts/lib/learnings/io.mjs +255 -0
- package/scripts/lib/learnings/schema.mjs +518 -0
- package/scripts/lib/learnings/surface.mjs +207 -0
- package/scripts/lib/learnings.mjs +42 -0
- package/scripts/lib/lock-reaper.mjs +648 -0
- package/scripts/lib/locks/index.mjs +31 -0
- package/scripts/lib/locks/lock-body.mjs +62 -0
- package/scripts/lib/locks/staging-fence-lock.mjs +267 -0
- package/scripts/lib/locks/state-md-lock.mjs +351 -0
- package/scripts/lib/loop-readiness-banner.mjs +144 -0
- package/scripts/lib/memory-banner.mjs +478 -0
- package/scripts/lib/memory-cleanup/worktree-sweep.mjs +108 -0
- package/scripts/lib/memory-cleanup-stamp.mjs +56 -0
- package/scripts/lib/memory-paths.mjs +31 -0
- package/scripts/lib/memory-proposals/collector.mjs +334 -0
- package/scripts/lib/memory-proposals/schema.mjs +289 -0
- package/scripts/lib/memory-proposals/sink.mjs +507 -0
- package/scripts/lib/memory-proposals/store.mjs +441 -0
- package/scripts/lib/mission-status-schema.mjs +114 -0
- package/scripts/lib/mode-selector/alternatives.mjs +64 -0
- package/scripts/lib/mode-selector/constants.mjs +29 -0
- package/scripts/lib/mode-selector/context-pressure.mjs +157 -0
- package/scripts/lib/mode-selector/rationale.mjs +55 -0
- package/scripts/lib/mode-selector/scoring.mjs +221 -0
- package/scripts/lib/mode-selector-accuracy.mjs +121 -0
- package/scripts/lib/mode-selector.mjs +160 -0
- package/scripts/lib/multi-provider-build/providers.mjs +64 -0
- package/scripts/lib/multi-provider-build/templating.mjs +130 -0
- package/scripts/lib/named-baseline-resolver.mjs +233 -0
- package/scripts/lib/named-vault-resolver.mjs +433 -0
- package/scripts/lib/owner-config/coerce.mjs +29 -0
- package/scripts/lib/owner-config/constants.mjs +21 -0
- package/scripts/lib/owner-config/defaults.mjs +50 -0
- package/scripts/lib/owner-config/error.mjs +19 -0
- package/scripts/lib/owner-config/index.mjs +13 -0
- package/scripts/lib/owner-config/merge.mjs +52 -0
- package/scripts/lib/owner-config/validate.mjs +259 -0
- package/scripts/lib/owner-config-banner.mjs +126 -0
- package/scripts/lib/owner-config-loader.mjs +159 -0
- package/scripts/lib/owner-config.example.yaml +72 -0
- package/scripts/lib/owner-config.mjs +28 -0
- package/scripts/lib/owner-interview.mjs +243 -0
- package/scripts/lib/owner-yaml.mjs +571 -0
- package/scripts/lib/package-manager.mjs +160 -0
- package/scripts/lib/path-utils.mjs +217 -0
- package/scripts/lib/peer-cards/merger.mjs +310 -0
- package/scripts/lib/peer-cards/reader.mjs +125 -0
- package/scripts/lib/peer-cards/schema.mjs +230 -0
- package/scripts/lib/peer-cards/staleness-banner.mjs +86 -0
- package/scripts/lib/peer-cards/writer.mjs +138 -0
- package/scripts/lib/peer-discovery.mjs +200 -0
- package/scripts/lib/persona-panel/catalog-loader.mjs +577 -0
- package/scripts/lib/persona-panel/consolidator.mjs +370 -0
- package/scripts/lib/persona-panel/persona-runner.mjs +375 -0
- package/scripts/lib/persona-panel/threshold.mjs +130 -0
- package/scripts/lib/pi-hook-bridge.mjs +328 -0
- package/scripts/lib/platform.mjs +266 -0
- package/scripts/lib/playwright-driver/runner.mjs +297 -0
- package/scripts/lib/plugin-root.mjs +210 -0
- package/scripts/lib/pre-dispatch-check.mjs +126 -0
- package/scripts/lib/product-repo-detect.mjs +121 -0
- package/scripts/lib/profiles/registry.mjs +176 -0
- package/scripts/lib/profiles/schema.mjs +209 -0
- package/scripts/lib/qg-command-drift-banner.mjs +88 -0
- package/scripts/lib/quality-gate/diagnostics.mjs +92 -0
- package/scripts/lib/quality-gate.mjs +536 -0
- package/scripts/lib/quality-gates-cache.mjs +228 -0
- package/scripts/lib/quality-gates-policy.mjs +95 -0
- package/scripts/lib/recommendations-v0.mjs +156 -0
- package/scripts/lib/reconcile/eligibility.mjs +203 -0
- package/scripts/lib/reconcile/emitter.mjs +244 -0
- package/scripts/lib/reconcile/engine.mjs +412 -0
- package/scripts/lib/reconcile/idempotency.mjs +239 -0
- package/scripts/lib/reconcile/renderer.mjs +211 -0
- package/scripts/lib/reconcile/writer.mjs +293 -0
- package/scripts/lib/reconcile-nudge-banner.mjs +284 -0
- package/scripts/lib/resource-probe/evaluate.mjs +190 -0
- package/scripts/lib/resource-probe/parsers.mjs +181 -0
- package/scripts/lib/resource-probe/probe-platform.mjs +300 -0
- package/scripts/lib/resource-probe.mjs +95 -0
- package/scripts/lib/rule-loader.mjs +552 -0
- package/scripts/lib/rules-sync.mjs +439 -0
- package/scripts/lib/scope-gate.mjs +496 -0
- package/scripts/lib/session-close-backfill.mjs +539 -0
- package/scripts/lib/session-discovery.mjs +256 -0
- package/scripts/lib/session-end/phase-skip.mjs +357 -0
- package/scripts/lib/session-end/worktree-cleanup.mjs +112 -0
- package/scripts/lib/session-id.mjs +362 -0
- package/scripts/lib/session-lock.mjs +703 -0
- package/scripts/lib/session-registry.mjs +355 -0
- package/scripts/lib/session-schema/aliases.mjs +71 -0
- package/scripts/lib/session-schema/constants.mjs +115 -0
- package/scripts/lib/session-schema/normalizer.mjs +66 -0
- package/scripts/lib/session-schema/timestamps.mjs +64 -0
- package/scripts/lib/session-schema/validator.mjs +453 -0
- package/scripts/lib/session-schema.mjs +71 -0
- package/scripts/lib/session-token-rollup.mjs +137 -0
- package/scripts/lib/sessions-staleness-banner.mjs +247 -0
- package/scripts/lib/skill-evolution/blast-radius-classifier.mjs +114 -0
- package/scripts/lib/skill-evolution/candidate-intake.mjs +270 -0
- package/scripts/lib/skill-evolution/config-validation-gate.mjs +279 -0
- package/scripts/lib/skill-evolution/engine.mjs +719 -0
- package/scripts/lib/skill-evolution/idempotency.mjs +279 -0
- package/scripts/lib/skill-evolution/mr-opener.mjs +507 -0
- package/scripts/lib/skill-health/join.mjs +181 -0
- package/scripts/lib/skill-health/score.mjs +123 -0
- package/scripts/lib/skill-invocations-schema.mjs +214 -0
- package/scripts/lib/skill-judge.mjs +348 -0
- package/scripts/lib/skill-judgments-schema.mjs +264 -0
- package/scripts/lib/slopcheck.mjs +501 -0
- package/scripts/lib/soul-resolve.mjs +118 -0
- package/scripts/lib/spiral-carryover.mjs +495 -0
- package/scripts/lib/state-md/body-sections.mjs +851 -0
- package/scripts/lib/state-md/frontmatter-mutators.mjs +453 -0
- package/scripts/lib/state-md/mission-status.mjs +247 -0
- package/scripts/lib/state-md/recommendations.mjs +57 -0
- package/scripts/lib/state-md/yaml-parser.mjs +234 -0
- package/scripts/lib/state-md-peer-guard.mjs +232 -0
- package/scripts/lib/state-md.mjs +53 -0
- package/scripts/lib/subagents-schema.mjs +309 -0
- package/scripts/lib/sunset/walker.mjs +1192 -0
- package/scripts/lib/test-runner/artifact-paths.mjs +94 -0
- package/scripts/lib/test-runner/fingerprint.mjs +33 -0
- package/scripts/lib/test-runner/issue-reconcile.mjs +770 -0
- package/scripts/lib/tmux-layout/layouts.mjs +224 -0
- package/scripts/lib/tmux-layout/telemetry-stats.mjs +100 -0
- package/scripts/lib/tmux-layout/telemetry.mjs +88 -0
- package/scripts/lib/tmux-layout/tmux-shell.mjs +82 -0
- package/scripts/lib/tmux-layout/vcs-detector.mjs +88 -0
- package/scripts/lib/validate/check-agents.mjs +457 -0
- package/scripts/lib/validate/check-codex-plugin.mjs +37 -0
- package/scripts/lib/validate/check-commands.mjs +148 -0
- package/scripts/lib/validate/check-component-paths.mjs +112 -0
- package/scripts/lib/validate/check-dead-bridge.mjs +180 -0
- package/scripts/lib/validate/check-hooks-symmetry.mjs +258 -0
- package/scripts/lib/validate/check-json-files.mjs +116 -0
- package/scripts/lib/validate/check-owner-leakage.mjs +1011 -0
- package/scripts/lib/validate/check-path-utils-canary.mjs +175 -0
- package/scripts/lib/validate/check-peekaboo-driver-canary.mjs +201 -0
- package/scripts/lib/validate/check-pi-package.mjs +110 -0
- package/scripts/lib/validate/check-pi-prompts.mjs +43 -0
- package/scripts/lib/validate/check-playwright-mcp-canary.mjs +154 -0
- package/scripts/lib/validate/check-plugin-json.mjs +96 -0
- package/scripts/lib/validate/check-plugin-monitors.mjs +206 -0
- package/scripts/lib/validate/check-plugin-schema.mjs +137 -0
- package/scripts/lib/validate/check-rules.mjs +143 -0
- package/scripts/lib/validate/check-session-plan-routing.mjs +154 -0
- package/scripts/lib/validate/check-test-fixture-shapes.mjs +280 -0
- package/scripts/lib/validate/check-unicode-safety.mjs +533 -0
- package/scripts/lib/validate/confidential-names.mjs +169 -0
- package/scripts/lib/validate/dead-bridge-corpus.mjs +141 -0
- package/scripts/lib/validate/dead-bridge-detectors.mjs +568 -0
- package/scripts/lib/validate/tier-inference.mjs +100 -0
- package/scripts/lib/validate-vendored-rules.mjs +519 -0
- package/scripts/lib/vault-archive.mjs +404 -0
- package/scripts/lib/vault-backfill/glab.mjs +164 -0
- package/scripts/lib/vault-backfill/manifest.mjs +75 -0
- package/scripts/lib/vault-backfill/template.mjs +130 -0
- package/scripts/lib/vault-consolidate-fs.mjs +331 -0
- package/scripts/lib/vault-migration-rules.mjs +155 -0
- package/scripts/lib/vault-mirror/auto-commit.mjs +203 -0
- package/scripts/lib/vault-mirror/namespace.mjs +152 -0
- package/scripts/lib/vault-mirror/process.mjs +567 -0
- package/scripts/lib/vault-mirror/pseudonym-map.mjs +164 -0
- package/scripts/lib/vault-mirror/render-learnings.mjs +201 -0
- package/scripts/lib/vault-mirror/render-sessions.mjs +367 -0
- package/scripts/lib/vault-mirror/render.mjs +8 -0
- package/scripts/lib/vault-mirror/utils.mjs +217 -0
- package/scripts/lib/vault-relocation-rules.mjs +555 -0
- package/scripts/lib/vault-repo-backfill.mjs +235 -0
- package/scripts/lib/vault-staleness-banner.mjs +142 -0
- package/scripts/lib/vault-status/board-writer.mjs +769 -0
- package/scripts/lib/vault-status/narrative-mirror.mjs +544 -0
- package/scripts/lib/vault-sync-baseline.mjs +152 -0
- package/scripts/lib/wave-context.mjs +29 -0
- package/scripts/lib/wave-executor/pool.mjs +248 -0
- package/scripts/lib/wave-resource-gate.mjs +204 -0
- package/scripts/lib/wave-sizing.mjs +75 -0
- package/scripts/lib/webhook-url.mjs +105 -0
- package/scripts/lib/workspace.mjs +198 -0
- package/scripts/lib/worktree/constants.mjs +35 -0
- package/scripts/lib/worktree/index.mjs +17 -0
- package/scripts/lib/worktree/lifecycle.mjs +287 -0
- package/scripts/lib/worktree/listing.mjs +118 -0
- package/scripts/lib/worktree/meta.mjs +64 -0
- package/scripts/lib/worktree-freshness.mjs +313 -0
- package/scripts/lib/worktree.mjs +15 -0
- package/scripts/lifecycle-sim-v6.mjs +347 -0
- package/scripts/lock-reaper.mjs +185 -0
- package/scripts/mcp-server.sh +241 -0
- package/scripts/measure-policy-cache-effectiveness.mjs +427 -0
- package/scripts/memory-propose.mjs +464 -0
- package/scripts/migrate-cold-start-seed.mjs +404 -0
- package/scripts/migrate-learnings-jsonl.mjs +189 -0
- package/scripts/migrate-legacy-learnings.sh +61 -0
- package/scripts/migrate-sessions-jsonl.mjs +448 -0
- package/scripts/migrate-subagents-jsonl.mjs +196 -0
- package/scripts/migrate-vault-paths.mjs +796 -0
- package/scripts/parse-config.mjs +149 -0
- package/scripts/pi-install.mjs +117 -0
- package/scripts/print-applicable-rules.mjs +247 -0
- package/scripts/promote-vault-strict.mjs +496 -0
- package/scripts/relocate-vault-corpus.mjs +1178 -0
- package/scripts/run-migrate-v2-cross-repo.mjs +385 -0
- package/scripts/run-quality-gate.mjs +216 -0
- package/scripts/spikes/h3-agent-teams/preflight.sh +53 -0
- package/scripts/spikes/h3-agent-teams/run-h3.sh +112 -0
- package/scripts/spikes/h3-agent-teams/setup.sh +137 -0
- package/scripts/spikes/h3-agent-teams/toggle.sh +38 -0
- package/scripts/sweep-expired-learnings.mjs +135 -0
- package/scripts/sync-vault-schema.mjs +376 -0
- package/scripts/tests/fixtures/fetch-baseline/sample-rule.md +8 -0
- package/scripts/tmux-layout.mjs +245 -0
- package/scripts/token-audit.sh +191 -0
- package/scripts/typecheck.mjs +42 -0
- package/scripts/upload-social-preview.mjs +316 -0
- package/scripts/validate-config.mjs +46 -0
- package/scripts/validate-plugin-manifests.mjs +163 -0
- package/scripts/validate-plugin.mjs +264 -0
- package/scripts/validate-wave-scope.mjs +289 -0
- package/scripts/vault-backfill.mjs +404 -0
- package/scripts/vault-consolidate.mjs +596 -0
- package/scripts/vault-integration-watcher.mjs +394 -0
- package/scripts/vault-mirror.mjs +430 -0
- package/skills/_shared/bootstrap-gate.md +111 -0
- package/skills/_shared/config-reading.md +226 -0
- package/skills/_shared/instruction-file-resolution.md +79 -0
- package/skills/_shared/model-selection.md +64 -0
- package/skills/_shared/monitor-patterns.md +300 -0
- package/skills/_shared/parallel-aware-auq.md +121 -0
- package/skills/_shared/parallel-aware-preamble.md +185 -0
- package/skills/_shared/platform-tools.md +96 -0
- package/skills/_shared/state-ownership.md +221 -0
- package/skills/architecture/DEEPENING.md +37 -0
- package/skills/architecture/INTERFACE-DESIGN.md +44 -0
- package/skills/architecture/LANGUAGE.md +53 -0
- package/skills/architecture/SKILL.md +92 -0
- package/skills/autopilot/SKILL.md +419 -0
- package/skills/bootstrap/SKILL.md +592 -0
- package/skills/bootstrap/STATE.md.template +24 -0
- package/skills/bootstrap/_shared-template.md +243 -0
- package/skills/bootstrap/deep-template.md +659 -0
- package/skills/bootstrap/fast-template.md +251 -0
- package/skills/bootstrap/intensity-heuristic.md +80 -0
- package/skills/bootstrap/public-fallback.md +342 -0
- package/skills/bootstrap/standard-template.md +736 -0
- package/skills/bootstrap/templates/agents/project-code-review.md +18 -0
- package/skills/bootstrap/templates/agents/project-discovery.md +18 -0
- package/skills/bootstrap/templates/agents/project-quality-gate.md +18 -0
- package/skills/brainstorm/SKILL.md +268 -0
- package/skills/brainstorm/soul.md +49 -0
- package/skills/claude-md-drift-check/SKILL.md +186 -0
- package/skills/claude-md-drift-check/checker.mjs +1380 -0
- package/skills/claude-md-drift-check/checker.sh +37 -0
- package/skills/claude-md-drift-check/package.json +16 -0
- package/skills/convergence-monitoring/README.md +39 -0
- package/skills/convergence-monitoring/SIGNALS.md +246 -0
- package/skills/convergence-monitoring/SKILL.md +285 -0
- package/skills/daily/SKILL.md +222 -0
- package/skills/daily/generate.sh +92 -0
- package/skills/daily/templates/daily.md.tpl +36 -0
- package/skills/debug/SKILL.md +188 -0
- package/skills/debug/soul.md +35 -0
- package/skills/discovery/SKILL.md +567 -0
- package/skills/discovery/issue-templates.md +237 -0
- package/skills/discovery/probes/docs-staleness.mjs +195 -0
- package/skills/discovery/probes/frontend-slop.mjs +186 -0
- package/skills/discovery/probes/ssot-code-diff.mjs +310 -0
- package/skills/discovery/probes/supply-chain-slopcheck.mjs +440 -0
- package/skills/discovery/probes/vault-narrative-staleness.mjs +355 -0
- package/skills/discovery/probes/vault-staleness.mjs +272 -0
- package/skills/discovery/probes-arch.md +252 -0
- package/skills/discovery/probes-audit.md +95 -0
- package/skills/discovery/probes-code.md +329 -0
- package/skills/discovery/probes-docs.md +76 -0
- package/skills/discovery/probes-feature.md +150 -0
- package/skills/discovery/probes-infra.md +138 -0
- package/skills/discovery/probes-intro.md +25 -0
- package/skills/discovery/probes-session.md +495 -0
- package/skills/discovery/probes-supply-chain.md +94 -0
- package/skills/discovery/probes-ui.md +147 -0
- package/skills/discovery/probes-vault.md +64 -0
- package/skills/discovery/slop-patterns.md +115 -0
- package/skills/dispatcher/SKILL.md +173 -0
- package/skills/docs-orchestrator/SKILL.md +362 -0
- package/skills/docs-orchestrator/audience-mapping.md +140 -0
- package/skills/domain-model/ADR-FORMAT.md +47 -0
- package/skills/domain-model/CONTEXT-FORMAT.md +77 -0
- package/skills/domain-model/SKILL.md +85 -0
- package/skills/ecosystem-health/SKILL.md +119 -0
- package/skills/ecosystem-health/wizard.md +193 -0
- package/skills/eval/SKILL.md +293 -0
- package/skills/eval/rubric-v1.md +218 -0
- package/skills/evolve/SKILL.md +546 -0
- package/skills/frontmatter-guard/SKILL.md +126 -0
- package/skills/gitlab-ops/SKILL.md +368 -0
- package/skills/gitlab-portfolio/SKILL.md +196 -0
- package/skills/grill/SKILL.md +185 -0
- package/skills/grill/soul.md +55 -0
- package/skills/hook-development/SKILL.md +413 -0
- package/skills/mcp-builder/SKILL.md +260 -0
- package/skills/memory-cleanup/SKILL.md +310 -0
- package/skills/mode-selector/SKILL.md +226 -0
- package/skills/peekaboo-driver/SKILL.md +237 -0
- package/skills/peekaboo-driver/soul.md +32 -0
- package/skills/persona-panel/SKILL.md +365 -0
- package/skills/persona-panel/persona-format.md +205 -0
- package/skills/persona-panel/presets/designer-lens.md +87 -0
- package/skills/persona-panel/presets/engineer-lens.md +88 -0
- package/skills/persona-panel/presets/pm-lens.md +86 -0
- package/skills/plan/SKILL.md +496 -0
- package/skills/plan/mode-feature.md +141 -0
- package/skills/plan/mode-new.md +297 -0
- package/skills/plan/mode-retro.md +271 -0
- package/skills/plan/prd-feature-template.md +132 -0
- package/skills/plan/prd-full-template.md +151 -0
- package/skills/plan/prd-reviewer-prompt.md +103 -0
- package/skills/plan/retro-template.md +75 -0
- package/skills/plan/soul.md +62 -0
- package/skills/playwright-driver/SKILL.md +226 -0
- package/skills/playwright-driver/soul.md +30 -0
- package/skills/quality-gates/SKILL.md +212 -0
- package/skills/reconcile/SKILL.md +324 -0
- package/skills/repo-audit/SKILL.md +272 -0
- package/skills/session-end/SKILL.md +1044 -0
- package/skills/session-end/discovery-scan.md +37 -0
- package/skills/session-end/drift-operations.md +97 -0
- package/skills/session-end/learning-patterns.md +78 -0
- package/skills/session-end/metrics-collection.md +175 -0
- package/skills/session-end/phase-3-2-docs-verification.md +148 -0
- package/skills/session-end/phase-3-6-tail.md +344 -0
- package/skills/session-end/phase-3-7a-recommendations.md +86 -0
- package/skills/session-end/plan-verification.md +288 -0
- package/skills/session-end/session-metrics-write.md +223 -0
- package/skills/session-end/vault-operations.md +50 -0
- package/skills/session-end/verification-checklist.md +20 -0
- package/skills/session-plan/SKILL.md +554 -0
- package/skills/session-plan/wave-template.md +37 -0
- package/skills/session-start/SKILL.md +1043 -0
- package/skills/session-start/phase-2-5-docs-planning.md +119 -0
- package/skills/session-start/phase-4-5-resource-health.md +49 -0
- package/skills/session-start/phase-7-1-premise-check.md +47 -0
- package/skills/session-start/phase-7-5-mode-selector.md +237 -0
- package/skills/session-start/phase-8-5-express-path.md +61 -0
- package/skills/session-start/presentation-format.md +81 -0
- package/skills/session-start/soul.md +57 -0
- package/skills/skill-creator/SKILL.md +168 -0
- package/skills/spinout/SKILL.md +76 -0
- package/skills/sunset-review/SKILL.md +96 -0
- package/skills/test-runner/SKILL.md +362 -0
- package/skills/test-runner/rubric-v1.md +388 -0
- package/skills/test-runner/soul.md +46 -0
- package/skills/tmux-layout/SKILL.md +104 -0
- package/skills/ubiquitous-language/SKILL.md +97 -0
- package/skills/using-orchestrator/SKILL.md +144 -0
- package/skills/vault-mirror/SKILL.md +234 -0
- package/skills/vault-sync/SKILL.md +319 -0
- package/skills/vault-sync/package-lock.json +40 -0
- package/skills/vault-sync/package.json +11 -0
- package/skills/vault-sync/tests/fixtures/archive-test-vault/90-archive/bad-archived.md +8 -0
- package/skills/vault-sync/tests/fixtures/archive-test-vault/_meta/.gitkeep +0 -0
- package/skills/vault-sync/tests/fixtures/archive-test-vault/live-note.md +8 -0
- package/skills/vault-sync/tests/fixtures/broken-frontmatter-vault/_meta/.gitkeep +0 -0
- package/skills/vault-sync/tests/fixtures/broken-frontmatter-vault/bad-type.md +8 -0
- package/skills/vault-sync/tests/fixtures/broken-frontmatter-vault/good-note.md +8 -0
- package/skills/vault-sync/tests/fixtures/clean-vault/.obsidian/config.md +8 -0
- package/skills/vault-sync/tests/fixtures/clean-vault/01-projects/foo/projects-baseline.md +10 -0
- package/skills/vault-sync/tests/fixtures/clean-vault/03-daily/daily-2026-04-13.md +8 -0
- package/skills/vault-sync/tests/fixtures/clean-vault/README.md +3 -0
- package/skills/vault-sync/tests/fixtures/clean-vault/hello-world.md +11 -0
- package/skills/vault-sync/tests/fixtures/dangling-link-vault/_meta/.gitkeep +0 -0
- package/skills/vault-sync/tests/fixtures/dangling-link-vault/has-dangling.md +9 -0
- package/skills/vault-sync/tests/fixtures/dangling-link-vault/real-target.md +8 -0
- package/skills/vault-sync/tests/fixtures/empty-vault/_meta/.gitkeep +0 -0
- package/skills/vault-sync/tests/fixtures/missing-field-vault/_meta/.gitkeep +0 -0
- package/skills/vault-sync/tests/fixtures/missing-field-vault/missing-id.md +7 -0
- package/skills/vault-sync/tests/fixtures/nested-tag-vault/03-daily/daily-2026-04-13.md +9 -0
- package/skills/vault-sync/tests/fixtures/nested-tag-vault/_meta/.gitkeep +0 -0
- package/skills/vault-sync/tests/fixtures/nested-tag-vault/nested-tags-note.md +11 -0
- package/skills/vault-sync/tests/fixtures/no-frontmatter-vault/README.md +3 -0
- package/skills/vault-sync/tests/fixtures/no-frontmatter-vault/_MOC.md +3 -0
- package/skills/vault-sync/tests/fixtures/no-frontmatter-vault/_meta/.gitkeep +0 -0
- package/skills/vault-sync/tests/fixtures/with-moc-vault/_MOC.md +11 -0
- package/skills/vault-sync/tests/fixtures/with-moc-vault/_meta/.gitkeep +0 -0
- package/skills/vault-sync/tests/fixtures/with-moc-vault/hello-world.md +11 -0
- package/skills/vault-sync/tests/schema-drift.test.mjs +133 -0
- package/skills/vault-sync/validator.mjs +658 -0
- package/skills/vault-sync/validator.sh +55 -0
- package/skills/wave-executor/SKILL.md +496 -0
- package/skills/wave-executor/circuit-breaker.md +169 -0
- package/skills/wave-executor/wave-loop.md +1043 -0
- package/skills/write-executable-plan/SKILL.md +237 -0
- package/skills/write-executable-plan/plan-template.md +154 -0
- package/templates/_minimal/CLAUDE.md.tmpl +41 -0
- package/templates/_minimal/README.md.tmpl +15 -0
- package/templates/_minimal/gitignore.tmpl +47 -0
- package/templates/_shared/harte-regeln.md +16 -0
- package/templates/_shared/loop.md +90 -0
- package/templates/_shared/rules/parallel-sessions.md +77 -0
- package/templates/nextjs-minimal/README.md +30 -0
- package/templates/nextjs-minimal/app/layout.tsx +18 -0
- package/templates/nextjs-minimal/app/page.tsx +7 -0
- package/templates/nextjs-minimal/eslint.config.mjs +16 -0
- package/templates/nextjs-minimal/next.config.mjs +4 -0
- package/templates/nextjs-minimal/package.json +27 -0
- package/templates/nextjs-minimal/tsconfig.json +23 -0
- package/templates/node-minimal/README.md +33 -0
- package/templates/node-minimal/eslint.config.mjs +10 -0
- package/templates/node-minimal/package.json +21 -0
- package/templates/node-minimal/src/index.ts +1 -0
- package/templates/node-minimal/tests/sanity.test.ts +5 -0
- package/templates/node-minimal/tsconfig.json +17 -0
- package/templates/personas/README.md +150 -0
- package/templates/personas/accounting-compliance.v1.md +120 -0
- package/templates/personas/accounting-tax-advisor.v1.md +116 -0
- package/templates/personas/buyer-p1-cto.v1.md +125 -0
- package/templates/personas/buyer-p2-kanzlei.v1.md +134 -0
- package/templates/personas/buyer-p3-build.v1.md +130 -0
- package/templates/personas/buyer-p4-tech-veto.v1.md +130 -0
- package/templates/personas/buyer-p5-solo.v1.md +132 -0
- package/templates/personas/buyer-p6-ld.v1.md +130 -0
- package/templates/personas/klima-ai-expert.v1.md +114 -0
- package/templates/personas/klima-physicist.v1.md +117 -0
- package/templates/python-uv/README.md +28 -0
- package/templates/python-uv/pyproject.toml +32 -0
- package/templates/python-uv/src/__PROJECT_NAME__/__init__.py +0 -0
- package/templates/python-uv/src/__PROJECT_NAME__/main.py +6 -0
- package/templates/python-uv/tests/test_sanity.py +2 -0
- package/templates/static-html/README.md +19 -0
- package/templates/static-html/index.html +15 -0
- package/templates/static-html/script.js +1 -0
- package/templates/static-html/styles.css +26 -0
|
@@ -0,0 +1,618 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* eval/schema.mjs — canonical record-schema contract for the aiat-llm-eval
|
|
3
|
+
* standard (Epic #803, S2). The single source of truth every other eval module
|
|
4
|
+
* (engine, report renderer, session-end phase, judge) builds against.
|
|
5
|
+
*
|
|
6
|
+
* Zod-free plain-JS validator + normalizer, matching the repo convention for
|
|
7
|
+
* script-layer utilities (mirrors scripts/lib/skill-judgments-schema.mjs and
|
|
8
|
+
* scripts/lib/session-schema/*). No top-level side effects, no I/O — the sink
|
|
9
|
+
* (eval/sink.mjs) owns the append/read path.
|
|
10
|
+
*
|
|
11
|
+
* ── CANONICAL RECORD (schema_version: 1, record_kind: "session-eval") ────────
|
|
12
|
+
*
|
|
13
|
+
* A session-eval record scores one orchestrator session against a pre-registered
|
|
14
|
+
* rubric. Written append-only to .orchestrator/metrics/eval.jsonl (the journal is
|
|
15
|
+
* the Single Source of Truth; the HTML report is a derived, rebuildable view).
|
|
16
|
+
*
|
|
17
|
+
* schema_version 1 (stamped by validateEvalRecord if absent)
|
|
18
|
+
* record_kind 'session-eval' — discriminator; future benchmark records
|
|
19
|
+
* reuse this stream with a different record_kind.
|
|
20
|
+
* run_id string (unique). Format: `<session_id>-eval-<compactISO>`
|
|
21
|
+
* where compactISO strips `- : .` from the ISO timestamp,
|
|
22
|
+
* e.g. main-2026-07-16-deep-1-eval-20260716T100000000Z.
|
|
23
|
+
* Build via buildRunId(sessionId, timestamp) — deterministic,
|
|
24
|
+
* no Date.now (determinism is load-bearing for --verify).
|
|
25
|
+
* session_id string (non-empty) — the session being evaluated.
|
|
26
|
+
* standard_version string — CURRENT_STANDARD_VERSION ('aiat-llm-eval/1.0').
|
|
27
|
+
* rubric_version string (non-empty) — e.g. 'rubric-v1'.
|
|
28
|
+
* provenance { rubric_sha256: string, engine_commit: string|null }
|
|
29
|
+
* Hash-bound drift detection. The ENGINE computes the hash
|
|
30
|
+
* and the commit; this module only validates their shape.
|
|
31
|
+
* model { id: string, source: 'self-report'|'env'|'config' }
|
|
32
|
+
* Honest model-capture provenance (PRD §4 Modell-Capture).
|
|
33
|
+
* harness { plugin_version: string, platform: string,
|
|
34
|
+
* host_class: string|null, hostname_hash: string|null }
|
|
35
|
+
* hostname_hash is ALWAYS a sha256 short-form hex string —
|
|
36
|
+
* NEVER a cleartext hostname (validated via HEX_RE below).
|
|
37
|
+
* kpis { duration_seconds, total_waves, total_agents,
|
|
38
|
+
* token_input, token_output, carryover } — each field is
|
|
39
|
+
* number|null. "Don't fake perfect": a missing KPI is null,
|
|
40
|
+
* NEVER guessed as 0. normalizeEvalRecord fills undefined→null.
|
|
41
|
+
* dimensions array of per-dimension results, see DIMENSION contract.
|
|
42
|
+
* handle string|null — optional self-chosen pseudonym; default null.
|
|
43
|
+
* anonymized boolean — true = anonymous submission posture.
|
|
44
|
+
* timestamp ISO 8601 string — passed as a PARAMETER (no Date.now in the
|
|
45
|
+
* validator path, so re-verify stays byte-deterministic).
|
|
46
|
+
*
|
|
47
|
+
* DIMENSION (element of dimensions[]):
|
|
48
|
+
* id string (non-empty).
|
|
49
|
+
* method 'deterministic' | 'judge'.
|
|
50
|
+
* status 'pass' | 'fail' | 'not-applicable' | 'cannot-determine'.
|
|
51
|
+
* evidence string — visible justification (may embed paths/prompts;
|
|
52
|
+
* therefore EXCLUDED from SUBMISSION_FIELDS, see below).
|
|
53
|
+
* score (optional) number|null.
|
|
54
|
+
* advisory (judge only) boolean true — the load-bearing firewall:
|
|
55
|
+
* a judge dimension can NEVER be persisted as advisory:false
|
|
56
|
+
* (mirrors skill-judgments-schema advisory:true guarantee).
|
|
57
|
+
* calibration_status(judge only) 'uncalibrated' — v1 judge is never calibrated.
|
|
58
|
+
* Deterministic dimensions MUST NOT carry advisory / calibration_status.
|
|
59
|
+
*
|
|
60
|
+
* ── NO GLOBALSCORE, BY CONSTRUCTION ──────────────────────────────────────────
|
|
61
|
+
*
|
|
62
|
+
* validateEvalRecord REJECTS any record carrying a top-level overall / total /
|
|
63
|
+
* mean / global_score key (FORBIDDEN_GLOBALSCORE_KEYS). Aggregation across many
|
|
64
|
+
* records (CIs, trends, rankings) is a downstream concern, never a single-record
|
|
65
|
+
* field — a core promise of the standard ("kein Globalscore per Konstruktion").
|
|
66
|
+
*
|
|
67
|
+
* ── DATA-MINIMIZATION (SUBMISSION_FIELDS / projectSubmission) ─────────────────
|
|
68
|
+
*
|
|
69
|
+
* SUBMISSION_FIELDS is the frozen whitelist of submission-safe fields for the
|
|
70
|
+
* (future) leaderboard. projectSubmission(record) applies it nested-aware:
|
|
71
|
+
* anything not on the whitelist — file paths, prompts, repo names, dimension
|
|
72
|
+
* evidence, unhashed hostnames, rogue extra fields — is dropped. This is the
|
|
73
|
+
* contract the Leaderboard-Epic consumes.
|
|
74
|
+
*
|
|
75
|
+
* ── CONTRACT SUMMARY ─────────────────────────────────────────────────────────
|
|
76
|
+
*
|
|
77
|
+
* validateEvalRecord(entry) — throws ValidationError on any violation; returns
|
|
78
|
+
* a NEW object with schema_version stamped.
|
|
79
|
+
* normalizeEvalRecord(entry) — never throws; applies read-path defaults
|
|
80
|
+
* (handle→null, kpi undefined→null, judge-dim
|
|
81
|
+
* advisory→true / calibration_status→uncalibrated).
|
|
82
|
+
* buildRunId(sessionId, ts) — deterministic run-id builder.
|
|
83
|
+
* projectSubmission(record) — nested-aware whitelist projection.
|
|
84
|
+
*/
|
|
85
|
+
|
|
86
|
+
// ---------------------------------------------------------------------------
|
|
87
|
+
// Constants
|
|
88
|
+
// ---------------------------------------------------------------------------
|
|
89
|
+
|
|
90
|
+
/** Current eval-record schema version. */
|
|
91
|
+
export const CURRENT_EVAL_SCHEMA_VERSION = 1;
|
|
92
|
+
|
|
93
|
+
/** Versioned standard identifier stamped on every record. */
|
|
94
|
+
export const CURRENT_STANDARD_VERSION = 'aiat-llm-eval/1.0';
|
|
95
|
+
|
|
96
|
+
/** Allowed record_kind discriminator values. v1: session-eval only. */
|
|
97
|
+
export const VALID_RECORD_KINDS = Object.freeze(['session-eval']);
|
|
98
|
+
|
|
99
|
+
/** Provenance of the captured model id. */
|
|
100
|
+
export const VALID_MODEL_SOURCES = Object.freeze(['self-report', 'env', 'config']);
|
|
101
|
+
|
|
102
|
+
/** Per-dimension scoring method. */
|
|
103
|
+
export const VALID_DIMENSION_METHODS = Object.freeze(['deterministic', 'judge']);
|
|
104
|
+
|
|
105
|
+
/** 3-state verdict + abstention (never guess) for a dimension. */
|
|
106
|
+
export const VALID_DIMENSION_STATUSES = Object.freeze([
|
|
107
|
+
'pass',
|
|
108
|
+
'fail',
|
|
109
|
+
'not-applicable',
|
|
110
|
+
'cannot-determine',
|
|
111
|
+
]);
|
|
112
|
+
|
|
113
|
+
/** Judge calibration status. v1: uncalibrated only (κ-calibration is a later stage). */
|
|
114
|
+
export const VALID_CALIBRATION_STATUSES = Object.freeze(['uncalibrated']);
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Top-level keys that a valid record must NOT carry — the standard forbids any
|
|
118
|
+
* aggregated global score inside a single record.
|
|
119
|
+
*/
|
|
120
|
+
export const FORBIDDEN_GLOBALSCORE_KEYS = Object.freeze([
|
|
121
|
+
'overall',
|
|
122
|
+
'total',
|
|
123
|
+
'mean',
|
|
124
|
+
'global_score',
|
|
125
|
+
]);
|
|
126
|
+
|
|
127
|
+
/** KPI sub-fields. Each is number|null (undefined→null via normalize). */
|
|
128
|
+
export const KPI_FIELDS = Object.freeze([
|
|
129
|
+
'duration_seconds',
|
|
130
|
+
'total_waves',
|
|
131
|
+
'total_agents',
|
|
132
|
+
'token_input',
|
|
133
|
+
'token_output',
|
|
134
|
+
'carryover',
|
|
135
|
+
]);
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Required top-level fields (schema_version is stamped, handle defaults to null,
|
|
139
|
+
* so neither is required on input).
|
|
140
|
+
*/
|
|
141
|
+
export const REQUIRED_FIELDS = Object.freeze([
|
|
142
|
+
'record_kind',
|
|
143
|
+
'run_id',
|
|
144
|
+
'session_id',
|
|
145
|
+
'standard_version',
|
|
146
|
+
'rubric_version',
|
|
147
|
+
'provenance',
|
|
148
|
+
'model',
|
|
149
|
+
'harness',
|
|
150
|
+
'kpis',
|
|
151
|
+
'dimensions',
|
|
152
|
+
'anonymized',
|
|
153
|
+
'timestamp',
|
|
154
|
+
]);
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Submission-safe whitelist (Data-Minimization). Structured + nested-aware so
|
|
158
|
+
* projectSubmission is fully data-driven: a field absent from the relevant list
|
|
159
|
+
* is dropped from the projection. Deliberately EXCLUDES dimension `evidence`
|
|
160
|
+
* (may embed paths/prompts/repo names) and any unhashed hostname.
|
|
161
|
+
*
|
|
162
|
+
* Frozen at every level so a caller cannot silently widen the contract.
|
|
163
|
+
*/
|
|
164
|
+
export const SUBMISSION_FIELDS = Object.freeze({
|
|
165
|
+
/** Top-level scalar / already-safe fields copied verbatim. */
|
|
166
|
+
top: Object.freeze([
|
|
167
|
+
'schema_version',
|
|
168
|
+
'record_kind',
|
|
169
|
+
'run_id',
|
|
170
|
+
'standard_version',
|
|
171
|
+
'rubric_version',
|
|
172
|
+
'handle',
|
|
173
|
+
'anonymized',
|
|
174
|
+
'timestamp',
|
|
175
|
+
]),
|
|
176
|
+
/** provenance sub-fields (hash + public commit — safe). */
|
|
177
|
+
provenance: Object.freeze(['rubric_sha256', 'engine_commit']),
|
|
178
|
+
/** model sub-fields — the whole point of a leaderboard. */
|
|
179
|
+
model: Object.freeze(['id', 'source']),
|
|
180
|
+
/** harness sub-fields — hostname_hash is hashed; NO unhashed hostname exists here. */
|
|
181
|
+
harness: Object.freeze(['plugin_version', 'platform', 'host_class', 'hostname_hash']),
|
|
182
|
+
/** kpis — all numeric, safe. */
|
|
183
|
+
kpis: Object.freeze([...KPI_FIELDS]),
|
|
184
|
+
/** dimension sub-fields — NOTE: `evidence` is intentionally absent. */
|
|
185
|
+
dimensions: Object.freeze(['id', 'method', 'status', 'score', 'advisory', 'calibration_status']),
|
|
186
|
+
});
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* sha256 short-form hex guard. A real cleartext hostname (dots, uppercase,
|
|
190
|
+
* non-hex letters) fails this, structurally preventing an unhashed hostname
|
|
191
|
+
* from being stored in harness.hostname_hash.
|
|
192
|
+
*/
|
|
193
|
+
const HEX_RE = /^[a-f0-9]{8,}$/;
|
|
194
|
+
|
|
195
|
+
// ---------------------------------------------------------------------------
|
|
196
|
+
// ValidationError
|
|
197
|
+
// ---------------------------------------------------------------------------
|
|
198
|
+
|
|
199
|
+
export class ValidationError extends Error {
|
|
200
|
+
/**
|
|
201
|
+
* @param {string} message
|
|
202
|
+
* @param {string} [field] — optional field path that triggered the error
|
|
203
|
+
*/
|
|
204
|
+
constructor(message, field) {
|
|
205
|
+
super(message);
|
|
206
|
+
this.name = 'ValidationError';
|
|
207
|
+
this.field = field;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// ---------------------------------------------------------------------------
|
|
212
|
+
// Internal helpers
|
|
213
|
+
// ---------------------------------------------------------------------------
|
|
214
|
+
|
|
215
|
+
function isPlainObject(v) {
|
|
216
|
+
return v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
function isNonEmptyString(v) {
|
|
220
|
+
return typeof v === 'string' && v.trim().length > 0;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function isNumberOrNull(v) {
|
|
224
|
+
return v === null || (typeof v === 'number' && Number.isFinite(v));
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// ---------------------------------------------------------------------------
|
|
228
|
+
// run_id builder
|
|
229
|
+
// ---------------------------------------------------------------------------
|
|
230
|
+
|
|
231
|
+
/**
|
|
232
|
+
* Build a deterministic run_id: `<sessionId>-eval-<compactISO>` where compactISO
|
|
233
|
+
* strips `- : .` from the ISO timestamp (e.g. 2026-07-16T10:00:00.000Z →
|
|
234
|
+
* 20260716T100000000Z). Deterministic given the same inputs — never reads the
|
|
235
|
+
* clock, so a re-verify pass reproduces the same id.
|
|
236
|
+
*
|
|
237
|
+
* @param {string} sessionId — the session identifier.
|
|
238
|
+
* @param {string} timestamp — an ISO 8601 timestamp string.
|
|
239
|
+
* @returns {string}
|
|
240
|
+
*/
|
|
241
|
+
export function buildRunId(sessionId, timestamp) {
|
|
242
|
+
if (!isNonEmptyString(sessionId)) {
|
|
243
|
+
throw new ValidationError('buildRunId: sessionId must be a non-empty string', 'session_id');
|
|
244
|
+
}
|
|
245
|
+
if (!isNonEmptyString(timestamp) || Number.isNaN(Date.parse(timestamp))) {
|
|
246
|
+
throw new ValidationError('buildRunId: timestamp must be a valid ISO 8601 string', 'timestamp');
|
|
247
|
+
}
|
|
248
|
+
const compact = timestamp.replace(/[-:.]/g, '');
|
|
249
|
+
return `${sessionId}-eval-${compact}`;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
// ---------------------------------------------------------------------------
|
|
253
|
+
// Sub-validators (module-private)
|
|
254
|
+
// ---------------------------------------------------------------------------
|
|
255
|
+
|
|
256
|
+
function _validateSchemaVersion(entry) {
|
|
257
|
+
if ('schema_version' in entry && entry.schema_version !== undefined) {
|
|
258
|
+
if (entry.schema_version !== CURRENT_EVAL_SCHEMA_VERSION) {
|
|
259
|
+
throw new ValidationError(
|
|
260
|
+
`schema_version must be ${CURRENT_EVAL_SCHEMA_VERSION}, got: ${entry.schema_version}`,
|
|
261
|
+
'schema_version',
|
|
262
|
+
);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
function _rejectGlobalScore(entry) {
|
|
268
|
+
for (const key of FORBIDDEN_GLOBALSCORE_KEYS) {
|
|
269
|
+
if (key in entry) {
|
|
270
|
+
throw new ValidationError(
|
|
271
|
+
`record must NOT carry a global-score key '${key}' — the standard forbids any aggregated score in a single record`,
|
|
272
|
+
key,
|
|
273
|
+
);
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
function _validateRequiredPresence(entry) {
|
|
279
|
+
for (const field of REQUIRED_FIELDS) {
|
|
280
|
+
if (!(field in entry)) {
|
|
281
|
+
throw new ValidationError(`eval record missing required field: ${field}`, field);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
function _validateDiscriminators(entry) {
|
|
287
|
+
if (!VALID_RECORD_KINDS.includes(entry.record_kind)) {
|
|
288
|
+
throw new ValidationError(
|
|
289
|
+
`record_kind must be one of ${VALID_RECORD_KINDS.join('|')}, got: ${entry.record_kind}`,
|
|
290
|
+
'record_kind',
|
|
291
|
+
);
|
|
292
|
+
}
|
|
293
|
+
if (!isNonEmptyString(entry.run_id)) {
|
|
294
|
+
throw new ValidationError('run_id must be a non-empty string', 'run_id');
|
|
295
|
+
}
|
|
296
|
+
if (!isNonEmptyString(entry.session_id)) {
|
|
297
|
+
throw new ValidationError('session_id must be a non-empty string', 'session_id');
|
|
298
|
+
}
|
|
299
|
+
if (!isNonEmptyString(entry.standard_version)) {
|
|
300
|
+
throw new ValidationError('standard_version must be a non-empty string', 'standard_version');
|
|
301
|
+
}
|
|
302
|
+
if (!isNonEmptyString(entry.rubric_version)) {
|
|
303
|
+
throw new ValidationError('rubric_version must be a non-empty string', 'rubric_version');
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
function _validateTimestamp(entry) {
|
|
308
|
+
if (!isNonEmptyString(entry.timestamp) || Number.isNaN(Date.parse(entry.timestamp))) {
|
|
309
|
+
throw new ValidationError(
|
|
310
|
+
`timestamp must be a valid ISO 8601 string, got: ${entry.timestamp}`,
|
|
311
|
+
'timestamp',
|
|
312
|
+
);
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
function _validateProvenance(entry) {
|
|
317
|
+
const p = entry.provenance;
|
|
318
|
+
if (!isPlainObject(p)) {
|
|
319
|
+
throw new ValidationError('provenance must be an object', 'provenance');
|
|
320
|
+
}
|
|
321
|
+
if (!isNonEmptyString(p.rubric_sha256)) {
|
|
322
|
+
throw new ValidationError(
|
|
323
|
+
'provenance.rubric_sha256 must be a non-empty string',
|
|
324
|
+
'provenance.rubric_sha256',
|
|
325
|
+
);
|
|
326
|
+
}
|
|
327
|
+
if (p.engine_commit !== null && !isNonEmptyString(p.engine_commit)) {
|
|
328
|
+
throw new ValidationError(
|
|
329
|
+
'provenance.engine_commit must be a non-empty string or null',
|
|
330
|
+
'provenance.engine_commit',
|
|
331
|
+
);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
function _validateModel(entry) {
|
|
336
|
+
const m = entry.model;
|
|
337
|
+
if (!isPlainObject(m)) {
|
|
338
|
+
throw new ValidationError('model must be an object', 'model');
|
|
339
|
+
}
|
|
340
|
+
if (!isNonEmptyString(m.id)) {
|
|
341
|
+
throw new ValidationError('model.id must be a non-empty string', 'model.id');
|
|
342
|
+
}
|
|
343
|
+
if (!VALID_MODEL_SOURCES.includes(m.source)) {
|
|
344
|
+
throw new ValidationError(
|
|
345
|
+
`model.source must be one of ${VALID_MODEL_SOURCES.join('|')}, got: ${m.source}`,
|
|
346
|
+
'model.source',
|
|
347
|
+
);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
function _validateHarness(entry) {
|
|
352
|
+
const h = entry.harness;
|
|
353
|
+
if (!isPlainObject(h)) {
|
|
354
|
+
throw new ValidationError('harness must be an object', 'harness');
|
|
355
|
+
}
|
|
356
|
+
if (!isNonEmptyString(h.plugin_version)) {
|
|
357
|
+
throw new ValidationError('harness.plugin_version must be a non-empty string', 'harness.plugin_version');
|
|
358
|
+
}
|
|
359
|
+
if (!isNonEmptyString(h.platform)) {
|
|
360
|
+
throw new ValidationError('harness.platform must be a non-empty string', 'harness.platform');
|
|
361
|
+
}
|
|
362
|
+
if (h.host_class !== null && !isNonEmptyString(h.host_class)) {
|
|
363
|
+
throw new ValidationError('harness.host_class must be a non-empty string or null', 'harness.host_class');
|
|
364
|
+
}
|
|
365
|
+
// hostname_hash: null OR a sha256 short-form hex string — NEVER a cleartext
|
|
366
|
+
// hostname. The HEX_RE guard structurally rejects a cleartext hostname
|
|
367
|
+
// (dots / uppercase / non-hex letters) from being persisted.
|
|
368
|
+
if (h.hostname_hash !== null) {
|
|
369
|
+
if (typeof h.hostname_hash !== 'string' || !HEX_RE.test(h.hostname_hash)) {
|
|
370
|
+
throw new ValidationError(
|
|
371
|
+
'harness.hostname_hash must be null or a sha256 short-form hex string (never a cleartext hostname)',
|
|
372
|
+
'harness.hostname_hash',
|
|
373
|
+
);
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
function _validateKpis(entry) {
|
|
379
|
+
const k = entry.kpis;
|
|
380
|
+
if (!isPlainObject(k)) {
|
|
381
|
+
throw new ValidationError('kpis must be an object', 'kpis');
|
|
382
|
+
}
|
|
383
|
+
for (const field of KPI_FIELDS) {
|
|
384
|
+
if (!(field in k)) {
|
|
385
|
+
throw new ValidationError(`kpis missing required field: ${field} (use null, never a guessed 0)`, `kpis.${field}`);
|
|
386
|
+
}
|
|
387
|
+
const v = k[field];
|
|
388
|
+
if (!isNumberOrNull(v)) {
|
|
389
|
+
throw new ValidationError(`kpis.${field} must be a finite number or null, got: ${v}`, `kpis.${field}`);
|
|
390
|
+
}
|
|
391
|
+
if (typeof v === 'number' && v < 0) {
|
|
392
|
+
throw new ValidationError(`kpis.${field} must be non-negative, got: ${v}`, `kpis.${field}`);
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
function _validateDimensions(entry) {
|
|
398
|
+
if (!Array.isArray(entry.dimensions)) {
|
|
399
|
+
throw new ValidationError(`dimensions must be an array, got: ${typeof entry.dimensions}`, 'dimensions');
|
|
400
|
+
}
|
|
401
|
+
for (let i = 0; i < entry.dimensions.length; i++) {
|
|
402
|
+
const d = entry.dimensions[i];
|
|
403
|
+
const at = `dimensions[${i}]`;
|
|
404
|
+
if (!isPlainObject(d)) {
|
|
405
|
+
throw new ValidationError(`${at} must be an object`, at);
|
|
406
|
+
}
|
|
407
|
+
if (!isNonEmptyString(d.id)) {
|
|
408
|
+
throw new ValidationError(`${at}.id must be a non-empty string`, `${at}.id`);
|
|
409
|
+
}
|
|
410
|
+
if (!VALID_DIMENSION_METHODS.includes(d.method)) {
|
|
411
|
+
throw new ValidationError(
|
|
412
|
+
`${at}.method must be one of ${VALID_DIMENSION_METHODS.join('|')}, got: ${d.method}`,
|
|
413
|
+
`${at}.method`,
|
|
414
|
+
);
|
|
415
|
+
}
|
|
416
|
+
if (!VALID_DIMENSION_STATUSES.includes(d.status)) {
|
|
417
|
+
throw new ValidationError(
|
|
418
|
+
`${at}.status must be one of ${VALID_DIMENSION_STATUSES.join('|')}, got: ${d.status}`,
|
|
419
|
+
`${at}.status`,
|
|
420
|
+
);
|
|
421
|
+
}
|
|
422
|
+
if (typeof d.evidence !== 'string') {
|
|
423
|
+
throw new ValidationError(`${at}.evidence must be a string`, `${at}.evidence`);
|
|
424
|
+
}
|
|
425
|
+
if (d.score !== undefined && !isNumberOrNull(d.score)) {
|
|
426
|
+
throw new ValidationError(`${at}.score must be a finite number or null`, `${at}.score`);
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
if (d.method === 'judge') {
|
|
430
|
+
// Judge dimensions carry the advisory firewall + calibration status.
|
|
431
|
+
if (d.advisory !== true) {
|
|
432
|
+
throw new ValidationError(
|
|
433
|
+
`${at}.advisory must be the literal value true for a judge dimension (advisory-only guarantee), got: ${d.advisory}`,
|
|
434
|
+
`${at}.advisory`,
|
|
435
|
+
);
|
|
436
|
+
}
|
|
437
|
+
if (!VALID_CALIBRATION_STATUSES.includes(d.calibration_status)) {
|
|
438
|
+
throw new ValidationError(
|
|
439
|
+
`${at}.calibration_status must be one of ${VALID_CALIBRATION_STATUSES.join('|')} for a judge dimension, got: ${d.calibration_status}`,
|
|
440
|
+
`${at}.calibration_status`,
|
|
441
|
+
);
|
|
442
|
+
}
|
|
443
|
+
} else {
|
|
444
|
+
// Deterministic dimensions MUST NOT carry judge-only fields.
|
|
445
|
+
if ('advisory' in d) {
|
|
446
|
+
throw new ValidationError(
|
|
447
|
+
`${at}.advisory is only valid for a judge dimension`,
|
|
448
|
+
`${at}.advisory`,
|
|
449
|
+
);
|
|
450
|
+
}
|
|
451
|
+
if ('calibration_status' in d) {
|
|
452
|
+
throw new ValidationError(
|
|
453
|
+
`${at}.calibration_status is only valid for a judge dimension`,
|
|
454
|
+
`${at}.calibration_status`,
|
|
455
|
+
);
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
function _validateHandleAndAnonymized(entry) {
|
|
462
|
+
if (entry.handle !== undefined && entry.handle !== null && !isNonEmptyString(entry.handle)) {
|
|
463
|
+
throw new ValidationError('handle must be a non-empty string or null', 'handle');
|
|
464
|
+
}
|
|
465
|
+
if (typeof entry.anonymized !== 'boolean') {
|
|
466
|
+
throw new ValidationError(`anonymized must be a boolean, got: ${entry.anonymized}`, 'anonymized');
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
// ---------------------------------------------------------------------------
|
|
471
|
+
// Public API — validate
|
|
472
|
+
// ---------------------------------------------------------------------------
|
|
473
|
+
|
|
474
|
+
/**
|
|
475
|
+
* Validate a session-eval record for writing. Throws ValidationError on any
|
|
476
|
+
* required-field / type / enum / range violation, INCLUDING the presence of a
|
|
477
|
+
* forbidden global-score key. Returns a NEW object (input not mutated) with
|
|
478
|
+
* schema_version stamped to CURRENT_EVAL_SCHEMA_VERSION when absent.
|
|
479
|
+
*
|
|
480
|
+
* @param {object} entry
|
|
481
|
+
* @returns {object} validated entry (new object)
|
|
482
|
+
* @throws {ValidationError}
|
|
483
|
+
*/
|
|
484
|
+
export function validateEvalRecord(entry) {
|
|
485
|
+
if (!isPlainObject(entry)) {
|
|
486
|
+
throw new ValidationError('eval record must be a non-null object');
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
_rejectGlobalScore(entry);
|
|
490
|
+
_validateSchemaVersion(entry);
|
|
491
|
+
_validateRequiredPresence(entry);
|
|
492
|
+
_validateDiscriminators(entry);
|
|
493
|
+
_validateTimestamp(entry);
|
|
494
|
+
_validateProvenance(entry);
|
|
495
|
+
_validateModel(entry);
|
|
496
|
+
_validateHarness(entry);
|
|
497
|
+
_validateKpis(entry);
|
|
498
|
+
_validateDimensions(entry);
|
|
499
|
+
_validateHandleAndAnonymized(entry);
|
|
500
|
+
|
|
501
|
+
return {
|
|
502
|
+
...entry,
|
|
503
|
+
schema_version: entry.schema_version ?? CURRENT_EVAL_SCHEMA_VERSION,
|
|
504
|
+
};
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
// ---------------------------------------------------------------------------
|
|
508
|
+
// Public API — normalize (never throws)
|
|
509
|
+
// ---------------------------------------------------------------------------
|
|
510
|
+
|
|
511
|
+
/**
|
|
512
|
+
* Normalize a session-eval record read from disk. Never throws — malformed
|
|
513
|
+
* non-object input is returned unchanged. Applies read-path defaults so callers
|
|
514
|
+
* treat legacy and new records uniformly:
|
|
515
|
+
* - schema_version → CURRENT when absent
|
|
516
|
+
* - record_kind → 'session-eval' when absent
|
|
517
|
+
* - standard_version → CURRENT_STANDARD_VERSION when absent
|
|
518
|
+
* - handle → null when absent
|
|
519
|
+
* - anonymized → derived from handle presence when absent (no handle ⇒
|
|
520
|
+
* anonymous ⇒ true)
|
|
521
|
+
* - kpis → every KPI sub-field undefined → null ("don't fake perfect")
|
|
522
|
+
* - judge dimensions → advisory → true, calibration_status → 'uncalibrated'
|
|
523
|
+
*
|
|
524
|
+
* @param {object} entry
|
|
525
|
+
* @returns {object}
|
|
526
|
+
*/
|
|
527
|
+
export function normalizeEvalRecord(entry) {
|
|
528
|
+
if (!isPlainObject(entry)) return entry;
|
|
529
|
+
|
|
530
|
+
const handle = entry.handle ?? null;
|
|
531
|
+
|
|
532
|
+
const kpisIn = isPlainObject(entry.kpis) ? entry.kpis : {};
|
|
533
|
+
const kpis = {};
|
|
534
|
+
for (const field of KPI_FIELDS) {
|
|
535
|
+
kpis[field] = kpisIn[field] ?? null;
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
const dimensions = Array.isArray(entry.dimensions)
|
|
539
|
+
? entry.dimensions.map((d) => {
|
|
540
|
+
if (!isPlainObject(d)) return d;
|
|
541
|
+
if (d.method === 'judge') {
|
|
542
|
+
return {
|
|
543
|
+
...d,
|
|
544
|
+
advisory: d.advisory ?? true,
|
|
545
|
+
calibration_status: d.calibration_status ?? 'uncalibrated',
|
|
546
|
+
};
|
|
547
|
+
}
|
|
548
|
+
return d;
|
|
549
|
+
})
|
|
550
|
+
: entry.dimensions;
|
|
551
|
+
|
|
552
|
+
return {
|
|
553
|
+
...entry,
|
|
554
|
+
schema_version: entry.schema_version ?? CURRENT_EVAL_SCHEMA_VERSION,
|
|
555
|
+
record_kind: entry.record_kind ?? 'session-eval',
|
|
556
|
+
standard_version: entry.standard_version ?? CURRENT_STANDARD_VERSION,
|
|
557
|
+
handle,
|
|
558
|
+
anonymized: typeof entry.anonymized === 'boolean' ? entry.anonymized : handle === null,
|
|
559
|
+
kpis,
|
|
560
|
+
dimensions,
|
|
561
|
+
};
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
// ---------------------------------------------------------------------------
|
|
565
|
+
// Public API — submission projection (Data-Minimization)
|
|
566
|
+
// ---------------------------------------------------------------------------
|
|
567
|
+
|
|
568
|
+
/**
|
|
569
|
+
* Project a record onto the submission-safe whitelist (SUBMISSION_FIELDS),
|
|
570
|
+
* nested-aware. Anything not on the whitelist — paths, prompts, repo names,
|
|
571
|
+
* dimension evidence, unhashed hostnames, rogue extra fields — is dropped.
|
|
572
|
+
*
|
|
573
|
+
* Fully data-driven from SUBMISSION_FIELDS: a caller cannot widen the projection
|
|
574
|
+
* without editing that frozen whitelist (which turns the data-minimization test
|
|
575
|
+
* RED — the intended fake-regression tripwire).
|
|
576
|
+
*
|
|
577
|
+
* @remarks no v1 runtime caller — consumed by the future leaderboard epic
|
|
578
|
+
* (#803 follow-up); shipped now so the data-minimization contract is
|
|
579
|
+
* fixed before any submission surface exists.
|
|
580
|
+
* @param {object} record — a (normalized or raw) session-eval record.
|
|
581
|
+
* @returns {object} submission-safe projection.
|
|
582
|
+
*/
|
|
583
|
+
export function projectSubmission(record) {
|
|
584
|
+
if (!isPlainObject(record)) return {};
|
|
585
|
+
|
|
586
|
+
const out = {};
|
|
587
|
+
|
|
588
|
+
for (const key of SUBMISSION_FIELDS.top) {
|
|
589
|
+
if (key in record) out[key] = record[key];
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
out.provenance = _pickNested(record.provenance, SUBMISSION_FIELDS.provenance);
|
|
593
|
+
out.model = _pickNested(record.model, SUBMISSION_FIELDS.model);
|
|
594
|
+
out.harness = _pickNested(record.harness, SUBMISSION_FIELDS.harness);
|
|
595
|
+
out.kpis = _pickNested(record.kpis, SUBMISSION_FIELDS.kpis);
|
|
596
|
+
|
|
597
|
+
out.dimensions = Array.isArray(record.dimensions)
|
|
598
|
+
? record.dimensions.map((d) => _pickNested(d, SUBMISSION_FIELDS.dimensions))
|
|
599
|
+
: [];
|
|
600
|
+
|
|
601
|
+
return out;
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
/**
|
|
605
|
+
* Pick only the whitelisted keys from a nested object. Returns {} when the input
|
|
606
|
+
* is not an object, so the projection shape is stable regardless of input.
|
|
607
|
+
* @param {*} obj
|
|
608
|
+
* @param {readonly string[]} allowed
|
|
609
|
+
* @returns {object}
|
|
610
|
+
*/
|
|
611
|
+
function _pickNested(obj, allowed) {
|
|
612
|
+
const out = {};
|
|
613
|
+
if (!isPlainObject(obj)) return out;
|
|
614
|
+
for (const key of allowed) {
|
|
615
|
+
if (key in obj) out[key] = obj[key];
|
|
616
|
+
}
|
|
617
|
+
return out;
|
|
618
|
+
}
|