vieval 0.0.9 → 0.0.11

package/README.md

@@ -104,10 +104,11 @@ Use `describeTask` for the common Vitest-like authoring path:
 
 ```ts
 import { caseOf, describeTask, expect } from 'vieval'
+import { modelFromRun } from 'vieval/plugins/chat-models'
 
 describeTask('prompt-language-ablation', () => {
   caseOf('resolves matrix axes', async (context) => {
-    const selectedModel = context.model()
+    const selectedModel = modelFromRun(context, { axis: 'model' })
     const language = context.task.matrix.run.promptLanguage
     const scenario = context.task.matrix.run.scenario
 
@@ -166,6 +167,21 @@ Each scheduled task receives stable matrix metadata:
 - `task.matrix.meta.evalRowId`
 - `task.matrix.inputs` for `caseOf(..., { input })` and `casesFromInputs(...)`
 
+## Orchestration Model
+
+`vieval` separates benchmark management from run reliability:
+
+- `comparison`: cross-project or cross-workspace benchmark. Use it for horizontal evaluation across multiple agent, memory, model, paper, or backend implementations. Methods do not need perfect project/case alignment; compare artifacts report project and case coverage.
+- `workspace`: a batch-management boundary for related eval projects. Use it when one benchmark family spans multiple task projects, roots, env settings, or model registrations.
+- `project`: one eval task project with discovery rules, model registrations, optional executor, matrix layers, and scoring/reporting behavior.
+- `experiment`: run metadata derived from explicit `--experiment` or, when omitted, stable matrix row metadata. It does not create an extra scheduler layer.
+- `task`: one eval definition discovered from files and expanded across inference executors plus run/eval matrix rows.
+- `case`: the scoring and evidence source inside a task. `context.score(...)` contributes normalized score evidence; `context.metric(...)` emits benchmark metadata for reports.
+- `attempt`: a full task rerun used to estimate reliability. With `autoAttempt`, each full attempt contributes evidence, so a fail-then-pass pair scores as a success rate rather than replacing the earlier failure.
+- `retry`: an in-case retry. With `autoRetry`, a case can recover inside one attempt; a retry pass still counts as that attempt passing.
+
+Config inheritance follows the same outside-in model: top-level defaults apply first, workspace/project entries refine them, and project-local plugins can append or override project-local models/reporters/concurrency without leaking to sibling projects.
+
 ## Config Example
 
 ```ts
@@ -256,7 +272,16 @@ export default defineConfig({
         },
       ],
       async executor(task, context) {
-        const model = context.model({ name: 'motion-default' })
+        const model = context.models.find(model =>
+          model.id === 'motion-default'
+          || model.model === 'motion-default'
+          || model.aliases.includes('motion-default'),
+        )
+
+        if (model == null) {
+          throw new Error('Missing configured model "motion-default".')
+        }
+
         const success = model.model === 'v2' && task.matrix.run.scenario === 'baseline'
 
         return {

package/dist/bin/vieval.mjs

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { n as runTopLevelCli } from "../cli-Dao25VxV.mjs";
+import { n as runTopLevelCli } from "../cli-CHFCF8UR.mjs";
 import process from "node:process";
 import { errorMessageFrom } from "@moeru/std";
 //#region src/bin/vieval.ts

package/dist/bin/vieval.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"vieval.mjs","names":[],"sources":["../../src/bin/vieval.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport process from 'node:process'\n\nimport { errorMessageFrom } from '@moeru/std'\n\nimport { runTopLevelCli } from '../cli/index'\n\n/**\n * Bootstraps the published `vieval` executable.\n *\n * Call stack:\n *\n * package manager shim / direct node execution\n *   -> {@link runTopLevelCli} (`../cli`)\n *     -> subcommand orchestration modules\n *\n * Use when:\n * - the installed `vieval` binary starts from the command line\n * - process-bound startup must stay outside import-safe CLI modules\n *\n * Expects:\n * - `process.argv` contains the raw CLI arguments after the node executable path\n *\n * Returns:\n * - resolves after the selected subcommand finishes and updates `process.exitCode`\n */\nrunTopLevelCli(process.argv.slice(2)).catch((error) => {\n  const errorMessage = errorMessageFrom(error) ?? 'Unknown CLI failure.'\n  process.stderr.write(`[vieval] ${errorMessage}\\n`)\n  process.exitCode = 1\n})\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AA2BA,eAAe,QAAQ,KAAK,MAAM,EAAE,CAAC,CAAC,OAAO,UAAU;CACrD,MAAM,eAAe,iBAAiB,MAAM,IAAI;AAChD,SAAQ,OAAO,MAAM,YAAY,aAAa,IAAI;AAClD,SAAQ,WAAW;EACnB"}
+{"version":3,"file":"vieval.mjs","names":[],"sources":["../../src/bin/vieval.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport process from 'node:process'\n\nimport { errorMessageFrom } from '@moeru/std'\n\nimport { runTopLevelCli } from '../cli/index'\n\n/**\n * Bootstraps the published `vieval` executable.\n *\n * Call stack:\n *\n * package manager shim / direct node execution\n *   -> {@link runTopLevelCli} (`../cli`)\n *     -> subcommand orchestration modules\n *\n * Use when:\n * - the installed `vieval` binary starts from the command line\n * - process-bound startup must stay outside import-safe CLI modules\n *\n * Expects:\n * - `process.argv` contains the raw CLI arguments after the node executable path\n *\n * Returns:\n * - resolves after the selected subcommand finishes and updates `process.exitCode`\n */\nrunTopLevelCli(process.argv.slice(2)).catch((error) => {\n  const errorMessage = errorMessageFrom(error) ?? 'Unknown CLI failure.'\n  process.stderr.write(`[vieval] ${errorMessage}\\n`)\n  process.exitCode = 1\n})\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AA2BA,eAAe,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,UAAU;CACrD,MAAM,eAAe,iBAAiB,KAAK,KAAK;CAChD,QAAQ,OAAO,MAAM,YAAY,aAAa,GAAG;CACjD,QAAQ,WAAW;AACrB,CAAC"}

package/dist/cli/index.mjs

@@ -1,2 +1,2 @@
-import { n as runTopLevelCli, t as parseTopLevelCliArguments } from "../cli-Dao25VxV.mjs";
+import { n as runTopLevelCli, t as parseTopLevelCliArguments } from "../cli-CHFCF8UR.mjs";
 export { parseTopLevelCliArguments, runTopLevelCli };