vieval 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -3
- package/dist/bin/vieval.d.mts +1 -0
- package/dist/bin/vieval.mjs +33 -0
- package/dist/bin/vieval.mjs.map +1 -0
- package/dist/cli/index.d.mts +32 -0
- package/dist/cli/index.mjs +1 -2582
- package/dist/cli-sanbKtQq.mjs +2821 -0
- package/dist/cli-sanbKtQq.mjs.map +1 -0
- package/dist/config.d.mts +2 -2
- package/dist/config.mjs +16 -1
- package/dist/config.mjs.map +1 -0
- package/dist/core/assertions/index.d.mts +314 -2
- package/dist/core/assertions/index.mjs +182 -1
- package/dist/core/assertions/index.mjs.map +1 -0
- package/dist/core/inference-executors/index.d.mts +1 -1
- package/dist/core/inference-executors/index.mjs +1 -1
- package/dist/core/processors/results/index.d.mts +1 -1
- package/dist/core/runner/index.d.mts +3 -2
- package/dist/core/runner/index.mjs +637 -2
- package/dist/core/runner/index.mjs.map +1 -0
- package/dist/core/scheduler/index.d.mts +2 -0
- package/dist/core/scheduler/index.mjs +188 -0
- package/dist/core/scheduler/index.mjs.map +1 -0
- package/dist/{env-C7X81PWa.mjs → env--94B0UtW.mjs} +1 -1
- package/dist/{env-C7X81PWa.mjs.map → env--94B0UtW.mjs.map} +1 -1
- package/dist/{env-DtpjACOW.d.mts → env-BeHv_5mo.d.mts} +1 -1
- package/dist/{expect-extensions-BOzwV5EJ.mjs → expect-extensions-DCSqlneN.mjs} +2 -2
- package/dist/{expect-extensions-BOzwV5EJ.mjs.map → expect-extensions-DCSqlneN.mjs.map} +1 -1
- package/dist/expect.d.mts +10 -2
- package/dist/expect.mjs +16 -1
- package/dist/expect.mjs.map +1 -0
- package/dist/{index-BDMEAmf2.d.mts → index-DBZKkpBe.d.mts} +106 -4
- package/dist/index-fakXoZEe.d.mts +147 -0
- package/dist/index.d.mts +111 -12
- package/dist/index.mjs +216 -55
- package/dist/index.mjs.map +1 -1
- package/dist/models-DIGdOUpJ.mjs.map +1 -1
- package/dist/plugins/chat-models/index.d.mts +21 -1
- package/dist/plugins/chat-models/index.mjs +27 -1
- package/dist/plugins/chat-models/index.mjs.map +1 -1
- package/dist/queue-DsZQkZO_.mjs +21 -0
- package/dist/queue-DsZQkZO_.mjs.map +1 -0
- package/dist/{registry-CHJcTN2W.mjs → registry-CcKZqDJY.mjs} +27 -5
- package/dist/registry-CcKZqDJY.mjs.map +1 -0
- package/dist/testing/expect-extensions.d.mts +1 -1
- package/dist/testing/expect-extensions.mjs +1 -1
- package/package.json +9 -3
- package/dist/assertions-DcAjfVDA.mjs +0 -183
- package/dist/assertions-DcAjfVDA.mjs.map +0 -1
- package/dist/cli/index.mjs.map +0 -1
- package/dist/config-CHN24egi.mjs +0 -17
- package/dist/config-CHN24egi.mjs.map +0 -1
- package/dist/expect-B2vaoRVZ.d.mts +0 -10
- package/dist/expect-CaXiUkwY.mjs +0 -17
- package/dist/expect-CaXiUkwY.mjs.map +0 -1
- package/dist/index-C3gPFmcR.d.mts +0 -314
- package/dist/registry-CHJcTN2W.mjs.map +0 -1
- package/dist/runner-Dpy-eivM.mjs +0 -636
- package/dist/runner-Dpy-eivM.mjs.map +0 -1
package/README.md
CHANGED
|
@@ -93,7 +93,8 @@ Use these fields to group and compare runs across models, rubrics, and scenarios
|
|
|
93
93
|
|
|
94
94
|
```mermaid
|
|
95
95
|
flowchart LR
|
|
96
|
-
|
|
96
|
+
BIN["src/bin/vieval.ts\n(executable shim)"] --> CLI["src/cli/index.ts\n(runTopLevelCli)"]
|
|
97
|
+
CLI --> RUN["src/cli/run.ts\n(runVievalCli + formatter)"]
|
|
97
98
|
RUN --> CFG["src/cli/config.ts\n(loadVievalCliConfig)"]
|
|
98
99
|
RUN --> DISC["src/cli/discovery.ts\n(discoverEvalFiles)"]
|
|
99
100
|
RUN --> REG["src/dsl/registry.ts\n(module registrations)"]
|
|
@@ -131,6 +132,7 @@ flowchart LR
|
|
|
131
132
|
```mermaid
|
|
132
133
|
sequenceDiagram
|
|
133
134
|
participant U as User
|
|
135
|
+
participant B as src/bin/vieval.ts
|
|
134
136
|
participant C as src/cli/index.ts
|
|
135
137
|
participant R as src/cli/run.ts
|
|
136
138
|
participant L as src/cli/config.ts
|
|
@@ -139,7 +141,8 @@ sequenceDiagram
|
|
|
139
141
|
participant T as src/dsl/task.ts
|
|
140
142
|
participant P as src/cli/reporters/*
|
|
141
143
|
|
|
142
|
-
U->>
|
|
144
|
+
U->>B: pnpm run eval:run -- --config ...
|
|
145
|
+
B->>C: runTopLevelCli(argv)
|
|
143
146
|
C->>R: runVievalCli(options)
|
|
144
147
|
R->>L: loadVievalCliConfig()
|
|
145
148
|
R->>D: discoverEvalFiles()
|
|
@@ -235,7 +238,7 @@ pnpm -F vieval eval:run
|
|
|
235
238
|
pnpm -F vieval eval:run -- --config ./vieval.config.ts
|
|
236
239
|
pnpm -F vieval eval:run -- --config ./vieval.config.ts --project chess --project moderation
|
|
237
240
|
pnpm -F vieval eval:run -- --json
|
|
238
|
-
pnpm -F vieval exec tsx src/
|
|
241
|
+
pnpm -F vieval exec tsx src/bin/vieval.ts compare --config ../../vieval.config.ts --comparison <comparison-id>
|
|
239
242
|
pnpm -F vieval eval:run -- --help
|
|
240
243
|
```
|
|
241
244
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { };
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { n as runTopLevelCli } from "../cli-sanbKtQq.mjs";
|
|
3
|
+
import process from "node:process";
|
|
4
|
+
import { errorMessageFrom } from "@moeru/std";
|
|
5
|
+
//#region src/bin/vieval.ts
|
|
6
|
+
/**
|
|
7
|
+
* Bootstraps the published `vieval` executable.
|
|
8
|
+
*
|
|
9
|
+
* Call stack:
|
|
10
|
+
*
|
|
11
|
+
* package manager shim / direct node execution
|
|
12
|
+
* -> {@link runTopLevelCli} (`../cli`)
|
|
13
|
+
* -> subcommand orchestration modules
|
|
14
|
+
*
|
|
15
|
+
* Use when:
|
|
16
|
+
* - the installed `vieval` binary starts from the command line
|
|
17
|
+
* - process-bound startup must stay outside import-safe CLI modules
|
|
18
|
+
*
|
|
19
|
+
* Expects:
|
|
20
|
+
* - `process.argv` contains the raw CLI arguments after the node executable path
|
|
21
|
+
*
|
|
22
|
+
* Returns:
|
|
23
|
+
* - resolves after the selected subcommand finishes and updates `process.exitCode`
|
|
24
|
+
*/
|
|
25
|
+
runTopLevelCli(process.argv.slice(2)).catch((error) => {
|
|
26
|
+
const errorMessage = errorMessageFrom(error) ?? "Unknown CLI failure.";
|
|
27
|
+
process.stderr.write(`[vieval] ${errorMessage}\n`);
|
|
28
|
+
process.exitCode = 1;
|
|
29
|
+
});
|
|
30
|
+
//#endregion
|
|
31
|
+
export {};
|
|
32
|
+
|
|
33
|
+
//# sourceMappingURL=vieval.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vieval.mjs","names":[],"sources":["../../src/bin/vieval.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport process from 'node:process'\n\nimport { errorMessageFrom } from '@moeru/std'\n\nimport { runTopLevelCli } from '../cli/index'\n\n/**\n * Bootstraps the published `vieval` executable.\n *\n * Call stack:\n *\n * package manager shim / direct node execution\n * -> {@link runTopLevelCli} (`../cli`)\n * -> subcommand orchestration modules\n *\n * Use when:\n * - the installed `vieval` binary starts from the command line\n * - process-bound startup must stay outside import-safe CLI modules\n *\n * Expects:\n * - `process.argv` contains the raw CLI arguments after the node executable path\n *\n * Returns:\n * - resolves after the selected subcommand finishes and updates `process.exitCode`\n */\nrunTopLevelCli(process.argv.slice(2)).catch((error) => {\n const errorMessage = errorMessageFrom(error) ?? 'Unknown CLI failure.'\n process.stderr.write(`[vieval] ${errorMessage}\\n`)\n process.exitCode = 1\n})\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AA2BA,eAAe,QAAQ,KAAK,MAAM,EAAE,CAAC,CAAC,OAAO,UAAU;CACrD,MAAM,eAAe,iBAAiB,MAAM,IAAI;AAChD,SAAQ,OAAO,MAAM,YAAY,aAAa,IAAI;AAClD,SAAQ,WAAW;EACnB"}
|
package/dist/cli/index.d.mts
CHANGED
|
@@ -4,7 +4,39 @@ interface ParsedTopLevelCliArguments {
|
|
|
4
4
|
command: Command | 'help';
|
|
5
5
|
commandArgv: string[];
|
|
6
6
|
}
|
|
7
|
+
/**
|
|
8
|
+
* Parses top-level `vieval` CLI arguments into one command dispatch payload.
|
|
9
|
+
*
|
|
10
|
+
* Use when:
|
|
11
|
+
* - the executable needs to resolve which subcommand should run
|
|
12
|
+
* - tests need stable top-level argv normalization without invoking subcommands
|
|
13
|
+
*
|
|
14
|
+
* Expects:
|
|
15
|
+
* - argv excludes the node executable and script path
|
|
16
|
+
*
|
|
17
|
+
* Returns:
|
|
18
|
+
* - the normalized top-level command plus subcommand argv
|
|
19
|
+
*/
|
|
7
20
|
declare function parseTopLevelCliArguments(argv: readonly string[]): ParsedTopLevelCliArguments;
|
|
21
|
+
/**
|
|
22
|
+
* Dispatches the top-level `vieval` command to one concrete subcommand module.
|
|
23
|
+
*
|
|
24
|
+
* Call stack:
|
|
25
|
+
*
|
|
26
|
+
* published executable (`../bin/vieval`)
|
|
27
|
+
* -> {@link runTopLevelCli}
|
|
28
|
+
* -> {@link runEvalRunCli} / report CLI / compare CLI
|
|
29
|
+
*
|
|
30
|
+
* Use when:
|
|
31
|
+
* - the executable or tests need import-safe CLI orchestration
|
|
32
|
+
* - subcommands should remain reusable without process-bound startup code
|
|
33
|
+
*
|
|
34
|
+
* Expects:
|
|
35
|
+
* - argv excludes the node executable and script path
|
|
36
|
+
*
|
|
37
|
+
* Returns:
|
|
38
|
+
* - resolves after the selected subcommand completes
|
|
39
|
+
*/
|
|
8
40
|
declare function runTopLevelCli(argv: readonly string[]): Promise<void>;
|
|
9
41
|
//#endregion
|
|
10
42
|
export { parseTopLevelCliArguments, runTopLevelCli };
|