veryfront 0.1.871 → 0.1.873
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/cli/commands/eval/command-help.d.ts +3 -0
- package/esm/cli/commands/eval/command-help.d.ts.map +1 -0
- package/esm/cli/commands/eval/command-help.js +42 -0
- package/esm/cli/commands/eval/command.d.ts +21 -0
- package/esm/cli/commands/eval/command.d.ts.map +1 -0
- package/esm/cli/commands/eval/command.js +289 -0
- package/esm/cli/commands/eval/handler.d.ts +26 -0
- package/esm/cli/commands/eval/handler.d.ts.map +1 -0
- package/esm/cli/commands/eval/handler.js +28 -0
- package/esm/cli/help/command-definitions.d.ts.map +1 -1
- package/esm/cli/help/command-definitions.js +2 -0
- package/esm/cli/router.d.ts.map +1 -1
- package/esm/cli/router.js +2 -0
- package/esm/deno.d.ts +5 -2
- package/esm/deno.js +7 -4
- package/esm/src/agent/runtime/client-profile.d.ts.map +1 -1
- package/esm/src/agent/runtime/client-profile.js +10 -1
- package/esm/src/chat/ag-ui.d.ts.map +1 -1
- package/esm/src/chat/protocol.d.ts.map +1 -1
- package/esm/src/chat/protocol.js +0 -1
- package/esm/src/config/schemas/config.schema.d.ts +12 -0
- package/esm/src/config/schemas/config.schema.d.ts.map +1 -1
- package/esm/src/config/schemas/config.schema.js +12 -0
- package/esm/src/discovery/discovery-engine.d.ts.map +1 -1
- package/esm/src/discovery/discovery-engine.js +6 -1
- package/esm/src/discovery/handlers/eval-handler.d.ts +7 -0
- package/esm/src/discovery/handlers/eval-handler.d.ts.map +1 -0
- package/esm/src/discovery/handlers/eval-handler.js +16 -0
- package/esm/src/discovery/handlers/index.d.ts +1 -0
- package/esm/src/discovery/handlers/index.d.ts.map +1 -1
- package/esm/src/discovery/handlers/index.js +1 -0
- package/esm/src/discovery/import-rewriter.d.ts +1 -1
- package/esm/src/discovery/import-rewriter.d.ts.map +1 -1
- package/esm/src/discovery/import-rewriter.js +1 -0
- package/esm/src/discovery/project-discovery-config.d.ts +2 -0
- package/esm/src/discovery/project-discovery-config.d.ts.map +1 -1
- package/esm/src/discovery/project-discovery-config.js +2 -0
- package/esm/src/discovery/transpiler.d.ts.map +1 -1
- package/esm/src/discovery/transpiler.js +2 -0
- package/esm/src/discovery/types.d.ts +3 -0
- package/esm/src/discovery/types.d.ts.map +1 -1
- package/esm/src/eval/datasets.d.ts +8 -0
- package/esm/src/eval/datasets.d.ts.map +1 -0
- package/esm/src/eval/datasets.js +66 -0
- package/esm/src/eval/discovery.d.ts +43 -0
- package/esm/src/eval/discovery.d.ts.map +1 -0
- package/esm/src/eval/discovery.js +123 -0
- package/esm/src/eval/expect.d.ts +16 -0
- package/esm/src/eval/expect.d.ts.map +1 -0
- package/esm/src/eval/expect.js +62 -0
- package/esm/src/eval/factory.d.ts +6 -0
- package/esm/src/eval/factory.d.ts.map +1 -0
- package/esm/src/eval/factory.js +50 -0
- package/esm/src/eval/index.d.ts +33 -0
- package/esm/src/eval/index.d.ts.map +1 -0
- package/esm/src/eval/index.js +29 -0
- package/esm/src/eval/metrics.d.ts +55 -0
- package/esm/src/eval/metrics.d.ts.map +1 -0
- package/esm/src/eval/metrics.js +223 -0
- package/esm/src/eval/report.d.ts +12 -0
- package/esm/src/eval/report.d.ts.map +1 -0
- package/esm/src/eval/report.js +64 -0
- package/esm/src/eval/runner.d.ts +4 -0
- package/esm/src/eval/runner.d.ts.map +1 -0
- package/esm/src/eval/runner.js +101 -0
- package/esm/src/eval/studio.d.ts +177 -0
- package/esm/src/eval/studio.d.ts.map +1 -0
- package/esm/src/eval/studio.js +176 -0
- package/esm/src/eval/types.d.ts +221 -0
- package/esm/src/eval/types.d.ts.map +1 -0
- package/esm/src/eval/types.js +6 -0
- package/esm/src/eval/validation.d.ts +4 -0
- package/esm/src/eval/validation.d.ts.map +1 -0
- package/esm/src/eval/validation.js +41 -0
- package/esm/src/runs/index.d.ts +1 -1
- package/esm/src/runs/index.js +1 -1
- package/esm/src/runs/schemas.d.ts +9 -9
- package/esm/src/runs/schemas.d.ts.map +1 -1
- package/esm/src/runs/schemas.js +1 -1
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/package.json +12 -4
- package/esm/src/agent/testing/agent-tester.d.ts +0 -53
- package/esm/src/agent/testing/agent-tester.d.ts.map +0 -1
- package/esm/src/agent/testing/agent-tester.js +0 -132
- package/esm/src/agent/testing/durable-run-canaries/cli-runner.d.ts +0 -21
- package/esm/src/agent/testing/durable-run-canaries/cli-runner.d.ts.map +0 -1
- package/esm/src/agent/testing/durable-run-canaries/cli-runner.js +0 -66
- package/esm/src/agent/testing/durable-run-canaries/environment.d.ts +0 -14
- package/esm/src/agent/testing/durable-run-canaries/environment.d.ts.map +0 -1
- package/esm/src/agent/testing/durable-run-canaries/environment.js +0 -15
- package/esm/src/agent/testing/durable-run-canaries/index.d.ts +0 -5
- package/esm/src/agent/testing/durable-run-canaries/index.d.ts.map +0 -1
- package/esm/src/agent/testing/durable-run-canaries/index.js +0 -4
- package/esm/src/agent/testing/durable-run-canaries/runner.d.ts +0 -118
- package/esm/src/agent/testing/durable-run-canaries/runner.d.ts.map +0 -1
- package/esm/src/agent/testing/durable-run-canaries/runner.js +0 -391
- package/esm/src/agent/testing/durable-run-canaries/validation.d.ts +0 -12
- package/esm/src/agent/testing/durable-run-canaries/validation.d.ts.map +0 -1
- package/esm/src/agent/testing/durable-run-canaries/validation.js +0 -71
- package/esm/src/agent/testing/index.d.ts +0 -10
- package/esm/src/agent/testing/index.d.ts.map +0 -1
- package/esm/src/agent/testing/index.js +0 -9
- package/esm/src/agent/testing/live-evals/api-client.d.ts +0 -102
- package/esm/src/agent/testing/live-evals/api-client.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/api-client.js +0 -313
- package/esm/src/agent/testing/live-evals/cli-runner.d.ts +0 -40
- package/esm/src/agent/testing/live-evals/cli-runner.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/cli-runner.js +0 -144
- package/esm/src/agent/testing/live-evals/environment.d.ts +0 -15
- package/esm/src/agent/testing/live-evals/environment.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/environment.js +0 -20
- package/esm/src/agent/testing/live-evals/formatting.d.ts +0 -15
- package/esm/src/agent/testing/live-evals/formatting.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/formatting.js +0 -75
- package/esm/src/agent/testing/live-evals/index.d.ts +0 -12
- package/esm/src/agent/testing/live-evals/index.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/index.js +0 -11
- package/esm/src/agent/testing/live-evals/metadata.d.ts +0 -34
- package/esm/src/agent/testing/live-evals/metadata.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/metadata.js +0 -116
- package/esm/src/agent/testing/live-evals/performance.d.ts +0 -19
- package/esm/src/agent/testing/live-evals/performance.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/performance.js +0 -22
- package/esm/src/agent/testing/live-evals/preflight.d.ts +0 -12
- package/esm/src/agent/testing/live-evals/preflight.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/preflight.js +0 -27
- package/esm/src/agent/testing/live-evals/report.d.ts +0 -55
- package/esm/src/agent/testing/live-evals/report.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/report.js +0 -79
- package/esm/src/agent/testing/live-evals/request.d.ts +0 -31
- package/esm/src/agent/testing/live-evals/request.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/request.js +0 -48
- package/esm/src/agent/testing/live-evals/result.d.ts +0 -56
- package/esm/src/agent/testing/live-evals/result.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/result.js +0 -36
- package/esm/src/agent/testing/live-evals/runner.d.ts +0 -135
- package/esm/src/agent/testing/live-evals/runner.d.ts.map +0 -1
- package/esm/src/agent/testing/live-evals/runner.js +0 -396
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../../../src/src/chat/protocol.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;
|
|
1
|
+
{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../../../src/src/chat/protocol.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,yBAAyB,CAAC;AAGjC,MAAM,MAAM,aAAa,GAAG,WAAW,GAAG,MAAM,CAAC;AAEjD,2CAA2C;AAC3C,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,aAAa,CAAC;CACvB;AAED,qDAAqD;AACrD,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,WAAW,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,aAAa,CAAC;CACvB;AAED,2BAA2B;AAC3B,MAAM,MAAM,aAAa,GACrB,iBAAiB,GACjB,iBAAiB,GACjB,kBAAkB,GAClB,kBAAkB,GAClB,cAAc,CAAC;AAEnB,8CAA8C;AAC9C,MAAM,WAAW,YAAY,CAAC,IAAI,SAAS,MAAM,GAAG,MAAM,EAAE,KAAK,GAAG,OAAO,EAAE,MAAM,GAAG,OAAO;IAC3F,IAAI,EAAE,QAAQ,IAAI,EAAE,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,IAAI,CAAC;IACf,KAAK,EAAE,aAAa,CAAC;IACrB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,oDAAoD;AACpD,MAAM,WAAW,kBAAkB,CAAC,MAAM,GAAG,OAAO;IAClD,IAAI,EAAE,aAAa,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,sDAAsD;AACtD,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,cAAc,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,aAAa,CAAC;IACrB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,8CAA8C;AAC9C,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,YAAY,GAAG,UAAU,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,8CAA8C;AAC9C,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,QAAQ,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,OAAO,CAAC;CACf;AAED,iDAAiD;AACjD,MAAM,MAAM,eAAe,GACvB,YAAY,GACZ,iBAAiB,GACjB,YAAY,GACZ,kBAAkB,GAClB,mBAAmB,GACnB,YAAY,GACZ,YAAY,CAAC;AAEjB,8BAA8B;AAC9B,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;IACtC,KAAK,EAAE,eAAe,EAAE,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,SAAS,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC;CAC3B;AAED,2DAA2D;AAC3D,MAAM,WAAW,wBAAwB;IACvC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,yDAAyD;AACzD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,iDAAiD;AACjD,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,+CAA+C;AAC/C,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;IACzD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,qBAAqB,EAAE,CAAC;IACpC,WAAW,CAAC,EAAE,uBAAuB,EAAE,CAAC;IACxC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACtC;AAED,qDAAqD;AACrD,MAAM,WAAW,mBAAmB;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,KAAK,CAAC,EAAE,wBAAwB,CAAC;CAClC;AAED,kDAAkD;AAClD,MAAM,MAAM,gBAAgB,GACxB,MAAM,GACN,QAAQ,GACR,YAAY,GACZ,gBAAgB,GAChB,OAAO,GACP,OAAO,CAAC;AAEZ,sDAAsD;AACtD,KAAK,mBAAmB,GAAG;IACzB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,qCAAqC;AACrC,MAAM,MAAM,eAAe,GACvB;IACA,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B,GACC;IACA,IAAI,EAAE,kBAAkB,CAAC;IACzB,eAAe,EAAE,OAAO,CAAC;CAC1B,GACC;IACA,IAAI,EAAE,YAAY,CAAC;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GACC;IACA,IAAI,EAAE,YAAY,CAAC;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf,GACC;IACA,IAAI,EAAE,UAAU,CAAC;IACjB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GACC;IACA,IAAI,EAAE,iBAAiB,CAAC;IACxB,EAAE,EAAE,MAAM,CAAC;CACZ,GACC;IACA,IAAI,EAAE,iBAAiB,CAAC;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf,GACC;IACA,IAAI,EAAE,eAAe,CAAC;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,GACC,CAAC;IACD,IAAI,EAAE,kBAAkB,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,mBAAmB,CAAC,GACtB,CAAC;IACD,IAAI,EAAE,kBAAkB,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;CACxB,GAAG,mBAAmB,CAAC,GACtB,CAAC;IACD,IAAI,EAAE,sBAAsB,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,OAAO,CAAC;CAChB,GAAG,mBAAmB,CAAC,GACtB,CAAC;IACD,IAAI,EAAE,kBAAkB,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,mBAAmB,CAAC,GACtB;IACA,IAAI,EAAE,uBAAuB,CAAC;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,GACC,CAAC;IACD,IAAI,EAAE,uBAAuB,CAAC;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;CACjB,GAAG,mBAAmB,CAAC,GACtB;IACA,IAAI,EAAE,oBAAoB,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;CACpB,GACC,CAAC;IACD,IAAI,EAAE,mBAAmB,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,mBAAmB,CAAC,GACtB;IACA,IAAI,EAAE,YAAY,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GACC;IACA,IAAI,EAAE,iBAAiB,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,GACC;IACA,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,GACC;IACA,IAAI,EAAE,YAAY,CAAC;CACpB,GACC;IACA,IAAI,EAAE,aAAa,CAAC;CACrB,GACC;IACA,IAAI,EAAE,QAAQ,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,OAAO,CAAC;CACf,GACC;IACA,IAAI,EAAE,QAAQ,CAAC;IACf,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC,GACC;IACA,IAAI,EAAE,OAAO,CAAC;CACf,GACC;IACA,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEJ,uDAAuD;AACvD,KAAK,qBAAqB,CAAC,gBAAgB,IACvC;IACA,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,gBAAgB,CAAC;CACpC,GACC;IACA,IAAI,EAAE,QAAQ,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,gBAAgB,CAAC;CACpC,GACC;IACA,IAAI,EAAE,kBAAkB,CAAC;IACzB,eAAe,EAAE,gBAAgB,CAAC;CACnC,CAAC;AAEJ,wCAAwC;AACxC,KAAK,OAAO,CAAC,KAAK,SAAS,MAAM,IAAI;IACnC,IAAI,EAAE,KAAK,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;CACZ,CAAC;AAEF,8CAA8C;AAC9C,KAAK,YAAY,CAAC,KAAK,SAAS,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,GAAG;IACzD,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,+CAA+C;AAC/C,KAAK,aAAa,CAAC,KAAK,SAAS,MAAM,IAAI;IACzC,IAAI,EAAE,KAAK,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,qDAAqD;AACrD,KAAK,kBAAkB,CAAC,KAAK,SAAS,MAAM,IAAI,aAAa,CAAC,KAAK,CAAC,GAAG;IACrE,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,gDAAgD;AAChD,KAAK,cAAc,CAAC,KAAK,SAAS,MAAM,IAAI,kBAAkB,CAAC,KAAK,CAAC,GAAG;IACtE,KAAK,EAAE,OAAO,CAAC;CAChB,CAAC;AAEF,gDAAgD;AAChD,KAAK,cAAc,CAAC,KAAK,SAAS,MAAM,IAAI,aAAa,CAAC,KAAK,CAAC,GAAG;IACjE,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,qDAAqD;AACrD,MAAM,MAAM,kBAAkB,CAAC,gBAAgB,GAAG,mBAAmB,IACjE,qBAAqB,CAAC,gBAAgB,CAAC,GACvC;IACA,IAAI,EAAE,YAAY,CAAC;CACpB,GACC;IACA,IAAI,EAAE,aAAa,CAAC;CACrB,GACC;IACA,IAAI,EAAE,OAAO,CAAC;CACf,GACC,OAAO,CAAC,iBAAiB,CAAC,GAC1B,YAAY,CAAC,iBAAiB,CAAC,GAC/B;IACA,IAAI,EAAE,eAAe,CAAC;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,GACC,OAAO,CAAC,YAAY,CAAC,GACrB,YAAY,CAAC,YAAY,CAAC,GAC1B,OAAO,CAAC,UAAU,CAAC,GACnB;IACA,IAAI,EAAE,YAAY,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GACC;IACA,IAAI,EAAE,iBAAiB,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,GACC;IACA,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;CACb,GACC,kBAAkB,CAAC,kBAAkB,CAAC,GACtC,CAAC,aAAa,CAAC,kBAAkB,CAAC,GAAG;IACrC,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC,GACA,cAAc,CAAC,sBAAsB,CAAC,GACtC,CAAC,cAAc,CAAC,kBAAkB,CAAC,GAAG;IACtC,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC,GACA,CAAC,aAAa,CAAC,uBAAuB,CAAC,GAAG;IAC1C,MAAM,EAAE,OAAO,CAAC;CACjB,CAAC,GACA,cAAc,CAAC,mBAAmB,CAAC,GACnC,aAAa,CAAC,oBAAoB,CAAC,GACnC,CAAC,aAAa,CAAC,uBAAuB,CAAC,GAAG;IAC1C,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,GACA;IACA,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB,GACC;IACA,IAAI,EAAE,QAAQ,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,OAAO,CAAC;CACf,CAAC"}
|
package/esm/src/chat/protocol.js
CHANGED
|
@@ -313,6 +313,12 @@ export declare const getVeryfrontConfigSchema: () => import("../../internal-agen
|
|
|
313
313
|
paths: import("../../internal-agents/schema.js").Schema<string[] | undefined>;
|
|
314
314
|
}>> | undefined>;
|
|
315
315
|
}>> | undefined>;
|
|
316
|
+
evals: import("../../internal-agents/schema.js").Schema<Partial<import("../../extensions/schema/schema-validator.js").InferShape<{
|
|
317
|
+
discovery: import("../../internal-agents/schema.js").Schema<Partial<import("../../extensions/schema/schema-validator.js").InferShape<{
|
|
318
|
+
enabled: import("../../internal-agents/schema.js").Schema<boolean | undefined>;
|
|
319
|
+
paths: import("../../internal-agents/schema.js").Schema<string[] | undefined>;
|
|
320
|
+
}>> | undefined>;
|
|
321
|
+
}>> | undefined>;
|
|
316
322
|
mcp: import("../../internal-agents/schema.js").Schema<Partial<import("../../extensions/schema/schema-validator.js").InferShape<{
|
|
317
323
|
enabled: import("../../internal-agents/schema.js").Schema<boolean | undefined>;
|
|
318
324
|
port: import("../../internal-agents/schema.js").Schema<number | undefined>;
|
|
@@ -717,6 +723,12 @@ export declare const veryfrontConfigSchema: import("../../internal-agents/schema
|
|
|
717
723
|
paths: import("../../internal-agents/schema.js").Schema<string[] | undefined>;
|
|
718
724
|
}>> | undefined>;
|
|
719
725
|
}>> | undefined>;
|
|
726
|
+
evals: import("../../internal-agents/schema.js").Schema<Partial<import("../../extensions/schema/schema-validator.js").InferShape<{
|
|
727
|
+
discovery: import("../../internal-agents/schema.js").Schema<Partial<import("../../extensions/schema/schema-validator.js").InferShape<{
|
|
728
|
+
enabled: import("../../internal-agents/schema.js").Schema<boolean | undefined>;
|
|
729
|
+
paths: import("../../internal-agents/schema.js").Schema<string[] | undefined>;
|
|
730
|
+
}>> | undefined>;
|
|
731
|
+
}>> | undefined>;
|
|
720
732
|
mcp: import("../../internal-agents/schema.js").Schema<Partial<import("../../extensions/schema/schema-validator.js").InferShape<{
|
|
721
733
|
enabled: import("../../internal-agents/schema.js").Schema<boolean | undefined>;
|
|
722
734
|
port: import("../../internal-agents/schema.js").Schema<number | undefined>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.schema.d.ts","sourceRoot":"","sources":["../../../../src/src/config/schemas/config.schema.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,kCAAkC,CAAC;AAiChF,eAAO,MAAM,wBAAwB;;;;;QAQ3B,2GAA2G;;;;;;;;;;;;;;;IAuB/G,wFAAwF;;IAExF,0FAA0F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAyCtF;;;;;;;;;;;;;;;;;WAiBG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAgDH;;;;;;;;;;WAUG;;;;;;;;;;QAaH;;;;;WAKG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YA4HC,kFAAkF;;YAElF,oFAAoF;;YAEpF,+EAA+E;;YAE/E,mEAAmE;;;;;;;;;;;;;;;;;YA8BnE,mCAAmC;;YAEnC,8CAA8C;;YAE9C,sBAAsB;;YAEtB,mDAAmD
|
|
1
|
+
{"version":3,"file":"config.schema.d.ts","sourceRoot":"","sources":["../../../../src/src/config/schemas/config.schema.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,kCAAkC,CAAC;AAiChF,eAAO,MAAM,wBAAwB;;;;;QAQ3B,2GAA2G;;;;;;;;;;;;;;;IAuB/G,wFAAwF;;IAExF,0FAA0F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAyCtF;;;;;;;;;;;;;;;;;WAiBG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAgDH;;;;;;;;;;WAUG;;;;;;;;;;QAaH;;;;;WAKG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YA4HC,kFAAkF;;YAElF,oFAAoF;;YAEpF,+EAA+E;;YAE/E,mEAAmE;;;;;;;;;;;;;;;;;YA8BnE,mCAAmC;;YAEnC,8CAA8C;;YAE9C,sBAAsB;;YAEtB,mDAAmD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QA6JvD,yDAAyD;;QAEzD,iDAAiD;;;YAI7C,iEAAiE;;;;;;;IAgBzE,uCAAuC;;QAGnC,2CAA2C;;;;QAO3C,6DAA6D;;QAE7D,gGAAgG;;QAGhG,iEAAiE;;;;QAOjE,sEAAsE;;;IAK1E,kEAAkE;;QAM1D,yEAAyE;;QAEzE,kDAAkD;;QAElD;;sDAE8C;;;IAOtD;;;;;;;;;OASG;;IAEH,0CAA0C;;QAGtC,8CAA8C;;QAE9C,gEAAgE;;QAEhE,yCAAyC;;QAEzC,qCAAqC;;QAErC,sBAAsB;;QAEtB,gCAAgC;;YAG5B,qDAAqD;;YAErD,qDAAqD;;YAErD,oDAAoD;;;QAKxD,oCAAoC;;YAGhC,4EAA4E;;YAE5E,8DAA8D;;YAE9D,8EAA8E;;;;IAU3F,CAAC;AACF,eAAO,MAAM,qBAAqB;;;;;QA5lBxB,2GAA2G;;;;;;;;;;;;;;;IAuB/G,wFAAwF;;IAExF,0FAA0F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAyCtF;;;;;;;;;;;;;;;;;WAiBG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAgDH;;;;;;;;;;WAUG;;;;;;;;;;QAaH;;;;;WAKG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YA4HC,kFAAkF;;YAElF,oFAAoF;;YAEpF,+EAA+E;;YAE/E,mEAAmE;;;;;;;;;;;;;;;;;YA8BnE,mCAAmC;;YAEnC,8CAA8C;;YAE9C,sBAAsB;;YAEtB,mDAAmD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QA6JvD,yDAAyD;;QAEzD,iDAAiD;;;YAI7C,iEAAiE;;;;;;;IAgBzE,uCAAuC;;QAGnC,2CAA2C;;;;QAO3C,6DAA6D;;QAE7D,gGAAgG;;QAGhG,iEAAiE;;;;QAOjE,sEAAsE;;;IAK1E,kEAAkE;;QAM1D,yEAAyE;;QAEzE,kDAAkD;;QAElD;;sDAE8C;;;IAOtD;;;;;;;;;OASG;;IAEH,0CAA0C;;QAGtC,8CAA8C;;QAE9C,gEAAgE;;QAEhE,yCAAyC;;QAEzC,qCAAqC;;QAErC,sBAAsB;;QAEtB,gCAAgC;;YAG5B,qDAAqD;;YAErD,qDAAqD;;YAErD,oDAAoD;;;QAKxD,oCAAoC;;YAGhC,4EAA4E;;YAE5E,8DAA8D;;YAE9D,8EAA8E;;;;IAWnB,CAAC;AAG1E,MAAM,MAAM,eAAe,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,wBAAwB,CAAC,CAAC,CAAC;AACvF,MAAM,MAAM,oBAAoB,GAAG,UAAU,CAAC,UAAU,CAAC,OAAO,wBAAwB,CAAC,CAAC,CAAC;AAG3F,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,eAAe,CA0BvE;AAqCD,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,EAAE,CAEhF"}
|
|
@@ -479,6 +479,18 @@ export const getVeryfrontConfigSchema = defineSchema((v) => v
|
|
|
479
479
|
})
|
|
480
480
|
.partial()
|
|
481
481
|
.optional(),
|
|
482
|
+
evals: v
|
|
483
|
+
.object({
|
|
484
|
+
discovery: v
|
|
485
|
+
.object({
|
|
486
|
+
enabled: v.boolean().optional(),
|
|
487
|
+
paths: v.array(v.string()).optional(),
|
|
488
|
+
})
|
|
489
|
+
.partial()
|
|
490
|
+
.optional(),
|
|
491
|
+
})
|
|
492
|
+
.partial()
|
|
493
|
+
.optional(),
|
|
482
494
|
mcp: v
|
|
483
495
|
.object({
|
|
484
496
|
enabled: v.boolean().optional(),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"discovery-engine.d.ts","sourceRoot":"","sources":["../../../src/src/discovery/discovery-engine.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,OAAO,KAAK,EACV,eAAe,EAEf,eAAe,EAEhB,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"discovery-engine.d.ts","sourceRoot":"","sources":["../../../src/src/discovery/discovery-engine.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,OAAO,KAAK,EACV,eAAe,EAEf,eAAe,EAEhB,MAAM,YAAY,CAAC;AAwIpB;;GAEG;AACH,wBAAsB,WAAW,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CA2FnF"}
|
|
@@ -10,7 +10,7 @@ import { ensureError } from "../errors/veryfront-error.js";
|
|
|
10
10
|
import { registerSkill, skillRegistry } from "../skill/registry.js";
|
|
11
11
|
import { importModule } from "./transpiler.js";
|
|
12
12
|
import { findTypeScriptFiles } from "./file-discovery.js";
|
|
13
|
-
import { agentHandler, discoverSkills, promptHandler, resourceHandler, taskHandler, toolHandler, workflowHandler, workHandler, } from "./handlers/index.js";
|
|
13
|
+
import { agentHandler, discoverSkills, evalHandler, promptHandler, resourceHandler, taskHandler, toolHandler, workflowHandler, workHandler, } from "./handlers/index.js";
|
|
14
14
|
import { discoverRuntimeAgentMarkdownDefinitions } from "./handlers/runtime-agent-markdown-handler.js";
|
|
15
15
|
import { filenameToId } from "./discovery-utils.js";
|
|
16
16
|
import { join } from "../platform/compat/path/index.js";
|
|
@@ -115,6 +115,7 @@ export async function discoverAll(config) {
|
|
|
115
115
|
workflows: new Map(),
|
|
116
116
|
works: new Map(),
|
|
117
117
|
tasks: new Map(),
|
|
118
|
+
evals: new Map(),
|
|
118
119
|
errors: [],
|
|
119
120
|
};
|
|
120
121
|
// Discover tools
|
|
@@ -165,5 +166,9 @@ export async function discoverAll(config) {
|
|
|
165
166
|
for (const dir of config.taskDirs ?? ["tasks"]) {
|
|
166
167
|
await discoverItems(`${baseDir}/${dir}`, result, context, taskHandler, config.verbose);
|
|
167
168
|
}
|
|
169
|
+
// Discover eval definitions
|
|
170
|
+
for (const dir of config.evalDirs ?? ["evals"]) {
|
|
171
|
+
await discoverItems(`${baseDir}/${dir}`, result, context, evalHandler, config.verbose);
|
|
172
|
+
}
|
|
168
173
|
return result;
|
|
169
174
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-handler.d.ts","sourceRoot":"","sources":["../../../../src/src/discovery/handlers/eval-handler.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAE1D,OAAO,KAAK,EAAE,gBAAgB,EAAmB,MAAM,aAAa,CAAC;AAErE,eAAO,MAAM,WAAW,EAAE,gBAAgB,CAAC,cAAc,CAWxD,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Eval Discovery Handler
|
|
3
|
+
*/
|
|
4
|
+
import { deriveEvalId, isEvalDefinition } from "../../eval/index.js";
|
|
5
|
+
export const evalHandler = {
|
|
6
|
+
typeName: "eval",
|
|
7
|
+
validate: (item) => isEvalDefinition(item),
|
|
8
|
+
getId: (definition, file, dir) => definition.id || deriveEvalId(file, dir),
|
|
9
|
+
register: (id, definition, file) => ({
|
|
10
|
+
...definition,
|
|
11
|
+
id,
|
|
12
|
+
name: definition.name || id,
|
|
13
|
+
source: definition.source ?? { filePath: file, exportName: "default" },
|
|
14
|
+
}),
|
|
15
|
+
getResultMap: (result) => result.evals,
|
|
16
|
+
};
|
|
@@ -10,5 +10,6 @@ export { promptHandler } from "./prompt-handler.js";
|
|
|
10
10
|
export { workflowHandler } from "./workflow-handler.js";
|
|
11
11
|
export { workHandler } from "./work-handler.js";
|
|
12
12
|
export { taskHandler } from "./task-handler.js";
|
|
13
|
+
export { evalHandler } from "./eval-handler.js";
|
|
13
14
|
export { discoverSkills } from "./skill-handler.js";
|
|
14
15
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/src/discovery/handlers/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/src/discovery/handlers/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC"}
|
|
@@ -10,4 +10,5 @@ export { promptHandler } from "./prompt-handler.js";
|
|
|
10
10
|
export { workflowHandler } from "./workflow-handler.js";
|
|
11
11
|
export { workHandler } from "./work-handler.js";
|
|
12
12
|
export { taskHandler } from "./task-handler.js";
|
|
13
|
+
export { evalHandler } from "./eval-handler.js";
|
|
13
14
|
export { discoverSkills } from "./skill-handler.js";
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* (Deno, Node.js) and handles veryfront package resolution.
|
|
6
6
|
*/
|
|
7
7
|
import { createFileSystem } from "../platform/compat/fs.js";
|
|
8
|
-
export declare const DISCOVERY_GLOBAL_VERYFRONT_MODULES: readonly ["veryfront/agent", "veryfront/tool", "veryfront/platform", "veryfront/prompt", "veryfront/resource", "veryfront/embedding", "veryfront/workflow", "veryfront/work", "veryfront/schemas"];
|
|
8
|
+
export declare const DISCOVERY_GLOBAL_VERYFRONT_MODULES: readonly ["veryfront/agent", "veryfront/tool", "veryfront/platform", "veryfront/prompt", "veryfront/resource", "veryfront/embedding", "veryfront/workflow", "veryfront/work", "veryfront/eval", "veryfront/schemas"];
|
|
9
9
|
interface DenoRewriteOptions {
|
|
10
10
|
compiled?: boolean;
|
|
11
11
|
resolveSpecifier?: (specifier: string) => string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"import-rewriter.d.ts","sourceRoot":"","sources":["../../../src/src/discovery/import-rewriter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAI5D,eAAO,MAAM,kCAAkC,
|
|
1
|
+
{"version":3,"file":"import-rewriter.d.ts","sourceRoot":"","sources":["../../../src/src/discovery/import-rewriter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAI5D,eAAO,MAAM,kCAAkC,sNAWrC,CAAC;AAEX,UAAU,kBAAkB;IAC1B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,gBAAgB,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,CAAC;CAClD;AAkHD;;;;;GAKG;AACH,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,OAAO,GAAE,kBAAuB,GAC/B,MAAM,CA2BR;AAqFD;;;;;GAKG;AACH,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,UAAU,EAAE,MAAM,EAClB,EAAE,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,EACvC,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,CAAC,CA2PjB"}
|
|
@@ -10,6 +10,7 @@ export declare const DEFAULT_PROJECT_DISCOVERY_DIRS: {
|
|
|
10
10
|
workflowDirs: string[];
|
|
11
11
|
workDirs: string[];
|
|
12
12
|
taskDirs: string[];
|
|
13
|
+
evalDirs: string[];
|
|
13
14
|
};
|
|
14
15
|
type ProjectDiscoveryConfigInput = {
|
|
15
16
|
projectDir: string;
|
|
@@ -26,6 +27,7 @@ export type ProjectDiscoveryConfig = DiscoveryConfig & {
|
|
|
26
27
|
workflowDirs: string[];
|
|
27
28
|
workDirs: string[];
|
|
28
29
|
taskDirs: string[];
|
|
30
|
+
evalDirs: string[];
|
|
29
31
|
};
|
|
30
32
|
export declare function createProjectDiscoveryConfig(input: ProjectDiscoveryConfigInput): ProjectDiscoveryConfig;
|
|
31
33
|
export {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"project-discovery-config.d.ts","sourceRoot":"","sources":["../../../src/src/discovery/project-discovery-config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACtE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,eAAO,MAAM,8BAA8B
|
|
1
|
+
{"version":3,"file":"project-discovery-config.d.ts","sourceRoot":"","sources":["../../../src/src/discovery/project-discovery-config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACtE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,eAAO,MAAM,8BAA8B;;;;;;;;;;CAU1C,CAAC;AAOF,KAAK,2BAA2B,GAAG;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;IAChC,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG,eAAe,GAAG;IACrD,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB,CAAC;AAgBF,wBAAgB,4BAA4B,CAC1C,KAAK,EAAE,2BAA2B,GACjC,sBAAsB,CA4CxB"}
|
|
@@ -7,6 +7,7 @@ export const DEFAULT_PROJECT_DISCOVERY_DIRS = {
|
|
|
7
7
|
workflowDirs: ["workflows"],
|
|
8
8
|
workDirs: ["work"],
|
|
9
9
|
taskDirs: ["tasks"],
|
|
10
|
+
evalDirs: ["evals"],
|
|
10
11
|
};
|
|
11
12
|
function isDiscoveryEnabled(discovery) {
|
|
12
13
|
return discovery?.enabled ?? true;
|
|
@@ -29,6 +30,7 @@ export function createProjectDiscoveryConfig(input) {
|
|
|
29
30
|
workflowDirs: resolveDiscoveryPaths(aiConfig?.workflows?.discovery, DEFAULT_PROJECT_DISCOVERY_DIRS.workflowDirs),
|
|
30
31
|
workDirs: resolveDiscoveryPaths(aiConfig?.work?.discovery, DEFAULT_PROJECT_DISCOVERY_DIRS.workDirs),
|
|
31
32
|
taskDirs: resolveDiscoveryPaths(aiConfig?.tasks?.discovery, DEFAULT_PROJECT_DISCOVERY_DIRS.taskDirs),
|
|
33
|
+
evalDirs: resolveDiscoveryPaths(aiConfig?.evals?.discovery, DEFAULT_PROJECT_DISCOVERY_DIRS.evalDirs),
|
|
32
34
|
fsAdapter: input.fsAdapter,
|
|
33
35
|
verbose: input.verbose ?? false,
|
|
34
36
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transpiler.d.ts","sourceRoot":"","sources":["../../../src/src/discovery/transpiler.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"transpiler.d.ts","sourceRoot":"","sources":["../../../src/src/discovery/transpiler.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAuIvD;;GAEG;AACH,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,OAAO,CAAC,CAkGlB;AAED;;GAEG;AACH,wBAAgB,mBAAmB,IAAI,IAAI,CAE1C"}
|
|
@@ -21,6 +21,7 @@ import * as resourceMod from "../resource/index.js";
|
|
|
21
21
|
import * as embeddingMod from "../embedding/index.js";
|
|
22
22
|
import * as workflowMod from "../workflow/index.js";
|
|
23
23
|
import * as workMod from "../work/index.js";
|
|
24
|
+
import * as evalMod from "../eval/index.js";
|
|
24
25
|
import * as schemasMod from "../schemas/index.js";
|
|
25
26
|
const transpileCache = new Map();
|
|
26
27
|
// Setup veryfront modules as globals for compiled binary support
|
|
@@ -40,6 +41,7 @@ async function ensureVeryfrontGlobals() {
|
|
|
40
41
|
"veryfront/embedding": embeddingMod,
|
|
41
42
|
"veryfront/workflow": workflowMod,
|
|
42
43
|
"veryfront/work": workMod,
|
|
44
|
+
"veryfront/eval": evalMod,
|
|
43
45
|
"veryfront/schemas": schemasMod,
|
|
44
46
|
};
|
|
45
47
|
dntShim.dntGlobalThis.__VERYFRONT_MODULES__ = modules;
|
|
@@ -11,6 +11,7 @@ import type { Tool } from "../tool/types.js";
|
|
|
11
11
|
import type { WorkDefinition } from "../work/index.js";
|
|
12
12
|
import type { Workflow } from "../workflow/types.js";
|
|
13
13
|
import type { TaskDefinition } from "../task/types.js";
|
|
14
|
+
import type { EvalDefinition } from "../eval/index.js";
|
|
14
15
|
import type { Platform } from "../platform/core-platform.js";
|
|
15
16
|
import type { FileSystemAdapter } from "../platform/adapters/base.js";
|
|
16
17
|
/**
|
|
@@ -38,6 +39,7 @@ export interface DiscoveryConfig {
|
|
|
38
39
|
workflowDirs?: string[];
|
|
39
40
|
workDirs?: string[];
|
|
40
41
|
taskDirs?: string[];
|
|
42
|
+
evalDirs?: string[];
|
|
41
43
|
verbose?: boolean;
|
|
42
44
|
fsAdapter?: FileSystemAdapter;
|
|
43
45
|
}
|
|
@@ -53,6 +55,7 @@ export interface DiscoveryResult {
|
|
|
53
55
|
workflows: Map<string, Workflow>;
|
|
54
56
|
works: Map<string, WorkDefinition>;
|
|
55
57
|
tasks: Map<string, TaskDefinition>;
|
|
58
|
+
evals: Map<string, EvalDefinition>;
|
|
56
59
|
errors: Array<{
|
|
57
60
|
file: string;
|
|
58
61
|
error: Error;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/src/discovery/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AAEtE;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,QAAQ,CAAC,EAAE;QACT,EAAE,EAAE,cAAc,SAAS,CAAC,CAAC;QAC7B,IAAI,EAAE,cAAc,WAAW,CAAC,CAAC;KAClC,CAAC;IACF,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACzB,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC3B,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC3B,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACjC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACjC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IACnC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IACnC,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,KAAK,CAAA;KAAE,CAAC,CAAC;CAC/C;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB,CAAC,CAAC;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC;IACvC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC;IACtD,QAAQ,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC,CAAC;IAChE,YAAY,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;CAC3D"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/src/discovery/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AAEtE;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,QAAQ,CAAC,EAAE;QACT,EAAE,EAAE,cAAc,SAAS,CAAC,CAAC;QAC7B,IAAI,EAAE,cAAc,WAAW,CAAC,CAAC;KAClC,CAAC;IACF,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACzB,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC3B,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC3B,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACjC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACjC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IACnC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IACnC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IACnC,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,KAAK,CAAA;KAAE,CAAC,CAAC;CAC/C;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB,CAAC,CAAC;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC;IACvC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC;IACtD,QAAQ,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC,CAAC;IAChE,YAAY,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;CAC3D"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { EvalDataset, EvalExampleInput } from "./types.js";
|
|
2
|
+
/** Dataset factories for inline, JSON, and JSONL eval examples. */
|
|
3
|
+
export declare const datasets: {
|
|
4
|
+
readonly inline: (examples: readonly EvalExampleInput[]) => EvalDataset;
|
|
5
|
+
readonly json: (path: string) => EvalDataset;
|
|
6
|
+
readonly jsonl: (path: string) => EvalDataset;
|
|
7
|
+
};
|
|
8
|
+
//# sourceMappingURL=datasets.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"datasets.d.ts","sourceRoot":"","sources":["../../../src/src/eval/datasets.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAsChE,mEAAmE;AACnE,eAAO,MAAM,QAAQ;gCACF,SAAS,gBAAgB,EAAE,KAAG,WAAW;0BAW/C,MAAM,KAAG,WAAW;2BAYnB,MAAM,KAAG,WAAW;CAWxB,CAAC"}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import * as dntShim from "../../_dnt.shims.js";
|
|
2
|
+
import { isAbsolute, join } from "../../deps/jsr.io/@std/path/1.1.4/mod.js";
|
|
3
|
+
import { createEvalValidationError, normalizeEvalExamples } from "./validation.js";
|
|
4
|
+
function resolveDatasetPath(baseDir, filePath) {
|
|
5
|
+
return isAbsolute(filePath) ? filePath : join(baseDir, filePath);
|
|
6
|
+
}
|
|
7
|
+
function parseJsonDataset(text, source) {
|
|
8
|
+
const parsed = JSON.parse(text);
|
|
9
|
+
if (Array.isArray(parsed))
|
|
10
|
+
return parsed;
|
|
11
|
+
if (parsed &&
|
|
12
|
+
typeof parsed === "object" &&
|
|
13
|
+
Array.isArray(parsed.examples)) {
|
|
14
|
+
return parsed.examples;
|
|
15
|
+
}
|
|
16
|
+
throw createEvalValidationError(`${source} must contain an array or an object with examples`);
|
|
17
|
+
}
|
|
18
|
+
function parseJsonlDataset(text, source) {
|
|
19
|
+
return text
|
|
20
|
+
.split(/\r?\n/)
|
|
21
|
+
.map((line) => line.trim())
|
|
22
|
+
.filter(Boolean)
|
|
23
|
+
.map((line, index) => {
|
|
24
|
+
try {
|
|
25
|
+
return JSON.parse(line);
|
|
26
|
+
}
|
|
27
|
+
catch (error) {
|
|
28
|
+
throw createEvalValidationError(`${source}:${index + 1} must be valid JSON (${error instanceof Error ? error.message : error})`);
|
|
29
|
+
}
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
/** Dataset factories for inline, JSON, and JSONL eval examples. */
|
|
33
|
+
export const datasets = {
|
|
34
|
+
inline(examples) {
|
|
35
|
+
const normalized = normalizeEvalExamples(examples, "inline dataset");
|
|
36
|
+
return {
|
|
37
|
+
kind: "inline",
|
|
38
|
+
examples: normalized,
|
|
39
|
+
async load() {
|
|
40
|
+
return normalized.map((example) => ({ ...example }));
|
|
41
|
+
},
|
|
42
|
+
};
|
|
43
|
+
},
|
|
44
|
+
json(path) {
|
|
45
|
+
return {
|
|
46
|
+
kind: "json",
|
|
47
|
+
path,
|
|
48
|
+
async load({ baseDir }) {
|
|
49
|
+
const resolvedPath = resolveDatasetPath(baseDir, path);
|
|
50
|
+
const text = await dntShim.Deno.readTextFile(resolvedPath);
|
|
51
|
+
return normalizeEvalExamples(parseJsonDataset(text, path), path);
|
|
52
|
+
},
|
|
53
|
+
};
|
|
54
|
+
},
|
|
55
|
+
jsonl(path) {
|
|
56
|
+
return {
|
|
57
|
+
kind: "jsonl",
|
|
58
|
+
path,
|
|
59
|
+
async load({ baseDir }) {
|
|
60
|
+
const resolvedPath = resolveDatasetPath(baseDir, path);
|
|
61
|
+
const text = await dntShim.Deno.readTextFile(resolvedPath);
|
|
62
|
+
return normalizeEvalExamples(parseJsonlDataset(text, path), path);
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
},
|
|
66
|
+
};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Eval discovery for project-local eval definitions.
|
|
3
|
+
*/
|
|
4
|
+
import type { VeryfrontConfig } from "../config/index.js";
|
|
5
|
+
import type { RuntimeAdapter } from "../platform/index.js";
|
|
6
|
+
import type { EvalDefinition } from "./types.js";
|
|
7
|
+
/** Eval definition discovered from project source. */
|
|
8
|
+
export interface DiscoveredEval {
|
|
9
|
+
id: string;
|
|
10
|
+
name: string;
|
|
11
|
+
filePath: string;
|
|
12
|
+
exportName: string;
|
|
13
|
+
definition: EvalDefinition;
|
|
14
|
+
}
|
|
15
|
+
/** Loader used to import an eval source module during discovery. */
|
|
16
|
+
export type EvalModuleLoader = (filePath: string, options: {
|
|
17
|
+
adapter: RuntimeAdapter;
|
|
18
|
+
projectDir: string;
|
|
19
|
+
}) => Promise<Record<string, unknown>>;
|
|
20
|
+
/** Options for project-local eval discovery. */
|
|
21
|
+
export interface EvalDiscoveryOptions {
|
|
22
|
+
projectDir: string;
|
|
23
|
+
adapter: RuntimeAdapter;
|
|
24
|
+
config?: VeryfrontConfig;
|
|
25
|
+
evalsDir?: string;
|
|
26
|
+
/** @internal Override source loading for tests and custom runtimes. */
|
|
27
|
+
moduleLoader?: EvalModuleLoader;
|
|
28
|
+
}
|
|
29
|
+
/** Result returned by eval discovery. */
|
|
30
|
+
export interface EvalDiscoveryResult {
|
|
31
|
+
evals: DiscoveredEval[];
|
|
32
|
+
errors: Array<{
|
|
33
|
+
filePath: string;
|
|
34
|
+
error: string;
|
|
35
|
+
}>;
|
|
36
|
+
}
|
|
37
|
+
/** Derive the stable `eval:<path>` ID for an eval file. */
|
|
38
|
+
export declare function deriveEvalId(filePath: string, evalsDir: string): string;
|
|
39
|
+
/** Discover eval definitions from a project eval directory. */
|
|
40
|
+
export declare function discoverEvals(options: EvalDiscoveryOptions): Promise<EvalDiscoveryResult>;
|
|
41
|
+
/** Discover and return one eval definition by ID. */
|
|
42
|
+
export declare function findEvalById(evalId: string, options: EvalDiscoveryOptions): Promise<DiscoveredEval | null>;
|
|
43
|
+
//# sourceMappingURL=discovery.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"discovery.d.ts","sourceRoot":"","sources":["../../../src/src/eval/discovery.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAI3D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAWjD,sDAAsD;AACtD,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,cAAc,CAAC;CAC5B;AAED,oEAAoE;AACpE,MAAM,MAAM,gBAAgB,GAAG,CAC7B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE;IAAE,OAAO,EAAE,cAAc,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,KACrD,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAEtC,gDAAgD;AAChD,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,cAAc,CAAC;IACxB,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uEAAuE;IACvE,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC;AAED,yCAAyC;AACzC,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,MAAM,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACpD;AA0BD,2DAA2D;AAC3D,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAQvE;AAkED,+DAA+D;AAC/D,wBAAsB,aAAa,CACjC,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,mBAAmB,CAAC,CAsC9B;AAED,qDAAqD;AACrD,wBAAsB,YAAY,CAChC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAGhC"}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Eval discovery for project-local eval definitions.
|
|
3
|
+
*/
|
|
4
|
+
import { join } from "../../deps/jsr.io/@std/path/1.1.4/mod.js";
|
|
5
|
+
import { importDiscoveryModule } from "../discovery/module-import.js";
|
|
6
|
+
import { collectFiles } from "../utils/file-discovery.js";
|
|
7
|
+
import { isEvalDefinition } from "./factory.js";
|
|
8
|
+
const EVAL_FILE_EXTENSIONS = [".ts", ".tsx", ".js", ".jsx"];
|
|
9
|
+
const EVAL_IGNORE_PATTERNS = [
|
|
10
|
+
"node_modules",
|
|
11
|
+
".git",
|
|
12
|
+
"__tests__",
|
|
13
|
+
"*.test.*",
|
|
14
|
+
"*.spec.*",
|
|
15
|
+
];
|
|
16
|
+
function resolveEvalsBaseDir(projectDir, evalsDir, config) {
|
|
17
|
+
const fsType = config?.fs?.type ?? "local";
|
|
18
|
+
return fsType === "github" || fsType === "veryfront-api" ? evalsDir : join(projectDir, evalsDir);
|
|
19
|
+
}
|
|
20
|
+
function toErrorMessage(error) {
|
|
21
|
+
return error instanceof Error ? error.message : String(error);
|
|
22
|
+
}
|
|
23
|
+
function stripEvalExtension(relativePath) {
|
|
24
|
+
return relativePath
|
|
25
|
+
.replace(/\.(ts|tsx|js|jsx)$/, "")
|
|
26
|
+
.replace(/\.eval$/, "");
|
|
27
|
+
}
|
|
28
|
+
function stripFileProtocol(path) {
|
|
29
|
+
if (!path.startsWith("file://"))
|
|
30
|
+
return path;
|
|
31
|
+
return decodeURIComponent(new URL(path).pathname);
|
|
32
|
+
}
|
|
33
|
+
/** Derive the stable `eval:<path>` ID for an eval file. */
|
|
34
|
+
export function deriveEvalId(filePath, evalsDir) {
|
|
35
|
+
let relative = stripFileProtocol(filePath);
|
|
36
|
+
const normalizedDir = stripFileProtocol(evalsDir);
|
|
37
|
+
const dirPrefix = normalizedDir.endsWith("/") ? normalizedDir : `${normalizedDir}/`;
|
|
38
|
+
if (relative.startsWith(dirPrefix)) {
|
|
39
|
+
relative = relative.slice(dirPrefix.length);
|
|
40
|
+
}
|
|
41
|
+
return `eval:${stripEvalExtension(relative)}`;
|
|
42
|
+
}
|
|
43
|
+
function extractEvalExport(module) {
|
|
44
|
+
const defaultExport = module.default;
|
|
45
|
+
if (isEvalDefinition(defaultExport)) {
|
|
46
|
+
return { exportName: "default", definition: defaultExport };
|
|
47
|
+
}
|
|
48
|
+
for (const [exportName, value] of Object.entries(module)) {
|
|
49
|
+
if (exportName === "default")
|
|
50
|
+
continue;
|
|
51
|
+
if (!isEvalDefinition(value))
|
|
52
|
+
continue;
|
|
53
|
+
return { exportName, definition: value };
|
|
54
|
+
}
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
async function collectEvalFiles(baseDir, adapter) {
|
|
58
|
+
return await collectFiles({
|
|
59
|
+
baseDir,
|
|
60
|
+
extensions: [...EVAL_FILE_EXTENSIONS],
|
|
61
|
+
recursive: true,
|
|
62
|
+
ignorePatterns: [...EVAL_IGNORE_PATTERNS],
|
|
63
|
+
adapter,
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
async function loadEvalFromFile(filePath, fallbackId, adapter, projectDir, moduleLoader) {
|
|
67
|
+
const module = await moduleLoader(filePath, {
|
|
68
|
+
adapter,
|
|
69
|
+
projectDir,
|
|
70
|
+
});
|
|
71
|
+
const evalExport = extractEvalExport(module);
|
|
72
|
+
if (!evalExport)
|
|
73
|
+
return null;
|
|
74
|
+
const id = evalExport.definition.id || fallbackId;
|
|
75
|
+
const definition = {
|
|
76
|
+
...evalExport.definition,
|
|
77
|
+
id,
|
|
78
|
+
name: evalExport.definition.name || id,
|
|
79
|
+
source: {
|
|
80
|
+
filePath,
|
|
81
|
+
exportName: evalExport.exportName,
|
|
82
|
+
},
|
|
83
|
+
};
|
|
84
|
+
return {
|
|
85
|
+
id,
|
|
86
|
+
name: definition.name,
|
|
87
|
+
filePath,
|
|
88
|
+
exportName: evalExport.exportName,
|
|
89
|
+
definition,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
/** Discover eval definitions from a project eval directory. */
|
|
93
|
+
export async function discoverEvals(options) {
|
|
94
|
+
const { projectDir, adapter, config, evalsDir = "evals", moduleLoader = importDiscoveryModule, } = options;
|
|
95
|
+
const evals = [];
|
|
96
|
+
const errors = [];
|
|
97
|
+
const baseDir = resolveEvalsBaseDir(projectDir, evalsDir, config);
|
|
98
|
+
try {
|
|
99
|
+
if (!await adapter.fs.exists(baseDir)) {
|
|
100
|
+
return { evals, errors };
|
|
101
|
+
}
|
|
102
|
+
const files = await collectEvalFiles(baseDir, adapter);
|
|
103
|
+
for (const file of files) {
|
|
104
|
+
try {
|
|
105
|
+
const evalItem = await loadEvalFromFile(file.path, deriveEvalId(file.path, baseDir), adapter, projectDir, moduleLoader);
|
|
106
|
+
if (evalItem)
|
|
107
|
+
evals.push(evalItem);
|
|
108
|
+
}
|
|
109
|
+
catch (error) {
|
|
110
|
+
errors.push({ filePath: file.path, error: toErrorMessage(error) });
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
catch (error) {
|
|
115
|
+
errors.push({ filePath: baseDir, error: toErrorMessage(error) });
|
|
116
|
+
}
|
|
117
|
+
return { evals, errors };
|
|
118
|
+
}
|
|
119
|
+
/** Discover and return one eval definition by ID. */
|
|
120
|
+
export async function findEvalById(evalId, options) {
|
|
121
|
+
const result = await discoverEvals(options);
|
|
122
|
+
return result.evals.find((item) => item.id === evalId) ?? null;
|
|
123
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { EvalDefinition, EvalExample, EvalExpect, EvalMetricResult, EvalRecord } from "./types.js";
|
|
2
|
+
export declare function createEvalExpect(record: EvalRecord, checks: EvalMetricResult[]): EvalExpect;
|
|
3
|
+
export declare function createEvalCheckContext(input: {
|
|
4
|
+
definition: EvalDefinition;
|
|
5
|
+
example: EvalExample;
|
|
6
|
+
repetition: number;
|
|
7
|
+
record: EvalRecord;
|
|
8
|
+
checks: EvalMetricResult[];
|
|
9
|
+
}): {
|
|
10
|
+
definition: EvalDefinition;
|
|
11
|
+
example: EvalExample;
|
|
12
|
+
repetition: number;
|
|
13
|
+
record: EvalRecord;
|
|
14
|
+
expect: EvalExpect;
|
|
15
|
+
};
|
|
16
|
+
//# sourceMappingURL=expect.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"expect.d.ts","sourceRoot":"","sources":["../../../src/src/eval/expect.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,cAAc,EACd,WAAW,EACX,UAAU,EAEV,gBAAgB,EAEhB,UAAU,EAEX,MAAM,YAAY,CAAC;AAuBpB,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,gBAAgB,EAAE,GAAG,UAAU,CAqC3F;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE;IAC5C,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,WAAW,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,UAAU,CAAC;IACnB,MAAM,EAAE,gBAAgB,EAAE,CAAC;CAC5B;;;;;;EAQA"}
|