scorecard-ai-mcp 1.0.0-alpha.9 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +18 -14
  2. package/package.json +2 -2
  3. package/server.js +1 -1
  4. package/server.js.map +1 -1
  5. package/server.mjs +1 -1
  6. package/server.mjs.map +1 -1
  7. package/src/server.ts +1 -1
  8. package/src/tools/index.ts +8 -6
  9. package/src/tools/metrics/create-metrics.ts +251 -0
  10. package/src/tools/runs/create-runs.ts +2 -2
  11. package/src/tools/systems/delete-systems.ts +1 -1
  12. package/src/tools/systems/versions/create-systems-versions.ts +45 -0
  13. package/src/tools/{system-configs/get-system-configs.ts → systems/versions/get-systems-versions.ts} +9 -9
  14. package/src/tools/{system-configs/list-system-configs.ts → systems/versions/list-systems-versions.ts} +6 -6
  15. package/src/tools/testsets/get-testsets.ts +1 -1
  16. package/tools/index.d.mts.map +1 -1
  17. package/tools/index.d.ts.map +1 -1
  18. package/tools/index.js +8 -6
  19. package/tools/index.js.map +1 -1
  20. package/tools/index.mjs +8 -6
  21. package/tools/index.mjs.map +1 -1
  22. package/tools/{system-configs/get-system-configs.d.mts → metrics/create-metrics.d.mts} +1 -1
  23. package/tools/metrics/create-metrics.d.mts.map +1 -0
  24. package/tools/{system-configs/get-system-configs.d.ts → metrics/create-metrics.d.ts} +1 -1
  25. package/tools/metrics/create-metrics.d.ts.map +1 -0
  26. package/tools/metrics/create-metrics.js +245 -0
  27. package/tools/metrics/create-metrics.js.map +1 -0
  28. package/tools/metrics/create-metrics.mjs +241 -0
  29. package/tools/metrics/create-metrics.mjs.map +1 -0
  30. package/tools/runs/create-runs.js +2 -2
  31. package/tools/runs/create-runs.js.map +1 -1
  32. package/tools/runs/create-runs.mjs +2 -2
  33. package/tools/runs/create-runs.mjs.map +1 -1
  34. package/tools/systems/delete-systems.js +1 -1
  35. package/tools/systems/delete-systems.js.map +1 -1
  36. package/tools/systems/delete-systems.mjs +1 -1
  37. package/tools/systems/delete-systems.mjs.map +1 -1
  38. package/tools/systems/versions/create-systems-versions.d.mts +32 -0
  39. package/tools/systems/versions/create-systems-versions.d.mts.map +1 -0
  40. package/tools/systems/versions/create-systems-versions.d.ts +32 -0
  41. package/tools/systems/versions/create-systems-versions.d.ts.map +1 -0
  42. package/tools/systems/versions/create-systems-versions.js +40 -0
  43. package/tools/systems/versions/create-systems-versions.js.map +1 -0
  44. package/tools/systems/versions/create-systems-versions.mjs +36 -0
  45. package/tools/systems/versions/create-systems-versions.mjs.map +1 -0
  46. package/tools/systems/versions/get-systems-versions.d.mts +32 -0
  47. package/tools/systems/versions/get-systems-versions.d.mts.map +1 -0
  48. package/tools/{system-configs/create-system-configs.d.ts → systems/versions/get-systems-versions.d.ts} +2 -2
  49. package/tools/systems/versions/get-systems-versions.d.ts.map +1 -0
  50. package/tools/{system-configs/get-system-configs.js → systems/versions/get-systems-versions.js} +9 -9
  51. package/tools/systems/versions/get-systems-versions.js.map +1 -0
  52. package/tools/{system-configs/get-system-configs.mjs → systems/versions/get-systems-versions.mjs} +9 -9
  53. package/tools/systems/versions/get-systems-versions.mjs.map +1 -0
  54. package/tools/systems/versions/list-systems-versions.d.mts +32 -0
  55. package/tools/systems/versions/list-systems-versions.d.mts.map +1 -0
  56. package/tools/{system-configs/create-system-configs.d.mts → systems/versions/list-systems-versions.d.ts} +2 -2
  57. package/tools/systems/versions/list-systems-versions.d.ts.map +1 -0
  58. package/tools/{system-configs/list-system-configs.js → systems/versions/list-systems-versions.js} +6 -6
  59. package/tools/systems/versions/list-systems-versions.js.map +1 -0
  60. package/tools/{system-configs/list-system-configs.mjs → systems/versions/list-systems-versions.mjs} +6 -6
  61. package/tools/systems/versions/list-systems-versions.mjs.map +1 -0
  62. package/tools/testsets/get-testsets.js +1 -1
  63. package/tools/testsets/get-testsets.js.map +1 -1
  64. package/tools/testsets/get-testsets.mjs +1 -1
  65. package/tools/testsets/get-testsets.mjs.map +1 -1
  66. package/src/tools/system-configs/create-system-configs.ts +0 -64
  67. package/tools/system-configs/create-system-configs.d.mts.map +0 -1
  68. package/tools/system-configs/create-system-configs.d.ts.map +0 -1
  69. package/tools/system-configs/create-system-configs.js +0 -58
  70. package/tools/system-configs/create-system-configs.js.map +0 -1
  71. package/tools/system-configs/create-system-configs.mjs +0 -54
  72. package/tools/system-configs/create-system-configs.mjs.map +0 -1
  73. package/tools/system-configs/get-system-configs.d.mts.map +0 -1
  74. package/tools/system-configs/get-system-configs.d.ts.map +0 -1
  75. package/tools/system-configs/get-system-configs.js.map +0 -1
  76. package/tools/system-configs/get-system-configs.mjs.map +0 -1
  77. package/tools/system-configs/list-system-configs.d.mts +0 -32
  78. package/tools/system-configs/list-system-configs.d.mts.map +0 -1
  79. package/tools/system-configs/list-system-configs.d.ts +0 -32
  80. package/tools/system-configs/list-system-configs.d.ts.map +0 -1
  81. package/tools/system-configs/list-system-configs.js.map +0 -1
  82. package/tools/system-configs/list-system-configs.mjs.map +0 -1
package/README.md CHANGED
@@ -187,7 +187,7 @@ The following tools are available in this MCP server.
187
187
 
188
188
  - `list_testsets` (`read`): Retrieve a paginated list of Testsets belonging to a Project.
189
189
  - `delete_testsets` (`write`): Delete Testset
190
- - `get_testsets` (`read`): Get Testset by ID
190
+ - `get_testsets` (`read`): Get Testset
191
191
 
192
192
  ### Resource `testcases`:
193
193
 
@@ -201,6 +201,10 @@ The following tools are available in this MCP server.
201
201
 
202
202
  - `create_runs` (`write`): Create a new Run.
203
203
 
204
+ ### Resource `metrics`:
205
+
206
+ - `create_metrics` (`write`): Create a new Metric for evaluating system outputs.
207
+
204
208
  ### Resource `records`:
205
209
 
206
210
  - `create_records` (`write`): Create a new Record in a Run.
@@ -232,25 +236,25 @@ The following tools are available in this MCP server.
232
236
  - For significant redesigns, creating a new system definition provides a cleaner separation
233
237
 
234
238
  - `list_systems` (`read`): Retrieve a paginated list of all systems. Systems are ordered by creation date.
235
- - `delete_systems` (`write`): Delete a system definition by ID. This will not delete associated system configurations.
239
+ - `delete_systems` (`write`): Delete a system definition by ID. This will not delete associated system versions.
236
240
  - `get_systems` (`read`): Retrieve a specific system by ID.
237
241
 
238
- ### Resource `system_configs`:
242
+ ### Resource `systems.versions`:
239
243
 
240
- - `create_system_configs` (`write`): Create a new configuration for a system.
244
+ - `create_systems_versions` (`write`): Create a new version for a system.
241
245
 
242
- Each configuration contains specific parameter values that match the system's configSchema - things like model parameters, thresholds, or processing options.
243
- Once created, configurations cannot be modified, ensuring stable reference points for evaluations.
246
+ Each version contains specific parameter values that match the system's `configSchema` - things like model parameters, thresholds, or processing options.
247
+ Once created, versions cannot be modified, ensuring stable reference points for evaluations.
244
248
 
245
- When creating a configuration:
249
+ When creating a system version:
246
250
 
247
- - The 'config' object is validated against the parent system's configSchema
248
- - Configurations with validation errors are still stored, with errors included in the response
249
- - Validation errors indicate fields that don't match the schema but don't prevent creation
250
- - Having validation errors may affect how some evaluation metrics are calculated
251
+ - The `config` object is validated against the parent system's `configSchema`.
252
+ - System versions with validation errors are still stored, with errors included in the response.
253
+ - Validation errors indicate fields that don't match the schema but don't prevent creation.
254
+ - Having validation errors may affect how some evaluation metrics are calculated.
251
255
 
252
- - `list_system_configs` (`read`): Retrieve a paginated list of configurations for a specific system.
256
+ - `list_systems_versions` (`read`): Retrieve a paginated list of system versions for a specific system.
253
257
 
254
- System configurations provide concrete parameter values for a System Under Test, defining exactly how the system should be configured during an evaluation run.
258
+ System versions provide concrete parameter values for a System Under Test, defining exactly how the system should be configured during an evaluation run.
255
259
 
256
- - `get_system_configs` (`read`): Retrieve a specific system configuration by ID.
260
+ - `get_systems_versions` (`read`): Retrieve a specific system version by ID.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "scorecard-ai-mcp",
3
- "version": "1.0.0-alpha.9",
3
+ "version": "1.0.0",
4
4
  "description": "The official MCP Server for the Scorecard API",
5
5
  "author": "Scorecard <team@scorecard.io>",
6
6
  "types": "./index.d.ts",
@@ -24,7 +24,7 @@
24
24
  "fix": "eslint --fix --ext ts,js ."
25
25
  },
26
26
  "dependencies": {
27
- "scorecard-ai": "^1.0.0-alpha.9",
27
+ "scorecard-ai": "^1.0.0",
28
28
  "@modelcontextprotocol/sdk": "^1.6.1",
29
29
  "yargs": "^17.7.2",
30
30
  "@cloudflare/cabidela": "^0.2.4",
package/server.js CHANGED
@@ -19,7 +19,7 @@ Object.defineProperty(exports, "endpoints", { enumerable: true, get: function ()
19
19
  // Create server instance
20
20
  exports.server = new mcp_js_1.McpServer({
21
21
  name: 'scorecard_ai_api',
22
- version: '1.0.0-alpha.9',
22
+ version: '1.0.0',
23
23
  }, {
24
24
  capabilities: {
25
25
  tools: {},
package/server.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"server.js","sourceRoot":"","sources":["src/server.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;;;;AAkCtF,oBA6BC;AAKD,kCAkBC;AAKD,wCAYC;AApGD,oEAAoE;AACpE,sCAAsE;AACtE,iEAAyG;AACzG,gEAAqC;AACrC,wCAKkB;AAClB,sDAA+C;AAE/C,oCAAoC;AAA3B,kGAAA,SAAS,OAAA;AAElB,yBAAyB;AACZ,QAAA,MAAM,GAAG,IAAI,kBAAS,CACjC;IACE,IAAI,EAAE,kBAAkB;IACxB,OAAO,EAAE,eAAe;CACzB,EACD;IACE,YAAY,EAAE;QACZ,KAAK,EAAE,EAAE;KACV;CACF,CACF,CAAC;AAEF;;;GAGG;AACH,SAAgB,IAAI,CAAC,MAKpB;IACC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,YAAY,kBAAS,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;IACzF,MAAM,iBAAiB,GAAG,MAAM,CAAC,SAAS,IAAI,iBAAS,CAAC;IAExD,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;IAE5G,MAAM,MAAM,GACV,MAAM,CAAC,MAAM,IAAI,IAAI,sBAAS,CAAC,EAAE,WAAW,EAAE,CAAC,IAAA,eAAO,EAAC,uBAAuB,CAAC,IAAI,SAAS,CAAQ,EAAE,CAAC,CAAC;IAE1G,MAAM,CAAC,iBAAiB,CAAC,iCAAsB,EAAE,KAAK,IAAI,EAAE;QAC1D,OAAO;YACL,KAAK,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC;SAC1D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,iBAAiB,CAAC,gCAAqB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QAChE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;QACjD,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO,cAAc,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;IAC5F,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAgB,WAAW,CAAC,SAAqB,EAAE,OAAsB;IACvE,MAAM,iBAAiB,GAAG,IAAA,aAAK,EAAC,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAE5D,MAAM,aAAa,GAAG,iBAAiB,CAAC;IAExC,IAAI,OAAO,CAAC,eAAe,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1D,aAAa,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IACnC,CAAC;IAED,IAAI,OAAO,CAAC,mBAAmB,EAAE,CAAC;QAChC,aAAa,CAAC,IAAI,CAAC,GAAG,IAAA,4BAAY,EAAC,SAAS,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,aAAa,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,IAAA,0CAAiC,EAAC,aAAa,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;AAChF,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,cAAc,CAClC,IAAU,EACV,OAAwB,EACxB,MAAiB,EACjB,IAAyC,EACzC,oBAAkD;IAElD,MAAM,OAAO,GAAG,EAAE,GAAG,kCAAyB,EAAE,GAAG,oBAAoB,EAAE,CAAC;IAC1E,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;QAC9B,IAAI,GAAG,IAAA,0BAAiB,EAAC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;AAC3C,CAAC;AAEM,MAAM,OAAO,GAAG,CAAC,GAAW,EAAsB,EAAE;IACzD,IAAI,OAAQ,UAAkB,CAAC,OAAO,KAAK,WAAW,EAAE,CAAC;QACvD,OAAQ,UAAkB,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC;IACxD,CAAC;SAAM,IAAI,OAAQ,UAAkB,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC3D,OAAQ,UAAkB,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC;IAC1D,CAAC;IACD,OAAO;AACT,CAAC,CAAC;AAPW,QAAA,OAAO,WAOlB;AAEK,MAAM,cAAc,GAAG,CAAC,GAAW,EAAU,EAAE;IACpD,IAAI,QAAQ,GAAG,IAAA,eAAO,EAAC,GAAG,CAAC,CAAC;IAC5B,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,aAAa,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AANW,QAAA,cAAc,kBAMzB"}
1
+ {"version":3,"file":"server.js","sourceRoot":"","sources":["src/server.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;;;;AAkCtF,oBA6BC;AAKD,kCAkBC;AAKD,wCAYC;AApGD,oEAAoE;AACpE,sCAAsE;AACtE,iEAAyG;AACzG,gEAAqC;AACrC,wCAKkB;AAClB,sDAA+C;AAE/C,oCAAoC;AAA3B,kGAAA,SAAS,OAAA;AAElB,yBAAyB;AACZ,QAAA,MAAM,GAAG,IAAI,kBAAS,CACjC;IACE,IAAI,EAAE,kBAAkB;IACxB,OAAO,EAAE,OAAO;CACjB,EACD;IACE,YAAY,EAAE;QACZ,KAAK,EAAE,EAAE;KACV;CACF,CACF,CAAC;AAEF;;;GAGG;AACH,SAAgB,IAAI,CAAC,MAKpB;IACC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,YAAY,kBAAS,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;IACzF,MAAM,iBAAiB,GAAG,MAAM,CAAC,SAAS,IAAI,iBAAS,CAAC;IAExD,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;IAE5G,MAAM,MAAM,GACV,MAAM,CAAC,MAAM,IAAI,IAAI,sBAAS,CAAC,EAAE,WAAW,EAAE,CAAC,IAAA,eAAO,EAAC,uBAAuB,CAAC,IAAI,SAAS,CAAQ,EAAE,CAAC,CAAC;IAE1G,MAAM,CAAC,iBAAiB,CAAC,iCAAsB,EAAE,KAAK,IAAI,EAAE;QAC1D,OAAO;YACL,KAAK,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC;SAC1D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,iBAAiB,CAAC,gCAAqB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QAChE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;QACjD,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO,cAAc,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;IAC5F,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAgB,WAAW,CAAC,SAAqB,EAAE,OAAsB;IACvE,MAAM,iBAAiB,GAAG,IAAA,aAAK,EAAC,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAE5D,MAAM,aAAa,GAAG,iBAAiB,CAAC;IAExC,IAAI,OAAO,CAAC,eAAe,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1D,aAAa,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IACnC,CAAC;IAED,IAAI,OAAO,CAAC,mBAAmB,EAAE,CAAC;QAChC,aAAa,CAAC,IAAI,CAAC,GAAG,IAAA,4BAAY,EAAC,SAAS,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,aAAa,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,IAAA,0CAAiC,EAAC,aAAa,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;AAChF,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,cAAc,CAClC,IAAU,EACV,OAAwB,EACxB,MAAiB,EACjB,IAAyC,EACzC,oBAAkD;IAElD,MAAM,OAAO,GAAG,EAAE,GAAG,kCAAyB,EAAE,GAAG,oBAAoB,EAAE,CAAC;IAC1E,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;QAC9B,IAAI,GAAG,IAAA,0BAAiB,EAAC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;AAC3C,CAAC;AAEM,MAAM,OAAO,GAAG,CAAC,GAAW,EAAsB,EAAE;IACzD,IAAI,OAAQ,UAAkB,CAAC,OAAO,KAAK,WAAW,EAAE,CAAC;QACvD,OAAQ,UAAkB,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC;IACxD,CAAC;SAAM,IAAI,OAAQ,UAAkB,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC3D,OAAQ,UAAkB,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC;IAC1D,CAAC;IACD,OAAO;AACT,CAAC,CAAC;AAPW,QAAA,OAAO,WAOlB;AAEK,MAAM,cAAc,GAAG,CAAC,GAAW,EAAU,EAAE;IACpD,IAAI,QAAQ,GAAG,IAAA,eAAO,EAAC,GAAG,CAAC,CAAC;IAC5B,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,aAAa,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AANW,QAAA,cAAc,kBAMzB"}
package/server.mjs CHANGED
@@ -9,7 +9,7 @@ export { endpoints } from "./tools.mjs";
9
9
  // Create server instance
10
10
  export const server = new McpServer({
11
11
  name: 'scorecard_ai_api',
12
- version: '1.0.0-alpha.9',
12
+ version: '1.0.0',
13
13
  }, {
14
14
  capabilities: {
15
15
  tools: {},
package/server.mjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"server.mjs","sourceRoot":"","sources":["src/server.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAG/E,EAAE,SAAS,EAAE,MAAM,yCAAyC;OAC5D,EAAY,SAAS,EAAmB,KAAK,EAAE;OAC/C,EAAE,qBAAqB,EAAE,sBAAsB,EAAQ,MAAM,oCAAoC;OACjG,SAAS,MAAM,cAAc;OAC7B,EACL,iCAAiC,EAEjC,yBAAyB,EACzB,iBAAiB,GAClB;OACM,EAAE,YAAY,EAAE;OAEhB,EAAE,SAAS,EAAE;AAEpB,yBAAyB;AACzB,MAAM,CAAC,MAAM,MAAM,GAAG,IAAI,SAAS,CACjC;IACE,IAAI,EAAE,kBAAkB;IACxB,OAAO,EAAE,eAAe;CACzB,EACD;IACE,YAAY,EAAE;QACZ,KAAK,EAAE,EAAE;KACV;CACF,CACF,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,IAAI,CAAC,MAKpB;IACC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,YAAY,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;IACzF,MAAM,iBAAiB,GAAG,MAAM,CAAC,SAAS,IAAI,SAAS,CAAC;IAExD,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;IAE5G,MAAM,MAAM,GACV,MAAM,CAAC,MAAM,IAAI,IAAI,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC,OAAO,CAAC,uBAAuB,CAAC,IAAI,SAAS,CAAQ,EAAE,CAAC,CAAC;IAE1G,MAAM,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE;QAC1D,OAAO;YACL,KAAK,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC;SAC1D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QAChE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;QACjD,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO,cAAc,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;IAC5F,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,SAAqB,EAAE,OAAsB;IACvE,MAAM,iBAAiB,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAE5D,MAAM,aAAa,GAAG,iBAAiB,CAAC;IAExC,IAAI,OAAO,CAAC,eAAe,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1D,aAAa,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IACnC,CAAC;IAED,IAAI,OAAO,CAAC,mBAAmB,EAAE,CAAC;QAChC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,aAAa,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,iCAAiC,CAAC,aAAa,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;AAChF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,IAAU,EACV,OAAwB,EACxB,MAAiB,EACjB,IAAyC,EACzC,oBAAkD;IAElD,MAAM,OAAO,GAAG,EAAE,GAAG,yBAAyB,EAAE,GAAG,oBAAoB,EAAE,CAAC;IAC1E,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;QAC9B,IAAI,GAAG,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,GAAW,EAAsB,EAAE;IACzD,IAAI,OAAQ,UAAkB,CAAC,OAAO,KAAK,WAAW,EAAE,CAAC;QACvD,OAAQ,UAAkB,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC;IACxD,CAAC;SAAM,IAAI,OAAQ,UAAkB,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC3D,OAAQ,UAAkB,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC;IAC1D,CAAC;IACD,OAAO;AACT,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,GAAW,EAAU,EAAE;IACpD,IAAI,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,aAAa,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC"}
1
+ {"version":3,"file":"server.mjs","sourceRoot":"","sources":["src/server.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAG/E,EAAE,SAAS,EAAE,MAAM,yCAAyC;OAC5D,EAAY,SAAS,EAAmB,KAAK,EAAE;OAC/C,EAAE,qBAAqB,EAAE,sBAAsB,EAAQ,MAAM,oCAAoC;OACjG,SAAS,MAAM,cAAc;OAC7B,EACL,iCAAiC,EAEjC,yBAAyB,EACzB,iBAAiB,GAClB;OACM,EAAE,YAAY,EAAE;OAEhB,EAAE,SAAS,EAAE;AAEpB,yBAAyB;AACzB,MAAM,CAAC,MAAM,MAAM,GAAG,IAAI,SAAS,CACjC;IACE,IAAI,EAAE,kBAAkB;IACxB,OAAO,EAAE,OAAO;CACjB,EACD;IACE,YAAY,EAAE;QACZ,KAAK,EAAE,EAAE;KACV;CACF,CACF,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,IAAI,CAAC,MAKpB;IACC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,YAAY,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;IACzF,MAAM,iBAAiB,GAAG,MAAM,CAAC,SAAS,IAAI,SAAS,CAAC;IAExD,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;IAE5G,MAAM,MAAM,GACV,MAAM,CAAC,MAAM,IAAI,IAAI,SAAS,CAAC,EAAE,WAAW,EAAE,CAAC,OAAO,CAAC,uBAAuB,CAAC,IAAI,SAAS,CAAQ,EAAE,CAAC,CAAC;IAE1G,MAAM,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE;QAC1D,OAAO;YACL,KAAK,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC;SAC1D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QAChE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;QACjD,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO,cAAc,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;IAC5F,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,SAAqB,EAAE,OAAsB;IACvE,MAAM,iBAAiB,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAE5D,MAAM,aAAa,GAAG,iBAAiB,CAAC;IAExC,IAAI,OAAO,CAAC,eAAe,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1D,aAAa,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IACnC,CAAC;IAED,IAAI,OAAO,CAAC,mBAAmB,EAAE,CAAC;QAChC,aAAa,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,aAAa,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,iCAAiC,CAAC,aAAa,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;AAChF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,IAAU,EACV,OAAwB,EACxB,MAAiB,EACjB,IAAyC,EACzC,oBAAkD;IAElD,MAAM,OAAO,GAAG,EAAE,GAAG,yBAAyB,EAAE,GAAG,oBAAoB,EAAE,CAAC;IAC1E,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;QAC9B,IAAI,GAAG,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,GAAW,EAAsB,EAAE;IACzD,IAAI,OAAQ,UAAkB,CAAC,OAAO,KAAK,WAAW,EAAE,CAAC;QACvD,OAAQ,UAAkB,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC;IACxD,CAAC;SAAM,IAAI,OAAQ,UAAkB,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC3D,OAAQ,UAAkB,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC;IAC1D,CAAC;IACD,OAAO;AACT,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,GAAW,EAAU,EAAE;IACpD,IAAI,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,aAAa,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC"}
package/src/server.ts CHANGED
@@ -19,7 +19,7 @@ export { endpoints } from './tools';
19
19
  export const server = new McpServer(
20
20
  {
21
21
  name: 'scorecard_ai_api',
22
- version: '1.0.0-alpha.9',
22
+ version: '1.0.0',
23
23
  },
24
24
  {
25
25
  capabilities: {
package/src/tools/index.ts CHANGED
@@ -17,6 +17,7 @@ import list_testcases from './testcases/list-testcases';
17
17
  import delete_testcases from './testcases/delete-testcases';
18
18
  import get_testcases from './testcases/get-testcases';
19
19
  import create_runs from './runs/create-runs';
20
+ import create_metrics from './metrics/create-metrics';
20
21
  import create_records from './records/create-records';
21
22
  import upsert_scores from './scores/upsert-scores';
22
23
  import create_systems from './systems/create-systems';
@@ -24,9 +25,9 @@ import update_systems from './systems/update-systems';
24
25
  import list_systems from './systems/list-systems';
25
26
  import delete_systems from './systems/delete-systems';
26
27
  import get_systems from './systems/get-systems';
27
- import create_system_configs from './system-configs/create-system-configs';
28
- import list_system_configs from './system-configs/list-system-configs';
29
- import get_system_configs from './system-configs/get-system-configs';
28
+ import create_systems_versions from './systems/versions/create-systems-versions';
29
+ import list_systems_versions from './systems/versions/list-systems-versions';
30
+ import get_systems_versions from './systems/versions/get-systems-versions';
30
31
 
31
32
  export const endpoints: Endpoint[] = [];
32
33
 
@@ -47,6 +48,7 @@ addEndpoint(list_testcases);
47
48
  addEndpoint(delete_testcases);
48
49
  addEndpoint(get_testcases);
49
50
  addEndpoint(create_runs);
51
+ addEndpoint(create_metrics);
50
52
  addEndpoint(create_records);
51
53
  addEndpoint(upsert_scores);
52
54
  addEndpoint(create_systems);
@@ -54,9 +56,9 @@ addEndpoint(update_systems);
54
56
  addEndpoint(list_systems);
55
57
  addEndpoint(delete_systems);
56
58
  addEndpoint(get_systems);
57
- addEndpoint(create_system_configs);
58
- addEndpoint(list_system_configs);
59
- addEndpoint(get_system_configs);
59
+ addEndpoint(create_systems_versions);
60
+ addEndpoint(list_systems_versions);
61
+ addEndpoint(get_systems_versions);
60
62
 
61
63
  export type Filter = {
62
64
  type: 'resource' | 'operation' | 'tag' | 'tool';
package/src/tools/metrics/create-metrics.ts ADDED
@@ -0,0 +1,251 @@
1
+ // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
2
+
3
+ import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
4
+
5
+ import { Tool } from '@modelcontextprotocol/sdk/types.js';
6
+ import type { Metadata } from '../';
7
+ import Scorecard from 'scorecard-ai';
8
+
9
+ export const metadata: Metadata = {
10
+ resource: 'metrics',
11
+ operation: 'write',
12
+ tags: [],
13
+ httpMethod: 'post',
14
+ httpPath: '/projects/{projectId}/metrics',
15
+ operationId: 'createMetric',
16
+ };
17
+
18
+ export const tool: Tool = {
19
+ name: 'create_metrics',
20
+ description: 'Create a new Metric for evaluating system outputs.',
21
+ inputSchema: {
22
+ type: 'object',
23
+ anyOf: [
24
+ {
25
+ type: 'object',
26
+ properties: {
27
+ projectId: {
28
+ type: 'string',
29
+ },
30
+ evalType: {
31
+ type: 'string',
32
+ description: 'AI-based evaluation type.',
33
+ enum: ['ai'],
34
+ },
35
+ name: {
36
+ type: 'string',
37
+ description: 'The name of the Metric.',
38
+ },
39
+ outputType: {
40
+ type: 'string',
41
+ description: 'Integer output type.',
42
+ enum: ['int'],
43
+ },
44
+ promptTemplate: {
45
+ type: 'string',
46
+ description:
47
+ 'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
48
+ },
49
+ description: {
50
+ type: 'string',
51
+ description: 'The description of the Metric.',
52
+ },
53
+ evalModelName: {
54
+ type: 'string',
55
+ description: 'The AI model to use for evaluation.',
56
+ },
57
+ guidelines: {
58
+ type: 'string',
59
+ description: 'Guidelines for AI evaluation on how to score the metric.',
60
+ },
61
+ passingThreshold: {
62
+ type: 'integer',
63
+ description: 'The threshold for determining pass/fail from integer scores (1-5).',
64
+ },
65
+ temperature: {
66
+ type: 'number',
67
+ description: 'The temperature for AI evaluation (0-2).',
68
+ },
69
+ },
70
+ },
71
+ {
72
+ type: 'object',
73
+ properties: {
74
+ projectId: {
75
+ type: 'string',
76
+ },
77
+ evalType: {
78
+ type: 'string',
79
+ description: 'Human-based evaluation type.',
80
+ enum: ['human'],
81
+ },
82
+ name: {
83
+ type: 'string',
84
+ description: 'The name of the Metric.',
85
+ },
86
+ outputType: {
87
+ type: 'string',
88
+ description: 'Integer output type.',
89
+ enum: ['int'],
90
+ },
91
+ description: {
92
+ type: 'string',
93
+ description: 'The description of the Metric.',
94
+ },
95
+ guidelines: {
96
+ type: 'string',
97
+ description: 'Guidelines for human evaluators.',
98
+ },
99
+ passingThreshold: {
100
+ type: 'integer',
101
+ description: 'The threshold for determining pass/fail from integer scores (1-5).',
102
+ },
103
+ },
104
+ },
105
+ {
106
+ type: 'object',
107
+ properties: {
108
+ projectId: {
109
+ type: 'string',
110
+ },
111
+ evalType: {
112
+ type: 'string',
113
+ description: 'Heuristic-based evaluation type.',
114
+ enum: ['heuristic'],
115
+ },
116
+ name: {
117
+ type: 'string',
118
+ description: 'The name of the Metric.',
119
+ },
120
+ outputType: {
121
+ type: 'string',
122
+ description: 'Integer output type.',
123
+ enum: ['int'],
124
+ },
125
+ description: {
126
+ type: 'string',
127
+ description: 'The description of the Metric.',
128
+ },
129
+ guidelines: {
130
+ type: 'string',
131
+ description: 'Optional guidelines for heuristic evaluation logic.',
132
+ },
133
+ passingThreshold: {
134
+ type: 'integer',
135
+ description: 'The threshold for determining pass/fail from integer scores (1-5).',
136
+ },
137
+ },
138
+ },
139
+ {
140
+ type: 'object',
141
+ properties: {
142
+ projectId: {
143
+ type: 'string',
144
+ },
145
+ evalType: {
146
+ type: 'string',
147
+ description: 'AI-based evaluation type.',
148
+ enum: ['ai'],
149
+ },
150
+ name: {
151
+ type: 'string',
152
+ description: 'The name of the Metric.',
153
+ },
154
+ outputType: {
155
+ type: 'string',
156
+ description: 'Boolean output type.',
157
+ enum: ['boolean'],
158
+ },
159
+ promptTemplate: {
160
+ type: 'string',
161
+ description:
162
+ 'The complete prompt template for AI evaluation. Should include placeholders for dynamic content.',
163
+ },
164
+ description: {
165
+ type: 'string',
166
+ description: 'The description of the Metric.',
167
+ },
168
+ evalModelName: {
169
+ type: 'string',
170
+ description: 'The AI model to use for evaluation.',
171
+ },
172
+ guidelines: {
173
+ type: 'string',
174
+ description: 'Guidelines for AI evaluation on how to score the metric.',
175
+ },
176
+ temperature: {
177
+ type: 'number',
178
+ description: 'The temperature for AI evaluation (0-2).',
179
+ },
180
+ },
181
+ },
182
+ {
183
+ type: 'object',
184
+ properties: {
185
+ projectId: {
186
+ type: 'string',
187
+ },
188
+ evalType: {
189
+ type: 'string',
190
+ description: 'Human-based evaluation type.',
191
+ enum: ['human'],
192
+ },
193
+ name: {
194
+ type: 'string',
195
+ description: 'The name of the Metric.',
196
+ },
197
+ outputType: {
198
+ type: 'string',
199
+ description: 'Boolean output type.',
200
+ enum: ['boolean'],
201
+ },
202
+ description: {
203
+ type: 'string',
204
+ description: 'The description of the Metric.',
205
+ },
206
+ guidelines: {
207
+ type: 'string',
208
+ description: 'Guidelines for human evaluators.',
209
+ },
210
+ },
211
+ },
212
+ {
213
+ type: 'object',
214
+ properties: {
215
+ projectId: {
216
+ type: 'string',
217
+ },
218
+ evalType: {
219
+ type: 'string',
220
+ description: 'Heuristic-based evaluation type.',
221
+ enum: ['heuristic'],
222
+ },
223
+ name: {
224
+ type: 'string',
225
+ description: 'The name of the Metric.',
226
+ },
227
+ outputType: {
228
+ type: 'string',
229
+ description: 'Boolean output type.',
230
+ enum: ['boolean'],
231
+ },
232
+ description: {
233
+ type: 'string',
234
+ description: 'The description of the Metric.',
235
+ },
236
+ guidelines: {
237
+ type: 'string',
238
+ description: 'Optional guidelines for heuristic evaluation logic.',
239
+ },
240
+ },
241
+ },
242
+ ],
243
+ },
244
+ };
245
+
246
+ export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
247
+ const { projectId, ...body } = args as any;
248
+ return asTextContentResult(await client.metrics.create(projectId, body));
249
+ };
250
+
251
+ export default { metadata, tool, handler };
package/src/tools/runs/create-runs.ts CHANGED
@@ -31,9 +31,9 @@ export const tool: Tool = {
31
31
  type: 'string',
32
32
  },
33
33
  },
34
- systemConfigId: {
34
+ systemVersionId: {
35
35
  type: 'string',
36
- description: 'The ID of the system configuration this Run is using.',
36
+ description: 'The ID of the system version this Run is using.',
37
37
  },
38
38
  testsetId: {
39
39
  type: 'string',
package/src/tools/systems/delete-systems.ts CHANGED
@@ -17,7 +17,7 @@ export const metadata: Metadata = {
17
17
 
18
18
  export const tool: Tool = {
19
19
  name: 'delete_systems',
20
- description: 'Delete a system definition by ID. This will not delete associated system configurations.',
20
+ description: 'Delete a system definition by ID. This will not delete associated system versions.',
21
21
  inputSchema: {
22
22
  type: 'object',
23
23
  properties: {
package/src/tools/systems/versions/create-systems-versions.ts ADDED
@@ -0,0 +1,45 @@
1
+ // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
2
+
3
+ import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
4
+
5
+ import { Tool } from '@modelcontextprotocol/sdk/types.js';
6
+ import type { Metadata } from '../../';
7
+ import Scorecard from 'scorecard-ai';
8
+
9
+ export const metadata: Metadata = {
10
+ resource: 'systems.versions',
11
+ operation: 'write',
12
+ tags: [],
13
+ httpMethod: 'post',
14
+ httpPath: '/systems/{systemId}/configs',
15
+ operationId: 'createSystemVersion',
16
+ };
17
+
18
+ export const tool: Tool = {
19
+ name: 'create_systems_versions',
20
+ description:
21
+ "Create a new version for a system.\n\nEach version contains specific parameter values that match the system's `configSchema` - things like model parameters, thresholds, or processing options.\nOnce created, versions cannot be modified, ensuring stable reference points for evaluations.\n\nWhen creating a system version:\n- The `config` object is validated against the parent system's `configSchema`.\n- System versions with validation errors are still stored, with errors included in the response.\n- Validation errors indicate fields that don't match the schema but don't prevent creation.\n- Having validation errors may affect how some evaluation metrics are calculated.",
22
+ inputSchema: {
23
+ type: 'object',
24
+ properties: {
25
+ systemId: {
26
+ type: 'string',
27
+ },
28
+ config: {
29
+ type: 'object',
30
+ description: 'The configuration of the system version.',
31
+ },
32
+ name: {
33
+ type: 'string',
34
+ description: 'The name of the system version.',
35
+ },
36
+ },
37
+ },
38
+ };
39
+
40
+ export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
41
+ const { systemId, ...body } = args as any;
42
+ return asTextContentResult(await client.systems.versions.create(systemId, body));
43
+ };
44
+
45
+ export default { metadata, tool, handler };
package/src/tools/{system-configs/get-system-configs.ts → systems/versions/get-systems-versions.ts} RENAMED
@@ -3,25 +3,25 @@
3
3
  import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
4
4
 
5
5
  import { Tool } from '@modelcontextprotocol/sdk/types.js';
6
- import type { Metadata } from '../';
6
+ import type { Metadata } from '../../';
7
7
  import Scorecard from 'scorecard-ai';
8
8
 
9
9
  export const metadata: Metadata = {
10
- resource: 'system_configs',
10
+ resource: 'systems.versions',
11
11
  operation: 'read',
12
12
  tags: [],
13
13
  httpMethod: 'get',
14
- httpPath: '/systems/configs/{systemConfigId}',
15
- operationId: 'getSystemConfig',
14
+ httpPath: '/systems/configs/{systemVersionId}',
15
+ operationId: 'getSystemVersion',
16
16
  };
17
17
 
18
18
  export const tool: Tool = {
19
- name: 'get_system_configs',
20
- description: 'Retrieve a specific system configuration by ID.',
19
+ name: 'get_systems_versions',
20
+ description: 'Retrieve a specific system version by ID.',
21
21
  inputSchema: {
22
22
  type: 'object',
23
23
  properties: {
24
- systemConfigId: {
24
+ systemVersionId: {
25
25
  type: 'string',
26
26
  },
27
27
  },
@@ -29,8 +29,8 @@ export const tool: Tool = {
29
29
  };
30
30
 
31
31
  export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
32
- const { systemConfigId, ...body } = args as any;
33
- return asTextContentResult(await client.systemConfigs.get(systemConfigId));
32
+ const { systemVersionId, ...body } = args as any;
33
+ return asTextContentResult(await client.systems.versions.get(systemVersionId));
34
34
  };
35
35
 
36
36
  export default { metadata, tool, handler };
package/src/tools/{system-configs/list-system-configs.ts → systems/versions/list-systems-versions.ts} RENAMED
@@ -3,22 +3,22 @@
3
3
  import { asTextContentResult } from 'scorecard-ai-mcp/tools/types';
4
4
 
5
5
  import { Tool } from '@modelcontextprotocol/sdk/types.js';
6
- import type { Metadata } from '../';
6
+ import type { Metadata } from '../../';
7
7
  import Scorecard from 'scorecard-ai';
8
8
 
9
9
  export const metadata: Metadata = {
10
- resource: 'system_configs',
10
+ resource: 'systems.versions',
11
11
  operation: 'read',
12
12
  tags: [],
13
13
  httpMethod: 'get',
14
14
  httpPath: '/systems/{systemId}/configs',
15
- operationId: 'listSystemConfigs',
15
+ operationId: 'listSystemVersions',
16
16
  };
17
17
 
18
18
  export const tool: Tool = {
19
- name: 'list_system_configs',
19
+ name: 'list_systems_versions',
20
20
  description:
21
- 'Retrieve a paginated list of configurations for a specific system.\n\nSystem configurations provide concrete parameter values for a System Under Test, defining exactly how the system should be configured during an evaluation run.',
21
+ 'Retrieve a paginated list of system versions for a specific system.\n\nSystem versions provide concrete parameter values for a System Under Test, defining exactly how the system should be configured during an evaluation run.',
22
22
  inputSchema: {
23
23
  type: 'object',
24
24
  properties: {
@@ -41,7 +41,7 @@ export const tool: Tool = {
41
41
 
42
42
  export const handler = async (client: Scorecard, args: Record<string, unknown> | undefined) => {
43
43
  const { systemId, ...body } = args as any;
44
- return asTextContentResult(await client.systemConfigs.list(systemId, body));
44
+ return asTextContentResult(await client.systems.versions.list(systemId, body));
45
45
  };
46
46
 
47
47
  export default { metadata, tool, handler };
package/src/tools/testsets/get-testsets.ts CHANGED
@@ -17,7 +17,7 @@ export const metadata: Metadata = {
17
17
 
18
18
  export const tool: Tool = {
19
19
  name: 'get_testsets',
20
- description: 'Get Testset by ID',
20
+ description: 'Get Testset',
21
21
  inputSchema: {
22
22
  type: 'object',
23
23
  properties: {
package/tools/index.d.mts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.mts","sourceRoot":"","sources":["../src/tools/index.ts"],"names":[],"mappings":"OAEO,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE;AAE9C,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC;AA0B/C,eAAO,MAAM,SAAS,EAAE,QAAQ,EAAO,CAAC;AA8BxC,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,UAAU,GAAG,WAAW,GAAG,KAAK,GAAG,MAAM,CAAC;IAChD,EAAE,EAAE,SAAS,GAAG,SAAS,CAAC;IAC1B,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,CA2B1E"}
1
+ {"version":3,"file":"index.d.mts","sourceRoot":"","sources":["../src/tools/index.ts"],"names":[],"mappings":"OAEO,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE;AAE9C,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC;AA2B/C,eAAO,MAAM,SAAS,EAAE,QAAQ,EAAO,CAAC;AA+BxC,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,UAAU,GAAG,WAAW,GAAG,KAAK,GAAG,MAAM,CAAC;IAChD,EAAE,EAAE,SAAS,GAAG,SAAS,CAAC;IAC1B,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,CA2B1E"}
package/tools/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/tools/index.ts"],"names":[],"mappings":"OAEO,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE;AAE9C,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC;AA0B/C,eAAO,MAAM,SAAS,EAAE,QAAQ,EAAO,CAAC;AA8BxC,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,UAAU,GAAG,WAAW,GAAG,KAAK,GAAG,MAAM,CAAC;IAChD,EAAE,EAAE,SAAS,GAAG,SAAS,CAAC;IAC1B,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,CA2B1E"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/tools/index.ts"],"names":[],"mappings":"OAEO,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE;AAE9C,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,CAAC;AA2B/C,eAAO,MAAM,SAAS,EAAE,QAAQ,EAAO,CAAC;AA+BxC,MAAM,MAAM,MAAM,GAAG;IACnB,IAAI,EAAE,UAAU,GAAG,WAAW,GAAG,KAAK,GAAG,MAAM,CAAC;IAChD,EAAE,EAAE,SAAS,GAAG,SAAS,CAAC;IAC1B,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,CA2B1E"}
package/tools/index.js CHANGED
@@ -19,6 +19,7 @@ const list_testcases_1 = __importDefault(require("./testcases/list-testcases.js"
19
19
  const delete_testcases_1 = __importDefault(require("./testcases/delete-testcases.js"));
20
20
  const get_testcases_1 = __importDefault(require("./testcases/get-testcases.js"));
21
21
  const create_runs_1 = __importDefault(require("./runs/create-runs.js"));
22
+ const create_metrics_1 = __importDefault(require("./metrics/create-metrics.js"));
22
23
  const create_records_1 = __importDefault(require("./records/create-records.js"));
23
24
  const upsert_scores_1 = __importDefault(require("./scores/upsert-scores.js"));
24
25
  const create_systems_1 = __importDefault(require("./systems/create-systems.js"));
@@ -26,9 +27,9 @@ const update_systems_1 = __importDefault(require("./systems/update-systems.js"))
26
27
  const list_systems_1 = __importDefault(require("./systems/list-systems.js"));
27
28
  const delete_systems_1 = __importDefault(require("./systems/delete-systems.js"));
28
29
  const get_systems_1 = __importDefault(require("./systems/get-systems.js"));
29
- const create_system_configs_1 = __importDefault(require("./system-configs/create-system-configs.js"));
30
- const list_system_configs_1 = __importDefault(require("./system-configs/list-system-configs.js"));
31
- const get_system_configs_1 = __importDefault(require("./system-configs/get-system-configs.js"));
30
+ const create_systems_versions_1 = __importDefault(require("./systems/versions/create-systems-versions.js"));
31
+ const list_systems_versions_1 = __importDefault(require("./systems/versions/list-systems-versions.js"));
32
+ const get_systems_versions_1 = __importDefault(require("./systems/versions/get-systems-versions.js"));
32
33
  exports.endpoints = [];
33
34
  function addEndpoint(endpoint) {
34
35
  exports.endpoints.push(endpoint);
@@ -46,6 +47,7 @@ addEndpoint(list_testcases_1.default);
46
47
  addEndpoint(delete_testcases_1.default);
47
48
  addEndpoint(get_testcases_1.default);
48
49
  addEndpoint(create_runs_1.default);
50
+ addEndpoint(create_metrics_1.default);
49
51
  addEndpoint(create_records_1.default);
50
52
  addEndpoint(upsert_scores_1.default);
51
53
  addEndpoint(create_systems_1.default);
@@ -53,9 +55,9 @@ addEndpoint(update_systems_1.default);
53
55
  addEndpoint(list_systems_1.default);
54
56
  addEndpoint(delete_systems_1.default);
55
57
  addEndpoint(get_systems_1.default);
56
- addEndpoint(create_system_configs_1.default);
57
- addEndpoint(list_system_configs_1.default);
58
- addEndpoint(get_system_configs_1.default);
58
+ addEndpoint(create_systems_versions_1.default);
59
+ addEndpoint(list_systems_versions_1.default);
60
+ addEndpoint(get_systems_versions_1.default);
59
61
  function query(filters, endpoints) {
60
62
  const allExcludes = filters.length > 0 && filters.every((filter) => filter.op === 'exclude');
61
63
  const unmatchedFilters = new Set(filters);