scorecard-ai 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +48 -19
- package/client.d.mts +2 -2
- package/client.d.mts.map +1 -1
- package/client.d.ts +2 -2
- package/client.d.ts.map +1 -1
- package/client.js +3 -1
- package/client.js.map +1 -1
- package/client.mjs +3 -1
- package/client.mjs.map +1 -1
- package/package.json +1 -1
- package/resources/index.d.mts +1 -1
- package/resources/index.d.mts.map +1 -1
- package/resources/index.d.ts +1 -1
- package/resources/index.d.ts.map +1 -1
- package/resources/index.js.map +1 -1
- package/resources/index.mjs.map +1 -1
- package/resources/metrics.d.mts +192 -2
- package/resources/metrics.d.mts.map +1 -1
- package/resources/metrics.d.ts +192 -2
- package/resources/metrics.d.ts.map +1 -1
- package/resources/metrics.js +20 -1
- package/resources/metrics.js.map +1 -1
- package/resources/metrics.mjs +20 -1
- package/resources/metrics.mjs.map +1 -1
- package/src/client.ts +10 -3
- package/src/resources/index.ts +1 -1
- package/src/resources/metrics.ts +245 -2
- package/src/version.ts +1 -1
- package/version.d.mts +1 -1
- package/version.d.ts +1 -1
- package/version.js +1 -1
- package/version.mjs +1 -1
package/resources/metrics.d.mts
CHANGED
|
@@ -3,7 +3,8 @@ import { APIPromise } from "../core/api-promise.mjs";
|
|
|
3
3
|
import { RequestOptions } from "../internal/request-options.mjs";
|
|
4
4
|
export declare class Metrics extends APIResource {
|
|
5
5
|
/**
|
|
6
|
-
* Create a new Metric for evaluating system outputs.
|
|
6
|
+
* Create a new Metric for evaluating system outputs. The structure of a metric
|
|
7
|
+
* depends on the evalType and outputType of the metric.
|
|
7
8
|
*
|
|
8
9
|
* @example
|
|
9
10
|
* ```ts
|
|
@@ -23,6 +24,22 @@ export declare class Metrics extends APIResource {
|
|
|
23
24
|
* ```
|
|
24
25
|
*/
|
|
25
26
|
create(projectID: string, body: MetricCreateParams, options?: RequestOptions): APIPromise<Metric>;
|
|
27
|
+
/**
|
|
28
|
+
* Update an existing Metric. You must specify the evalType and outputType of the
|
|
29
|
+
* metric. The structure of a metric depends on the evalType and outputType of the
|
|
30
|
+
* metric.
|
|
31
|
+
*
|
|
32
|
+
* @example
|
|
33
|
+
* ```ts
|
|
34
|
+
* const metric = await client.metrics.update('321', {
|
|
35
|
+
* evalType: 'ai',
|
|
36
|
+
* outputType: 'boolean',
|
|
37
|
+
* promptTemplate:
|
|
38
|
+
* 'Using the following guidelines, evaluate the response: {{ guidelines }}\n\nResponse: {{ outputs.response }}\n\nIdeal answer: {{ expected.idealResponse }}',
|
|
39
|
+
* });
|
|
40
|
+
* ```
|
|
41
|
+
*/
|
|
42
|
+
update(metricID: string, body: MetricUpdateParams, options?: RequestOptions): APIPromise<Metric>;
|
|
26
43
|
}
|
|
27
44
|
/**
|
|
28
45
|
* A Metric defines how to evaluate system outputs against expected results.
|
|
@@ -415,7 +432,180 @@ export declare namespace MetricCreateParams {
|
|
|
415
432
|
guidelines?: string;
|
|
416
433
|
}
|
|
417
434
|
}
|
|
435
|
+
export type MetricUpdateParams = MetricUpdateParams.AIIntMetric | MetricUpdateParams.HumanIntMetric | MetricUpdateParams.HeuristicIntMetric | MetricUpdateParams.AIBooleanMetric | MetricUpdateParams.HumanBooleanMetric | MetricUpdateParams.HeuristicBooleanMetric;
|
|
436
|
+
export declare namespace MetricUpdateParams {
|
|
437
|
+
interface AIIntMetric {
|
|
438
|
+
/**
|
|
439
|
+
* AI-based evaluation type.
|
|
440
|
+
*/
|
|
441
|
+
evalType: 'ai';
|
|
442
|
+
/**
|
|
443
|
+
* Integer output type.
|
|
444
|
+
*/
|
|
445
|
+
outputType: 'int';
|
|
446
|
+
/**
|
|
447
|
+
* The description of the Metric.
|
|
448
|
+
*/
|
|
449
|
+
description?: string | null;
|
|
450
|
+
/**
|
|
451
|
+
* The AI model to use for evaluation.
|
|
452
|
+
*/
|
|
453
|
+
evalModelName?: string;
|
|
454
|
+
/**
|
|
455
|
+
* Guidelines for AI evaluation on how to score the metric.
|
|
456
|
+
*/
|
|
457
|
+
guidelines?: string | null;
|
|
458
|
+
/**
|
|
459
|
+
* The name of the Metric.
|
|
460
|
+
*/
|
|
461
|
+
name?: string;
|
|
462
|
+
/**
|
|
463
|
+
* The threshold for determining pass/fail from integer scores (1-5).
|
|
464
|
+
*/
|
|
465
|
+
passingThreshold?: number;
|
|
466
|
+
/**
|
|
467
|
+
* The complete prompt template for AI evaluation. Should include placeholders for
|
|
468
|
+
* dynamic content.
|
|
469
|
+
*/
|
|
470
|
+
promptTemplate?: string;
|
|
471
|
+
/**
|
|
472
|
+
* The temperature for AI evaluation (0-2).
|
|
473
|
+
*/
|
|
474
|
+
temperature?: number;
|
|
475
|
+
}
|
|
476
|
+
interface HumanIntMetric {
|
|
477
|
+
/**
|
|
478
|
+
* Human-based evaluation type.
|
|
479
|
+
*/
|
|
480
|
+
evalType: 'human';
|
|
481
|
+
/**
|
|
482
|
+
* Integer output type.
|
|
483
|
+
*/
|
|
484
|
+
outputType: 'int';
|
|
485
|
+
/**
|
|
486
|
+
* The description of the Metric.
|
|
487
|
+
*/
|
|
488
|
+
description?: string | null;
|
|
489
|
+
/**
|
|
490
|
+
* Guidelines for human evaluators.
|
|
491
|
+
*/
|
|
492
|
+
guidelines?: string;
|
|
493
|
+
/**
|
|
494
|
+
* The name of the Metric.
|
|
495
|
+
*/
|
|
496
|
+
name?: string;
|
|
497
|
+
/**
|
|
498
|
+
* The threshold for determining pass/fail from integer scores (1-5).
|
|
499
|
+
*/
|
|
500
|
+
passingThreshold?: number;
|
|
501
|
+
}
|
|
502
|
+
interface HeuristicIntMetric {
|
|
503
|
+
/**
|
|
504
|
+
* Heuristic-based evaluation type.
|
|
505
|
+
*/
|
|
506
|
+
evalType: 'heuristic';
|
|
507
|
+
/**
|
|
508
|
+
* Integer output type.
|
|
509
|
+
*/
|
|
510
|
+
outputType: 'int';
|
|
511
|
+
/**
|
|
512
|
+
* The description of the Metric.
|
|
513
|
+
*/
|
|
514
|
+
description?: string | null;
|
|
515
|
+
/**
|
|
516
|
+
* Optional guidelines for heuristic evaluation logic.
|
|
517
|
+
*/
|
|
518
|
+
guidelines?: string;
|
|
519
|
+
/**
|
|
520
|
+
* The name of the Metric.
|
|
521
|
+
*/
|
|
522
|
+
name?: string;
|
|
523
|
+
/**
|
|
524
|
+
* The threshold for determining pass/fail from integer scores (1-5).
|
|
525
|
+
*/
|
|
526
|
+
passingThreshold?: number;
|
|
527
|
+
}
|
|
528
|
+
interface AIBooleanMetric {
|
|
529
|
+
/**
|
|
530
|
+
* AI-based evaluation type.
|
|
531
|
+
*/
|
|
532
|
+
evalType: 'ai';
|
|
533
|
+
/**
|
|
534
|
+
* Boolean output type.
|
|
535
|
+
*/
|
|
536
|
+
outputType: 'boolean';
|
|
537
|
+
/**
|
|
538
|
+
* The description of the Metric.
|
|
539
|
+
*/
|
|
540
|
+
description?: string | null;
|
|
541
|
+
/**
|
|
542
|
+
* The AI model to use for evaluation.
|
|
543
|
+
*/
|
|
544
|
+
evalModelName?: string;
|
|
545
|
+
/**
|
|
546
|
+
* Guidelines for AI evaluation on how to score the metric.
|
|
547
|
+
*/
|
|
548
|
+
guidelines?: string | null;
|
|
549
|
+
/**
|
|
550
|
+
* The name of the Metric.
|
|
551
|
+
*/
|
|
552
|
+
name?: string;
|
|
553
|
+
/**
|
|
554
|
+
* The complete prompt template for AI evaluation. Should include placeholders for
|
|
555
|
+
* dynamic content.
|
|
556
|
+
*/
|
|
557
|
+
promptTemplate?: string;
|
|
558
|
+
/**
|
|
559
|
+
* The temperature for AI evaluation (0-2).
|
|
560
|
+
*/
|
|
561
|
+
temperature?: number;
|
|
562
|
+
}
|
|
563
|
+
interface HumanBooleanMetric {
|
|
564
|
+
/**
|
|
565
|
+
* Human-based evaluation type.
|
|
566
|
+
*/
|
|
567
|
+
evalType: 'human';
|
|
568
|
+
/**
|
|
569
|
+
* Boolean output type.
|
|
570
|
+
*/
|
|
571
|
+
outputType: 'boolean';
|
|
572
|
+
/**
|
|
573
|
+
* The description of the Metric.
|
|
574
|
+
*/
|
|
575
|
+
description?: string | null;
|
|
576
|
+
/**
|
|
577
|
+
* Guidelines for human evaluators.
|
|
578
|
+
*/
|
|
579
|
+
guidelines?: string;
|
|
580
|
+
/**
|
|
581
|
+
* The name of the Metric.
|
|
582
|
+
*/
|
|
583
|
+
name?: string;
|
|
584
|
+
}
|
|
585
|
+
interface HeuristicBooleanMetric {
|
|
586
|
+
/**
|
|
587
|
+
* Heuristic-based evaluation type.
|
|
588
|
+
*/
|
|
589
|
+
evalType: 'heuristic';
|
|
590
|
+
/**
|
|
591
|
+
* Boolean output type.
|
|
592
|
+
*/
|
|
593
|
+
outputType: 'boolean';
|
|
594
|
+
/**
|
|
595
|
+
* The description of the Metric.
|
|
596
|
+
*/
|
|
597
|
+
description?: string | null;
|
|
598
|
+
/**
|
|
599
|
+
* Optional guidelines for heuristic evaluation logic.
|
|
600
|
+
*/
|
|
601
|
+
guidelines?: string;
|
|
602
|
+
/**
|
|
603
|
+
* The name of the Metric.
|
|
604
|
+
*/
|
|
605
|
+
name?: string;
|
|
606
|
+
}
|
|
607
|
+
}
|
|
418
608
|
export declare namespace Metrics {
|
|
419
|
-
export { type Metric as Metric, type MetricCreateParams as MetricCreateParams };
|
|
609
|
+
export { type Metric as Metric, type MetricCreateParams as MetricCreateParams, type MetricUpdateParams as MetricUpdateParams, };
|
|
420
610
|
}
|
|
421
611
|
//# sourceMappingURL=metrics.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.d.mts","sourceRoot":"","sources":["../src/resources/metrics.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,EAAE,UAAU,EAAE;OACd,EAAE,cAAc,EAAE;AAGzB,qBAAa,OAAQ,SAAQ,WAAW;IACtC
|
|
1
|
+
{"version":3,"file":"metrics.d.mts","sourceRoot":"","sources":["../src/resources/metrics.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,EAAE,UAAU,EAAE;OACd,EAAE,cAAc,EAAE;AAGzB,qBAAa,OAAQ,SAAQ,WAAW;IACtC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;IAIjG;;;;;;;;;;;;;;OAcG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;CAGjG;AAED;;GAEG;AACH,MAAM,MAAM,MAAM,GACd,MAAM,CAAC,WAAW,GAClB,MAAM,CAAC,cAAc,GACrB,MAAM,CAAC,kBAAkB,GACzB,MAAM,CAAC,eAAe,GACtB,MAAM,CAAC,kBAAkB,GACzB,MAAM,CAAC,sBAAsB,CAAC;AAElC,yBAAiB,MAAM,CAAC;IACtB;;OAEG;IACH,UAAiB,WAAW;QAC1B;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,aAAa,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,QAAQ,EAAE,IAAI,CAAC;QAEf;;WAEG;QACH,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAE1B;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,gBAAgB,EAAE,MAAM,CAAC;QAEzB;;;WAGG;QACH,cAAc,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,WAAW,EAAE,MAAM,CAAC;KACrB;IAED;;OAEG;IACH,UAAiB,cAAc;QAC7B;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,QAAQ,EAAE,OAAO,CAAC;QAElB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,gBAAgB,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;IAED;;OAEG;IACH,UAAiB,kBAAkB;QACjC;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,QAAQ,EAAE,WAAW,CAAC;QAEtB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,gBAAgB,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;IAED;;OAEG;IACH,UAAiB,eAAe;QAC9B;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,aAAa,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,QAAQ,EAAE,IAAI,CAAC;QAEf;;WAEG;QACH,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAE1B;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;;WAGG;QACH,cAAc,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,WAAW,EAAE,MAAM,CAAC;KACrB;IAED;;OAEG;IACH,UAAiB,kBAAkB;QACjC;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,QAAQ,EAAE,OAAO,CAAC;QAElB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;IAED;;OAEG;IACH,UAAiB,sBAAsB;QACrC;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,QAAQ,EAAE,WAAW,CAAC;QAEtB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;CACF;AAED,MAAM,MAAM,kBAAkB,GAC1B,kBAAkB,CAAC,WAAW,GAC9B,kBAAkB,CAAC,cAAc,GACjC,kBAAkB,CAAC,kBAAkB,GACrC,kBAAkB,CAAC,eAAe,GAClC,kBAAkB,CAAC,kBAAkB,GACrC,kBAAkB,CAAC,sBAAsB,CAAC;AAE9C,MAAM,CAAC,OAAO,WAAW,kBAAkB,CAAC;IAC1C,UAAiB,WAAW;QAC1B;;WAEG;QACH,QAAQ,EAAE,IAAI,CAAC;QAEf;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;;WAGG;QACH,cAAc,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,aAAa,CAAC,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB;IAED,UAAiB,cAAc;QAC7B;;WAEG;QACH,QAAQ,EAAE,OAAO,CAAC;QAElB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B;IAED,UAAiB,kBAAkB;QACjC;;WAEG;QACH,QAAQ,EAAE,WAAW,CAAC;QAEtB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B;IAED,UAAiB,eAAe;QAC9B;;WAEG;QACH,QAAQ,EAAE,IAAI,CAAC;QAEf;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;;WAGG;QACH,cAAc,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,aAAa,CAAC,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB;IAED,UAAiB,kBAAkB;QACjC;;WAEG;QACH,QAAQ,EAAE,OAAO,CAAC;QAElB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;IAED,UAAiB,sBAAsB;QACrC;;WAEG;QACH,QAAQ,EAAE,WAAW,CAAC;QAEtB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;CACF;AAED,MAAM,MAAM,kBAAkB,GAC1B,kBAAkB,CAAC,WAAW,GAC9B,kBAAkB,CAAC,cAAc,GACjC,kBAAkB,CAAC,kBAAkB,GACrC,kBAAkB,CAAC,eAAe,GAClC,kBAAkB,CAAC,kBAAkB,GACrC,kBAAkB,CAAC,sBAAsB,CAAC;AAE9C,MAAM,CAAC,OAAO,WAAW,kBAAkB,CAAC;IAC1C,UAAiB,WAAW;QAC1B;;WAEG;QACH,QAAQ,EAAE,IAAI,CAAC;QAEf;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,aAAa,CAAC,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B;;;WAGG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB;IAED,UAAiB,cAAc;QAC7B;;WAEG;QACH,QAAQ,EAAE,OAAO,CAAC;QAElB;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B;IAED,UAAiB,kBAAkB;QACjC;;WAEG;QACH,QAAQ,EAAE,WAAW,CAAC;QAEtB;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B;IAED,UAAiB,eAAe;QAC9B;;WAEG;QACH,QAAQ,EAAE,IAAI,CAAC;QAEf;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,aAAa,CAAC,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;;WAGG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB;IAED,UAAiB,kBAAkB;QACjC;;WAEG;QACH,QAAQ,EAAE,OAAO,CAAC;QAElB;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;KACf;IAED,UAAiB,sBAAsB;QACrC;;WAEG;QACH,QAAQ,EAAE,WAAW,CAAC;QAEtB;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;KACf;CACF;AAED,MAAM,CAAC,OAAO,WAAW,OAAO,CAAC;IAC/B,OAAO,EACL,KAAK,MAAM,IAAI,MAAM,EACrB,KAAK,kBAAkB,IAAI,kBAAkB,EAC7C,KAAK,kBAAkB,IAAI,kBAAkB,GAC9C,CAAC;CACH"}
|
package/resources/metrics.d.ts
CHANGED
|
@@ -3,7 +3,8 @@ import { APIPromise } from "../core/api-promise.js";
|
|
|
3
3
|
import { RequestOptions } from "../internal/request-options.js";
|
|
4
4
|
export declare class Metrics extends APIResource {
|
|
5
5
|
/**
|
|
6
|
-
* Create a new Metric for evaluating system outputs.
|
|
6
|
+
* Create a new Metric for evaluating system outputs. The structure of a metric
|
|
7
|
+
* depends on the evalType and outputType of the metric.
|
|
7
8
|
*
|
|
8
9
|
* @example
|
|
9
10
|
* ```ts
|
|
@@ -23,6 +24,22 @@ export declare class Metrics extends APIResource {
|
|
|
23
24
|
* ```
|
|
24
25
|
*/
|
|
25
26
|
create(projectID: string, body: MetricCreateParams, options?: RequestOptions): APIPromise<Metric>;
|
|
27
|
+
/**
|
|
28
|
+
* Update an existing Metric. You must specify the evalType and outputType of the
|
|
29
|
+
* metric. The structure of a metric depends on the evalType and outputType of the
|
|
30
|
+
* metric.
|
|
31
|
+
*
|
|
32
|
+
* @example
|
|
33
|
+
* ```ts
|
|
34
|
+
* const metric = await client.metrics.update('321', {
|
|
35
|
+
* evalType: 'ai',
|
|
36
|
+
* outputType: 'boolean',
|
|
37
|
+
* promptTemplate:
|
|
38
|
+
* 'Using the following guidelines, evaluate the response: {{ guidelines }}\n\nResponse: {{ outputs.response }}\n\nIdeal answer: {{ expected.idealResponse }}',
|
|
39
|
+
* });
|
|
40
|
+
* ```
|
|
41
|
+
*/
|
|
42
|
+
update(metricID: string, body: MetricUpdateParams, options?: RequestOptions): APIPromise<Metric>;
|
|
26
43
|
}
|
|
27
44
|
/**
|
|
28
45
|
* A Metric defines how to evaluate system outputs against expected results.
|
|
@@ -415,7 +432,180 @@ export declare namespace MetricCreateParams {
|
|
|
415
432
|
guidelines?: string;
|
|
416
433
|
}
|
|
417
434
|
}
|
|
435
|
+
export type MetricUpdateParams = MetricUpdateParams.AIIntMetric | MetricUpdateParams.HumanIntMetric | MetricUpdateParams.HeuristicIntMetric | MetricUpdateParams.AIBooleanMetric | MetricUpdateParams.HumanBooleanMetric | MetricUpdateParams.HeuristicBooleanMetric;
|
|
436
|
+
export declare namespace MetricUpdateParams {
|
|
437
|
+
interface AIIntMetric {
|
|
438
|
+
/**
|
|
439
|
+
* AI-based evaluation type.
|
|
440
|
+
*/
|
|
441
|
+
evalType: 'ai';
|
|
442
|
+
/**
|
|
443
|
+
* Integer output type.
|
|
444
|
+
*/
|
|
445
|
+
outputType: 'int';
|
|
446
|
+
/**
|
|
447
|
+
* The description of the Metric.
|
|
448
|
+
*/
|
|
449
|
+
description?: string | null;
|
|
450
|
+
/**
|
|
451
|
+
* The AI model to use for evaluation.
|
|
452
|
+
*/
|
|
453
|
+
evalModelName?: string;
|
|
454
|
+
/**
|
|
455
|
+
* Guidelines for AI evaluation on how to score the metric.
|
|
456
|
+
*/
|
|
457
|
+
guidelines?: string | null;
|
|
458
|
+
/**
|
|
459
|
+
* The name of the Metric.
|
|
460
|
+
*/
|
|
461
|
+
name?: string;
|
|
462
|
+
/**
|
|
463
|
+
* The threshold for determining pass/fail from integer scores (1-5).
|
|
464
|
+
*/
|
|
465
|
+
passingThreshold?: number;
|
|
466
|
+
/**
|
|
467
|
+
* The complete prompt template for AI evaluation. Should include placeholders for
|
|
468
|
+
* dynamic content.
|
|
469
|
+
*/
|
|
470
|
+
promptTemplate?: string;
|
|
471
|
+
/**
|
|
472
|
+
* The temperature for AI evaluation (0-2).
|
|
473
|
+
*/
|
|
474
|
+
temperature?: number;
|
|
475
|
+
}
|
|
476
|
+
interface HumanIntMetric {
|
|
477
|
+
/**
|
|
478
|
+
* Human-based evaluation type.
|
|
479
|
+
*/
|
|
480
|
+
evalType: 'human';
|
|
481
|
+
/**
|
|
482
|
+
* Integer output type.
|
|
483
|
+
*/
|
|
484
|
+
outputType: 'int';
|
|
485
|
+
/**
|
|
486
|
+
* The description of the Metric.
|
|
487
|
+
*/
|
|
488
|
+
description?: string | null;
|
|
489
|
+
/**
|
|
490
|
+
* Guidelines for human evaluators.
|
|
491
|
+
*/
|
|
492
|
+
guidelines?: string;
|
|
493
|
+
/**
|
|
494
|
+
* The name of the Metric.
|
|
495
|
+
*/
|
|
496
|
+
name?: string;
|
|
497
|
+
/**
|
|
498
|
+
* The threshold for determining pass/fail from integer scores (1-5).
|
|
499
|
+
*/
|
|
500
|
+
passingThreshold?: number;
|
|
501
|
+
}
|
|
502
|
+
interface HeuristicIntMetric {
|
|
503
|
+
/**
|
|
504
|
+
* Heuristic-based evaluation type.
|
|
505
|
+
*/
|
|
506
|
+
evalType: 'heuristic';
|
|
507
|
+
/**
|
|
508
|
+
* Integer output type.
|
|
509
|
+
*/
|
|
510
|
+
outputType: 'int';
|
|
511
|
+
/**
|
|
512
|
+
* The description of the Metric.
|
|
513
|
+
*/
|
|
514
|
+
description?: string | null;
|
|
515
|
+
/**
|
|
516
|
+
* Optional guidelines for heuristic evaluation logic.
|
|
517
|
+
*/
|
|
518
|
+
guidelines?: string;
|
|
519
|
+
/**
|
|
520
|
+
* The name of the Metric.
|
|
521
|
+
*/
|
|
522
|
+
name?: string;
|
|
523
|
+
/**
|
|
524
|
+
* The threshold for determining pass/fail from integer scores (1-5).
|
|
525
|
+
*/
|
|
526
|
+
passingThreshold?: number;
|
|
527
|
+
}
|
|
528
|
+
interface AIBooleanMetric {
|
|
529
|
+
/**
|
|
530
|
+
* AI-based evaluation type.
|
|
531
|
+
*/
|
|
532
|
+
evalType: 'ai';
|
|
533
|
+
/**
|
|
534
|
+
* Boolean output type.
|
|
535
|
+
*/
|
|
536
|
+
outputType: 'boolean';
|
|
537
|
+
/**
|
|
538
|
+
* The description of the Metric.
|
|
539
|
+
*/
|
|
540
|
+
description?: string | null;
|
|
541
|
+
/**
|
|
542
|
+
* The AI model to use for evaluation.
|
|
543
|
+
*/
|
|
544
|
+
evalModelName?: string;
|
|
545
|
+
/**
|
|
546
|
+
* Guidelines for AI evaluation on how to score the metric.
|
|
547
|
+
*/
|
|
548
|
+
guidelines?: string | null;
|
|
549
|
+
/**
|
|
550
|
+
* The name of the Metric.
|
|
551
|
+
*/
|
|
552
|
+
name?: string;
|
|
553
|
+
/**
|
|
554
|
+
* The complete prompt template for AI evaluation. Should include placeholders for
|
|
555
|
+
* dynamic content.
|
|
556
|
+
*/
|
|
557
|
+
promptTemplate?: string;
|
|
558
|
+
/**
|
|
559
|
+
* The temperature for AI evaluation (0-2).
|
|
560
|
+
*/
|
|
561
|
+
temperature?: number;
|
|
562
|
+
}
|
|
563
|
+
interface HumanBooleanMetric {
|
|
564
|
+
/**
|
|
565
|
+
* Human-based evaluation type.
|
|
566
|
+
*/
|
|
567
|
+
evalType: 'human';
|
|
568
|
+
/**
|
|
569
|
+
* Boolean output type.
|
|
570
|
+
*/
|
|
571
|
+
outputType: 'boolean';
|
|
572
|
+
/**
|
|
573
|
+
* The description of the Metric.
|
|
574
|
+
*/
|
|
575
|
+
description?: string | null;
|
|
576
|
+
/**
|
|
577
|
+
* Guidelines for human evaluators.
|
|
578
|
+
*/
|
|
579
|
+
guidelines?: string;
|
|
580
|
+
/**
|
|
581
|
+
* The name of the Metric.
|
|
582
|
+
*/
|
|
583
|
+
name?: string;
|
|
584
|
+
}
|
|
585
|
+
interface HeuristicBooleanMetric {
|
|
586
|
+
/**
|
|
587
|
+
* Heuristic-based evaluation type.
|
|
588
|
+
*/
|
|
589
|
+
evalType: 'heuristic';
|
|
590
|
+
/**
|
|
591
|
+
* Boolean output type.
|
|
592
|
+
*/
|
|
593
|
+
outputType: 'boolean';
|
|
594
|
+
/**
|
|
595
|
+
* The description of the Metric.
|
|
596
|
+
*/
|
|
597
|
+
description?: string | null;
|
|
598
|
+
/**
|
|
599
|
+
* Optional guidelines for heuristic evaluation logic.
|
|
600
|
+
*/
|
|
601
|
+
guidelines?: string;
|
|
602
|
+
/**
|
|
603
|
+
* The name of the Metric.
|
|
604
|
+
*/
|
|
605
|
+
name?: string;
|
|
606
|
+
}
|
|
607
|
+
}
|
|
418
608
|
export declare namespace Metrics {
|
|
419
|
-
export { type Metric as Metric, type MetricCreateParams as MetricCreateParams };
|
|
609
|
+
export { type Metric as Metric, type MetricCreateParams as MetricCreateParams, type MetricUpdateParams as MetricUpdateParams, };
|
|
420
610
|
}
|
|
421
611
|
//# sourceMappingURL=metrics.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/resources/metrics.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,EAAE,UAAU,EAAE;OACd,EAAE,cAAc,EAAE;AAGzB,qBAAa,OAAQ,SAAQ,WAAW;IACtC
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/resources/metrics.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,EAAE,UAAU,EAAE;OACd,EAAE,cAAc,EAAE;AAGzB,qBAAa,OAAQ,SAAQ,WAAW;IACtC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;IAIjG;;;;;;;;;;;;;;OAcG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;CAGjG;AAED;;GAEG;AACH,MAAM,MAAM,MAAM,GACd,MAAM,CAAC,WAAW,GAClB,MAAM,CAAC,cAAc,GACrB,MAAM,CAAC,kBAAkB,GACzB,MAAM,CAAC,eAAe,GACtB,MAAM,CAAC,kBAAkB,GACzB,MAAM,CAAC,sBAAsB,CAAC;AAElC,yBAAiB,MAAM,CAAC;IACtB;;OAEG;IACH,UAAiB,WAAW;QAC1B;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,aAAa,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,QAAQ,EAAE,IAAI,CAAC;QAEf;;WAEG;QACH,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAE1B;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,gBAAgB,EAAE,MAAM,CAAC;QAEzB;;;WAGG;QACH,cAAc,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,WAAW,EAAE,MAAM,CAAC;KACrB;IAED;;OAEG;IACH,UAAiB,cAAc;QAC7B;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,QAAQ,EAAE,OAAO,CAAC;QAElB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,gBAAgB,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;IAED;;OAEG;IACH,UAAiB,kBAAkB;QACjC;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,QAAQ,EAAE,WAAW,CAAC;QAEtB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,gBAAgB,EAAE,MAAM,CAAC;QAEzB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;IAED;;OAEG;IACH,UAAiB,eAAe;QAC9B;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,aAAa,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,QAAQ,EAAE,IAAI,CAAC;QAEf;;WAEG;QACH,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAE1B;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;;WAGG;QACH,cAAc,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,WAAW,EAAE,MAAM,CAAC;KACrB;IAED;;OAEG;IACH,UAAiB,kBAAkB;QACjC;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,QAAQ,EAAE,OAAO,CAAC;QAElB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;IAED;;OAEG;IACH,UAAiB,sBAAsB;QACrC;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,QAAQ,EAAE,WAAW,CAAC;QAEtB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;CACF;AAED,MAAM,MAAM,kBAAkB,GAC1B,kBAAkB,CAAC,WAAW,GAC9B,kBAAkB,CAAC,cAAc,GACjC,kBAAkB,CAAC,kBAAkB,GACrC,kBAAkB,CAAC,eAAe,GAClC,kBAAkB,CAAC,kBAAkB,GACrC,kBAAkB,CAAC,sBAAsB,CAAC;AAE9C,MAAM,CAAC,OAAO,WAAW,kBAAkB,CAAC;IAC1C,UAAiB,WAAW;QAC1B;;WAEG;QACH,QAAQ,EAAE,IAAI,CAAC;QAEf;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;;WAGG;QACH,cAAc,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,aAAa,CAAC,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB;IAED,UAAiB,cAAc;QAC7B;;WAEG;QACH,QAAQ,EAAE,OAAO,CAAC;QAElB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B;IAED,UAAiB,kBAAkB;QACjC;;WAEG;QACH,QAAQ,EAAE,WAAW,CAAC;QAEtB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B;IAED,UAAiB,eAAe;QAC9B;;WAEG;QACH,QAAQ,EAAE,IAAI,CAAC;QAEf;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;;WAGG;QACH,cAAc,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,aAAa,CAAC,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB;IAED,UAAiB,kBAAkB;QACjC;;WAEG;QACH,QAAQ,EAAE,OAAO,CAAC;QAElB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;IAED,UAAiB,sBAAsB;QACrC;;WAEG;QACH,QAAQ,EAAE,WAAW,CAAC;QAEtB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB;CACF;AAED,MAAM,MAAM,kBAAkB,GAC1B,kBAAkB,CAAC,WAAW,GAC9B,kBAAkB,CAAC,cAAc,GACjC,kBAAkB,CAAC,kBAAkB,GACrC,kBAAkB,CAAC,eAAe,GAClC,kBAAkB,CAAC,kBAAkB,GACrC,kBAAkB,CAAC,sBAAsB,CAAC;AAE9C,MAAM,CAAC,OAAO,WAAW,kBAAkB,CAAC;IAC1C,UAAiB,WAAW;QAC1B;;WAEG;QACH,QAAQ,EAAE,IAAI,CAAC;QAEf;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,aAAa,CAAC,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B;;;WAGG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB;IAED,UAAiB,cAAc;QAC7B;;WAEG;QACH,QAAQ,EAAE,OAAO,CAAC;QAElB;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B;IAED,UAAiB,kBAAkB;QACjC;;WAEG;QACH,QAAQ,EAAE,WAAW,CAAC;QAEtB;;WAEG;QACH,UAAU,EAAE,KAAK,CAAC;QAElB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B;IAED,UAAiB,eAAe;QAC9B;;WAEG;QACH,QAAQ,EAAE,IAAI,CAAC;QAEf;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,aAAa,CAAC,EAAE,MAAM,CAAC;QAEvB;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE3B;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd;;;WAGG;QACH,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB;IAED,UAAiB,kBAAkB;QACjC;;WAEG;QACH,QAAQ,EAAE,OAAO,CAAC;QAElB;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;KACf;IAED,UAAiB,sBAAsB;QACrC;;WAEG;QACH,QAAQ,EAAE,WAAW,CAAC;QAEtB;;WAEG;QACH,UAAU,EAAE,SAAS,CAAC;QAEtB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE5B;;WAEG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;KACf;CACF;AAED,MAAM,CAAC,OAAO,WAAW,OAAO,CAAC;IAC/B,OAAO,EACL,KAAK,MAAM,IAAI,MAAM,EACrB,KAAK,kBAAkB,IAAI,kBAAkB,EAC7C,KAAK,kBAAkB,IAAI,kBAAkB,GAC9C,CAAC;CACH"}
|
package/resources/metrics.js
CHANGED
|
@@ -6,7 +6,8 @@ const resource_1 = require("../core/resource.js");
|
|
|
6
6
|
const path_1 = require("../internal/utils/path.js");
|
|
7
7
|
class Metrics extends resource_1.APIResource {
|
|
8
8
|
/**
|
|
9
|
-
* Create a new Metric for evaluating system outputs.
|
|
9
|
+
* Create a new Metric for evaluating system outputs. The structure of a metric
|
|
10
|
+
* depends on the evalType and outputType of the metric.
|
|
10
11
|
*
|
|
11
12
|
* @example
|
|
12
13
|
* ```ts
|
|
@@ -28,6 +29,24 @@ class Metrics extends resource_1.APIResource {
|
|
|
28
29
|
create(projectID, body, options) {
|
|
29
30
|
return this._client.post((0, path_1.path) `/projects/${projectID}/metrics`, { body, ...options });
|
|
30
31
|
}
|
|
32
|
+
/**
|
|
33
|
+
* Update an existing Metric. You must specify the evalType and outputType of the
|
|
34
|
+
* metric. The structure of a metric depends on the evalType and outputType of the
|
|
35
|
+
* metric.
|
|
36
|
+
*
|
|
37
|
+
* @example
|
|
38
|
+
* ```ts
|
|
39
|
+
* const metric = await client.metrics.update('321', {
|
|
40
|
+
* evalType: 'ai',
|
|
41
|
+
* outputType: 'boolean',
|
|
42
|
+
* promptTemplate:
|
|
43
|
+
* 'Using the following guidelines, evaluate the response: {{ guidelines }}\n\nResponse: {{ outputs.response }}\n\nIdeal answer: {{ expected.idealResponse }}',
|
|
44
|
+
* });
|
|
45
|
+
* ```
|
|
46
|
+
*/
|
|
47
|
+
update(metricID, body, options) {
|
|
48
|
+
return this._client.patch((0, path_1.path) `/metrics/${metricID}`, { body, ...options });
|
|
49
|
+
}
|
|
31
50
|
}
|
|
32
51
|
exports.Metrics = Metrics;
|
|
33
52
|
//# sourceMappingURL=metrics.js.map
|
package/resources/metrics.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../src/resources/metrics.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,kDAA+C;AAG/C,oDAA8C;AAE9C,MAAa,OAAQ,SAAQ,sBAAW;IACtC
|
|
1
|
+
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../src/resources/metrics.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,kDAA+C;AAG/C,oDAA8C;AAE9C,MAAa,OAAQ,SAAQ,sBAAW;IACtC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,MAAM,CAAC,SAAiB,EAAE,IAAwB,EAAE,OAAwB;QAC1E,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAA,WAAI,EAAA,aAAa,SAAS,UAAU,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACvF,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,MAAM,CAAC,QAAgB,EAAE,IAAwB,EAAE,OAAwB;QACzE,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAA,WAAI,EAAA,YAAY,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC9E,CAAC;CACF;AA5CD,0BA4CC"}
|
package/resources/metrics.mjs
CHANGED
|
@@ -3,7 +3,8 @@ import { APIResource } from "../core/resource.mjs";
|
|
|
3
3
|
import { path } from "../internal/utils/path.mjs";
|
|
4
4
|
export class Metrics extends APIResource {
|
|
5
5
|
/**
|
|
6
|
-
* Create a new Metric for evaluating system outputs.
|
|
6
|
+
* Create a new Metric for evaluating system outputs. The structure of a metric
|
|
7
|
+
* depends on the evalType and outputType of the metric.
|
|
7
8
|
*
|
|
8
9
|
* @example
|
|
9
10
|
* ```ts
|
|
@@ -25,5 +26,23 @@ export class Metrics extends APIResource {
|
|
|
25
26
|
create(projectID, body, options) {
|
|
26
27
|
return this._client.post(path `/projects/${projectID}/metrics`, { body, ...options });
|
|
27
28
|
}
|
|
29
|
+
/**
|
|
30
|
+
* Update an existing Metric. You must specify the evalType and outputType of the
|
|
31
|
+
* metric. The structure of a metric depends on the evalType and outputType of the
|
|
32
|
+
* metric.
|
|
33
|
+
*
|
|
34
|
+
* @example
|
|
35
|
+
* ```ts
|
|
36
|
+
* const metric = await client.metrics.update('321', {
|
|
37
|
+
* evalType: 'ai',
|
|
38
|
+
* outputType: 'boolean',
|
|
39
|
+
* promptTemplate:
|
|
40
|
+
* 'Using the following guidelines, evaluate the response: {{ guidelines }}\n\nResponse: {{ outputs.response }}\n\nIdeal answer: {{ expected.idealResponse }}',
|
|
41
|
+
* });
|
|
42
|
+
* ```
|
|
43
|
+
*/
|
|
44
|
+
update(metricID, body, options) {
|
|
45
|
+
return this._client.patch(path `/metrics/${metricID}`, { body, ...options });
|
|
46
|
+
}
|
|
28
47
|
}
|
|
29
48
|
//# sourceMappingURL=metrics.mjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.mjs","sourceRoot":"","sources":["../src/resources/metrics.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;OAGf,EAAE,IAAI,EAAE;AAEf,MAAM,OAAO,OAAQ,SAAQ,WAAW;IACtC
|
|
1
|
+
{"version":3,"file":"metrics.mjs","sourceRoot":"","sources":["../src/resources/metrics.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;OAGf,EAAE,IAAI,EAAE;AAEf,MAAM,OAAO,OAAQ,SAAQ,WAAW;IACtC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,MAAM,CAAC,SAAiB,EAAE,IAAwB,EAAE,OAAwB;QAC1E,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAA,aAAa,SAAS,UAAU,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACvF,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,MAAM,CAAC,QAAgB,EAAE,IAAwB,EAAE,OAAwB;QACzE,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAA,YAAY,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC9E,CAAC;CACF"}
|
package/src/client.ts
CHANGED
|
@@ -22,7 +22,7 @@ import { APIPromise } from './core/api-promise';
|
|
|
22
22
|
import { type Fetch } from './internal/builtin-types';
|
|
23
23
|
import { HeadersLike, NullableHeaders, buildHeaders } from './internal/headers';
|
|
24
24
|
import { FinalRequestOptions, RequestOptions } from './internal/request-options';
|
|
25
|
-
import { Metric, MetricCreateParams, Metrics } from './resources/metrics';
|
|
25
|
+
import { Metric, MetricCreateParams, MetricUpdateParams, Metrics } from './resources/metrics';
|
|
26
26
|
import {
|
|
27
27
|
Project,
|
|
28
28
|
ProjectCreateParams,
|
|
@@ -210,6 +210,8 @@ export class Scorecard {
|
|
|
210
210
|
"The SCORECARD_API_KEY environment variable is missing or empty; either provide it, or instantiate the Scorecard client with an apiKey option, like new Scorecard({ apiKey: 'My API Key' }).",
|
|
211
211
|
);
|
|
212
212
|
}
|
|
213
|
+
// Support both API keys (which start with 'ak_') and legacy JWT bearer tokens
|
|
214
|
+
apiKey = !apiKey || apiKey.startsWith('ak_') ? apiKey : `Bearer ${apiKey}`;
|
|
213
215
|
|
|
214
216
|
const options: ClientOptions = {
|
|
215
217
|
apiKey,
|
|
@@ -272,7 +274,7 @@ export class Scorecard {
|
|
|
272
274
|
}
|
|
273
275
|
|
|
274
276
|
protected authHeaders(opts: FinalRequestOptions): NullableHeaders | undefined {
|
|
275
|
-
return buildHeaders([{ Authorization:
|
|
277
|
+
return buildHeaders([{ Authorization: this.apiKey }]);
|
|
276
278
|
}
|
|
277
279
|
|
|
278
280
|
/**
|
|
@@ -849,7 +851,12 @@ export declare namespace Scorecard {
|
|
|
849
851
|
|
|
850
852
|
export { Runs as Runs, type Run as Run, type RunCreateParams as RunCreateParams };
|
|
851
853
|
|
|
852
|
-
export {
|
|
854
|
+
export {
|
|
855
|
+
Metrics as Metrics,
|
|
856
|
+
type Metric as Metric,
|
|
857
|
+
type MetricCreateParams as MetricCreateParams,
|
|
858
|
+
type MetricUpdateParams as MetricUpdateParams,
|
|
859
|
+
};
|
|
853
860
|
|
|
854
861
|
export {
|
|
855
862
|
Records as Records,
|
package/src/resources/index.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
|
2
2
|
|
|
3
3
|
export * from './shared';
|
|
4
|
-
export { Metrics, type Metric, type MetricCreateParams } from './metrics';
|
|
4
|
+
export { Metrics, type Metric, type MetricCreateParams, type MetricUpdateParams } from './metrics';
|
|
5
5
|
export {
|
|
6
6
|
Projects,
|
|
7
7
|
type Project,
|