weflayr 0.31.0 → 0.32.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/package.json
CHANGED
|
@@ -423,6 +423,7 @@ export declare const setRoutingStrategy: (options: import("../makeEndpoint.js").
|
|
|
423
423
|
model: string;
|
|
424
424
|
provider: string;
|
|
425
425
|
region: string;
|
|
426
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
426
427
|
}[];
|
|
427
428
|
globalRule: "same_quality_lowest_cost" | "same_cost_best_quality" | "best_quality" | "fixed_order";
|
|
428
429
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -436,6 +437,7 @@ export declare const setRoutingStrategy: (options: import("../makeEndpoint.js").
|
|
|
436
437
|
model: string;
|
|
437
438
|
provider: string;
|
|
438
439
|
region: string;
|
|
440
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
439
441
|
}[];
|
|
440
442
|
excludedCustomers?: string[] | undefined;
|
|
441
443
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -450,6 +452,7 @@ export declare const setRoutingStrategy: (options: import("../makeEndpoint.js").
|
|
|
450
452
|
model: string;
|
|
451
453
|
provider: string;
|
|
452
454
|
region: string;
|
|
455
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
453
456
|
}[];
|
|
454
457
|
excludedCustomers?: string[] | undefined;
|
|
455
458
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -461,6 +464,7 @@ export declare const setRoutingStrategy: (options: import("../makeEndpoint.js").
|
|
|
461
464
|
model: string;
|
|
462
465
|
provider: string;
|
|
463
466
|
region: string;
|
|
467
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
464
468
|
}[];
|
|
465
469
|
excludedCustomers?: string[] | undefined;
|
|
466
470
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -489,6 +493,7 @@ export declare const setRoutingStrategy: (options: import("../makeEndpoint.js").
|
|
|
489
493
|
model: string;
|
|
490
494
|
provider: string;
|
|
491
495
|
region: string;
|
|
496
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
492
497
|
}[];
|
|
493
498
|
globalRule: "same_quality_lowest_cost" | "same_cost_best_quality" | "best_quality" | "fixed_order";
|
|
494
499
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -502,6 +507,7 @@ export declare const setRoutingStrategy: (options: import("../makeEndpoint.js").
|
|
|
502
507
|
model: string;
|
|
503
508
|
provider: string;
|
|
504
509
|
region: string;
|
|
510
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
505
511
|
}[];
|
|
506
512
|
excludedCustomers?: string[] | undefined;
|
|
507
513
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -516,6 +522,7 @@ export declare const setRoutingStrategy: (options: import("../makeEndpoint.js").
|
|
|
516
522
|
model: string;
|
|
517
523
|
provider: string;
|
|
518
524
|
region: string;
|
|
525
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
519
526
|
}[];
|
|
520
527
|
excludedCustomers?: string[] | undefined;
|
|
521
528
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -527,6 +534,7 @@ export declare const setRoutingStrategy: (options: import("../makeEndpoint.js").
|
|
|
527
534
|
model: string;
|
|
528
535
|
provider: string;
|
|
529
536
|
region: string;
|
|
537
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
530
538
|
}[];
|
|
531
539
|
excludedCustomers?: string[] | undefined;
|
|
532
540
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -564,6 +572,7 @@ export declare const getRoutingStrategy: (options: import("../makeEndpoint.js").
|
|
|
564
572
|
model: string;
|
|
565
573
|
provider: string;
|
|
566
574
|
region: string;
|
|
575
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
567
576
|
}[];
|
|
568
577
|
globalRule: "same_quality_lowest_cost" | "same_cost_best_quality" | "best_quality" | "fixed_order";
|
|
569
578
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -577,6 +586,7 @@ export declare const getRoutingStrategy: (options: import("../makeEndpoint.js").
|
|
|
577
586
|
model: string;
|
|
578
587
|
provider: string;
|
|
579
588
|
region: string;
|
|
589
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
580
590
|
}[];
|
|
581
591
|
excludedCustomers?: string[] | undefined;
|
|
582
592
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -591,6 +601,7 @@ export declare const getRoutingStrategy: (options: import("../makeEndpoint.js").
|
|
|
591
601
|
model: string;
|
|
592
602
|
provider: string;
|
|
593
603
|
region: string;
|
|
604
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
594
605
|
}[];
|
|
595
606
|
excludedCustomers?: string[] | undefined;
|
|
596
607
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -602,6 +613,7 @@ export declare const getRoutingStrategy: (options: import("../makeEndpoint.js").
|
|
|
602
613
|
model: string;
|
|
603
614
|
provider: string;
|
|
604
615
|
region: string;
|
|
616
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
605
617
|
}[];
|
|
606
618
|
excludedCustomers?: string[] | undefined;
|
|
607
619
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -628,6 +640,7 @@ export declare const getRoutingStrategy: (options: import("../makeEndpoint.js").
|
|
|
628
640
|
model: string;
|
|
629
641
|
provider: string;
|
|
630
642
|
region: string;
|
|
643
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
631
644
|
}[];
|
|
632
645
|
globalRule: "same_quality_lowest_cost" | "same_cost_best_quality" | "best_quality" | "fixed_order";
|
|
633
646
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -641,6 +654,7 @@ export declare const getRoutingStrategy: (options: import("../makeEndpoint.js").
|
|
|
641
654
|
model: string;
|
|
642
655
|
provider: string;
|
|
643
656
|
region: string;
|
|
657
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
644
658
|
}[];
|
|
645
659
|
excludedCustomers?: string[] | undefined;
|
|
646
660
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -655,6 +669,7 @@ export declare const getRoutingStrategy: (options: import("../makeEndpoint.js").
|
|
|
655
669
|
model: string;
|
|
656
670
|
provider: string;
|
|
657
671
|
region: string;
|
|
672
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
658
673
|
}[];
|
|
659
674
|
excludedCustomers?: string[] | undefined;
|
|
660
675
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -666,6 +681,7 @@ export declare const getRoutingStrategy: (options: import("../makeEndpoint.js").
|
|
|
666
681
|
model: string;
|
|
667
682
|
provider: string;
|
|
668
683
|
region: string;
|
|
684
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
669
685
|
}[];
|
|
670
686
|
excludedCustomers?: string[] | undefined;
|
|
671
687
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -702,6 +718,7 @@ export declare const archiveRoutingStrategy: (options: import("../makeEndpoint.j
|
|
|
702
718
|
model: string;
|
|
703
719
|
provider: string;
|
|
704
720
|
region: string;
|
|
721
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
705
722
|
}[];
|
|
706
723
|
globalRule: "same_quality_lowest_cost" | "same_cost_best_quality" | "best_quality" | "fixed_order";
|
|
707
724
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -715,6 +732,7 @@ export declare const archiveRoutingStrategy: (options: import("../makeEndpoint.j
|
|
|
715
732
|
model: string;
|
|
716
733
|
provider: string;
|
|
717
734
|
region: string;
|
|
735
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
718
736
|
}[];
|
|
719
737
|
excludedCustomers?: string[] | undefined;
|
|
720
738
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -729,6 +747,7 @@ export declare const archiveRoutingStrategy: (options: import("../makeEndpoint.j
|
|
|
729
747
|
model: string;
|
|
730
748
|
provider: string;
|
|
731
749
|
region: string;
|
|
750
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
732
751
|
}[];
|
|
733
752
|
excludedCustomers?: string[] | undefined;
|
|
734
753
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -740,6 +759,7 @@ export declare const archiveRoutingStrategy: (options: import("../makeEndpoint.j
|
|
|
740
759
|
model: string;
|
|
741
760
|
provider: string;
|
|
742
761
|
region: string;
|
|
762
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
743
763
|
}[];
|
|
744
764
|
excludedCustomers?: string[] | undefined;
|
|
745
765
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -775,6 +795,7 @@ export declare const restoreRoutingStrategy: (options: import("../makeEndpoint.j
|
|
|
775
795
|
model: string;
|
|
776
796
|
provider: string;
|
|
777
797
|
region: string;
|
|
798
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
778
799
|
}[];
|
|
779
800
|
globalRule: "same_quality_lowest_cost" | "same_cost_best_quality" | "best_quality" | "fixed_order";
|
|
780
801
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -788,6 +809,7 @@ export declare const restoreRoutingStrategy: (options: import("../makeEndpoint.j
|
|
|
788
809
|
model: string;
|
|
789
810
|
provider: string;
|
|
790
811
|
region: string;
|
|
812
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
791
813
|
}[];
|
|
792
814
|
excludedCustomers?: string[] | undefined;
|
|
793
815
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -802,6 +824,7 @@ export declare const restoreRoutingStrategy: (options: import("../makeEndpoint.j
|
|
|
802
824
|
model: string;
|
|
803
825
|
provider: string;
|
|
804
826
|
region: string;
|
|
827
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
805
828
|
}[];
|
|
806
829
|
excludedCustomers?: string[] | undefined;
|
|
807
830
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -813,6 +836,7 @@ export declare const restoreRoutingStrategy: (options: import("../makeEndpoint.j
|
|
|
813
836
|
model: string;
|
|
814
837
|
provider: string;
|
|
815
838
|
region: string;
|
|
839
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
816
840
|
}[];
|
|
817
841
|
excludedCustomers?: string[] | undefined;
|
|
818
842
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -1171,6 +1195,7 @@ export declare const getBenchmark: (options: import("../makeEndpoint.js").Client
|
|
|
1171
1195
|
}[];
|
|
1172
1196
|
maskPii: boolean;
|
|
1173
1197
|
autoDisallowTooCostlyReasoningEfforts: boolean;
|
|
1198
|
+
ignoreMaxTokens: boolean | null;
|
|
1174
1199
|
};
|
|
1175
1200
|
};
|
|
1176
1201
|
baseline: {
|
|
@@ -1332,6 +1357,7 @@ export declare const getBenchmark: (options: import("../makeEndpoint.js").Client
|
|
|
1332
1357
|
model: string;
|
|
1333
1358
|
provider: string;
|
|
1334
1359
|
region: string;
|
|
1360
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
1335
1361
|
}[];
|
|
1336
1362
|
globalRule: "same_quality_lowest_cost" | "same_cost_best_quality" | "best_quality" | "fixed_order";
|
|
1337
1363
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -1345,6 +1371,7 @@ export declare const getBenchmark: (options: import("../makeEndpoint.js").Client
|
|
|
1345
1371
|
model: string;
|
|
1346
1372
|
provider: string;
|
|
1347
1373
|
region: string;
|
|
1374
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
1348
1375
|
}[];
|
|
1349
1376
|
excludedCustomers?: string[] | undefined;
|
|
1350
1377
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -1359,6 +1386,7 @@ export declare const getBenchmark: (options: import("../makeEndpoint.js").Client
|
|
|
1359
1386
|
model: string;
|
|
1360
1387
|
provider: string;
|
|
1361
1388
|
region: string;
|
|
1389
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
1362
1390
|
}[];
|
|
1363
1391
|
excludedCustomers?: string[] | undefined;
|
|
1364
1392
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -1370,6 +1398,7 @@ export declare const getBenchmark: (options: import("../makeEndpoint.js").Client
|
|
|
1370
1398
|
model: string;
|
|
1371
1399
|
provider: string;
|
|
1372
1400
|
region: string;
|
|
1401
|
+
reasoningEffort?: "default" | "high" | "low" | "none" | "minimal" | "medium" | "xhigh" | "max" | null | undefined;
|
|
1373
1402
|
}[];
|
|
1374
1403
|
excludedCustomers?: string[] | undefined;
|
|
1375
1404
|
switchFromCachedModelOrCheaperBetterModelCondition?: "never" | "same_or_better_quality_is_unlikely" | "best_quality_is_unlikely" | "anytime" | null | undefined;
|
|
@@ -1499,6 +1528,7 @@ export declare const setBenchmark: (options: import("../makeEndpoint.js").Client
|
|
|
1499
1528
|
}[] | undefined;
|
|
1500
1529
|
maskPii?: boolean | undefined;
|
|
1501
1530
|
autoDisallowTooCostlyReasoningEfforts?: boolean | undefined;
|
|
1531
|
+
ignoreMaxTokens?: boolean | undefined;
|
|
1502
1532
|
};
|
|
1503
1533
|
benchmarkUuid?: string | null | undefined;
|
|
1504
1534
|
requestId?: string | null | undefined;
|
|
@@ -1564,6 +1594,7 @@ export declare const setBenchmark: (options: import("../makeEndpoint.js").Client
|
|
|
1564
1594
|
}[];
|
|
1565
1595
|
maskPii: boolean;
|
|
1566
1596
|
autoDisallowTooCostlyReasoningEfforts: boolean;
|
|
1597
|
+
ignoreMaxTokens: boolean | null;
|
|
1567
1598
|
};
|
|
1568
1599
|
};
|
|
1569
1600
|
baseline: {
|
|
@@ -1651,6 +1682,7 @@ export declare const startOrResumeBenchmark: (options: import("../makeEndpoint.j
|
|
|
1651
1682
|
}[];
|
|
1652
1683
|
maskPii: boolean;
|
|
1653
1684
|
autoDisallowTooCostlyReasoningEfforts: boolean;
|
|
1685
|
+
ignoreMaxTokens: boolean | null;
|
|
1654
1686
|
};
|
|
1655
1687
|
};
|
|
1656
1688
|
baseline: {
|
|
@@ -1735,6 +1767,7 @@ export declare const stopBenchmark: (options: import("../makeEndpoint.js").Clien
|
|
|
1735
1767
|
}[];
|
|
1736
1768
|
maskPii: boolean;
|
|
1737
1769
|
autoDisallowTooCostlyReasoningEfforts: boolean;
|
|
1770
|
+
ignoreMaxTokens: boolean | null;
|
|
1738
1771
|
};
|
|
1739
1772
|
};
|
|
1740
1773
|
baseline: {
|
|
@@ -23,10 +23,7 @@ export type AllCustomerTagsOutput = {
|
|
|
23
23
|
/**
|
|
24
24
|
* AllowedDeployment
|
|
25
25
|
*
|
|
26
|
-
* A deployment the strategy can route to
|
|
27
|
-
* it: the model, the provider it is called on and the region it is called
|
|
28
|
-
* in, either a macro region ("EU") or the provider's own region
|
|
29
|
-
* ("eu-west-1" on Bedrock).
|
|
26
|
+
* A deployment the strategy can route to.
|
|
30
27
|
*/
|
|
31
28
|
export type AllowedDeployment = {
|
|
32
29
|
/**
|
|
@@ -41,6 +38,10 @@ export type AllowedDeployment = {
|
|
|
41
38
|
* Region
|
|
42
39
|
*/
|
|
43
40
|
region: string;
|
|
41
|
+
/**
|
|
42
|
+
* Reasoning Effort
|
|
43
|
+
*/
|
|
44
|
+
reasoning_effort?: 'default' | 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'max' | null;
|
|
44
45
|
};
|
|
45
46
|
/**
|
|
46
47
|
* ApiErrorOutput
|
|
@@ -213,9 +214,15 @@ export type BenchmarkConfigInput = {
|
|
|
213
214
|
/**
|
|
214
215
|
* Auto Disallow Too Costly Reasoning Efforts
|
|
215
216
|
*
|
|
216
|
-
* Stop testing a model at a reasoning effort that turns out far more expensive than its medium effort, and leave its answers out of the results: once the first 3 examples are done, then after every example, an effort costing on average more than 6 times the model's medium on the same calls (its default effort when it has no medium), while writing at least twice as many output tokens as the baseline, is removed.
|
|
217
|
+
* Stop testing a model at a reasoning effort that turns out far more expensive than its medium effort, and leave its answers out of the results: once the first 3 examples are done, then after every example, an effort costing on average more than 6 times the model's medium on the same calls (its default effort when it has no medium), while writing at least twice as many output tokens as the baseline, is removed. A model is also dropped from the benchmark when, at the effort it is first tested at, it writes at least 250,000 output tokens to a call whose logged answer wrote fewer than 10,000.
|
|
217
218
|
*/
|
|
218
219
|
auto_disallow_too_costly_reasoning_efforts?: boolean;
|
|
220
|
+
/**
|
|
221
|
+
* Ignore Max Tokens
|
|
222
|
+
*
|
|
223
|
+
* Replay the calls without the max_tokens they set. A model's reasoning tokens count toward max_tokens, so a model at a high reasoning effort can reach it before writing any answer. An answer with no text is judged worse than the baseline's. Ignored when none of the calls of the period sets max_tokens.
|
|
224
|
+
*/
|
|
225
|
+
ignore_max_tokens?: boolean;
|
|
219
226
|
};
|
|
220
227
|
/**
|
|
221
228
|
* BenchmarkConfigOutput
|
|
@@ -261,6 +268,12 @@ export type BenchmarkConfigOutput = {
|
|
|
261
268
|
* Auto Disallow Too Costly Reasoning Efforts
|
|
262
269
|
*/
|
|
263
270
|
auto_disallow_too_costly_reasoning_efforts: boolean;
|
|
271
|
+
/**
|
|
272
|
+
* Ignore Max Tokens
|
|
273
|
+
*
|
|
274
|
+
* Null when none of the calls of the period sets max_tokens.
|
|
275
|
+
*/
|
|
276
|
+
ignore_max_tokens: boolean | null;
|
|
264
277
|
};
|
|
265
278
|
/**
|
|
266
279
|
* BenchmarkDeletedOutput
|