toolnexus 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -0
- package/dist/batteries.d.ts +223 -0
- package/dist/batteries.js +488 -0
- package/dist/classifier.d.ts +14 -0
- package/dist/classifier.js +34 -1
- package/dist/client.d.ts +17 -13
- package/dist/client.js +85 -55
- package/dist/index.d.ts +3 -0
- package/dist/index.js +6 -0
- package/dist/judge.d.ts +98 -0
- package/dist/judge.js +199 -0
- package/package.json +2 -2
- package/src/batteries.ts +614 -0
- package/src/classifier.ts +36 -1
- package/src/client.ts +98 -55
- package/src/index.ts +26 -0
- package/src/judge.ts +305 -0
package/dist/client.js
CHANGED
|
@@ -254,8 +254,8 @@ export class Client {
|
|
|
254
254
|
}
|
|
255
255
|
/** §8 Gap 5. OpenAI-style body; omits tools/tool_choice when the tool list is empty. Key order
|
|
256
256
|
* matches the pre-change body so a no-options call is byte-identical. */
|
|
257
|
-
openaiBody(messages, tools, stream) {
|
|
258
|
-
const b = { model
|
|
257
|
+
openaiBody(messages, tools, stream, model = this.opts.model) {
|
|
258
|
+
const b = { model, messages: toOpenAIWire(messages, this.wireOpts()) };
|
|
259
259
|
if (tools.length) {
|
|
260
260
|
b.tools = tools;
|
|
261
261
|
b.tool_choice = "auto";
|
|
@@ -267,8 +267,8 @@ export class Client {
|
|
|
267
267
|
return this.finalizeBody(b);
|
|
268
268
|
}
|
|
269
269
|
/** §8 Gap 5. Anthropic-style body; omits tools when the tool list is empty. */
|
|
270
|
-
anthropicBody(system, messages, tools, stream) {
|
|
271
|
-
const b = { model
|
|
270
|
+
anthropicBody(system, messages, tools, stream, model = this.opts.model) {
|
|
271
|
+
const b = { model, max_tokens: 4096, system, messages: toAnthropicWire(messages, this.wireOpts()) };
|
|
272
272
|
if (tools.length)
|
|
273
273
|
b.tools = tools;
|
|
274
274
|
if (stream)
|
|
@@ -290,13 +290,13 @@ export class Client {
|
|
|
290
290
|
}
|
|
291
291
|
/** Emit the terminal `run` metric event and build the RunResult. `exhausted` marks a
|
|
292
292
|
* maxTurns loop exit — with no final text that is a LOUD "incomplete", never "done" (§7D). */
|
|
293
|
-
endRun(runStart, text, messages, toolCalls, turns, usage, exhausted = false) {
|
|
294
|
-
this.emit({ event: "run", model
|
|
295
|
-
return this.result(text, messages, toolCalls, turns, usage, exhausted);
|
|
293
|
+
endRun(runStart, model, text, messages, toolCalls, turns, usage, exhausted = false) {
|
|
294
|
+
this.emit({ event: "run", model, turns, toolCalls: toolCalls.length, totalTokens: usage.totalTokens, ms: Date.now() - runStart });
|
|
295
|
+
return this.result(model, text, messages, toolCalls, turns, usage, exhausted);
|
|
296
296
|
}
|
|
297
297
|
/** Emit a `run` error metric event (once, on a thrown run). */
|
|
298
|
-
emitRunError(runStart, toolCalls, turns, usage, e) {
|
|
299
|
-
this.emit({ event: "run", model
|
|
298
|
+
emitRunError(runStart, model, toolCalls, turns, usage, e) {
|
|
299
|
+
this.emit({ event: "run", model, turns, toolCalls: toolCalls.length, totalTokens: usage.totalTokens, ms: Date.now() - runStart, error: e instanceof Error ? e.message : String(e) });
|
|
300
300
|
}
|
|
301
301
|
/**
|
|
302
302
|
* Run the loop. `ctx` — and `ctx.toolkit` inside it — are OPTIONAL: an absent toolkit is a
|
|
@@ -334,14 +334,17 @@ export class Client {
|
|
|
334
334
|
if (sys && !hasSystemMessage(messages))
|
|
335
335
|
messages = [{ role: "system", content: sys }, ...messages];
|
|
336
336
|
let tools = declared;
|
|
337
|
+
let turnModel = this.opts.model;
|
|
337
338
|
if (this.opts.hooks?.beforeLLM) {
|
|
338
339
|
const ov = await this.opts.hooks.beforeLLM({ messages, tools, model: this.opts.model, turn: 0 });
|
|
339
340
|
if (ov?.messages)
|
|
340
341
|
messages = ov.messages;
|
|
341
342
|
if (ov?.tools)
|
|
342
343
|
tools = ov.tools;
|
|
344
|
+
if (ov?.model)
|
|
345
|
+
turnModel = ov.model;
|
|
343
346
|
}
|
|
344
|
-
const body = { model:
|
|
347
|
+
const body = { model: turnModel, messages };
|
|
345
348
|
if (tools.length)
|
|
346
349
|
body.tools = tools;
|
|
347
350
|
if (req.toolChoice !== undefined)
|
|
@@ -354,9 +357,9 @@ export class Client {
|
|
|
354
357
|
method: "POST",
|
|
355
358
|
headers: { "content-type": "application/json", authorization: `Bearer ${key}`, ...this.opts.headers },
|
|
356
359
|
body: JSON.stringify(this.finalizeBody(body)),
|
|
357
|
-
}, signal, "openai");
|
|
360
|
+
}, signal, "openai", turnModel);
|
|
358
361
|
if (this.opts.hooks?.afterLLM)
|
|
359
|
-
await this.opts.hooks.afterLLM({ response: data, model:
|
|
362
|
+
await this.opts.hooks.afterLLM({ response: data, model: turnModel, turn: 0 });
|
|
360
363
|
const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
|
|
361
364
|
addUsage(usage, data.usage, "openai");
|
|
362
365
|
const choice = data.choices?.[0];
|
|
@@ -370,7 +373,7 @@ export class Client {
|
|
|
370
373
|
toolCalls,
|
|
371
374
|
finishReason: choice?.finish_reason ?? finishReasonFor(toolCalls.length > 0),
|
|
372
375
|
usage,
|
|
373
|
-
model:
|
|
376
|
+
model: turnModel,
|
|
374
377
|
raw: data,
|
|
375
378
|
};
|
|
376
379
|
}
|
|
@@ -390,15 +393,18 @@ export class Client {
|
|
|
390
393
|
...(req.toolkit ? req.toolkit.toAnthropic() : []),
|
|
391
394
|
...openAIToolsToAnthropic(req.tools),
|
|
392
395
|
];
|
|
396
|
+
let turnModel = this.opts.model;
|
|
393
397
|
if (this.opts.hooks?.beforeLLM) {
|
|
394
398
|
const ov = await this.opts.hooks.beforeLLM({ messages, tools, model: this.opts.model, turn: 0 });
|
|
395
399
|
if (ov?.messages)
|
|
396
400
|
messages = ov.messages;
|
|
397
401
|
if (ov?.tools)
|
|
398
402
|
tools = ov.tools;
|
|
403
|
+
if (ov?.model)
|
|
404
|
+
turnModel = ov.model;
|
|
399
405
|
}
|
|
400
406
|
const body = {
|
|
401
|
-
model:
|
|
407
|
+
model: turnModel,
|
|
402
408
|
max_tokens: req.maxTokens && req.maxTokens > 0 ? req.maxTokens : 4096,
|
|
403
409
|
messages,
|
|
404
410
|
};
|
|
@@ -420,9 +426,9 @@ export class Client {
|
|
|
420
426
|
...this.opts.headers,
|
|
421
427
|
},
|
|
422
428
|
body: JSON.stringify(this.finalizeBody(body)),
|
|
423
|
-
}, signal, "anthropic");
|
|
429
|
+
}, signal, "anthropic", turnModel);
|
|
424
430
|
if (this.opts.hooks?.afterLLM)
|
|
425
|
-
await this.opts.hooks.afterLLM({ response: data, model:
|
|
431
|
+
await this.opts.hooks.afterLLM({ response: data, model: turnModel, turn: 0 });
|
|
426
432
|
const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
|
|
427
433
|
addUsage(usage, data.usage, "anthropic");
|
|
428
434
|
const text = [];
|
|
@@ -443,7 +449,7 @@ export class Client {
|
|
|
443
449
|
toolCalls,
|
|
444
450
|
finishReason: finishReasonFor(toolCalls.length > 0, data.stop_reason),
|
|
445
451
|
usage,
|
|
446
|
-
model:
|
|
452
|
+
model: turnModel,
|
|
447
453
|
raw: data,
|
|
448
454
|
};
|
|
449
455
|
}
|
|
@@ -618,7 +624,7 @@ export class Client {
|
|
|
618
624
|
this.emit({ event: "tool", tool: name, source, isError: req ? false : result.isError, ms: Date.now() - t0, ...(req ? { pending: true } : {}) });
|
|
619
625
|
}
|
|
620
626
|
/** One non-streaming LLM call, with an `llm` metric event (ok/error + per-call tokens + ms). */
|
|
621
|
-
async llmCallJson(url, init, signal, style) {
|
|
627
|
+
async llmCallJson(url, init, signal, style, model) {
|
|
622
628
|
const t0 = Date.now();
|
|
623
629
|
try {
|
|
624
630
|
const res = await this.llmFetch(url, init, signal);
|
|
@@ -626,11 +632,11 @@ export class Client {
|
|
|
626
632
|
throw await llmHttpError(res);
|
|
627
633
|
const data = await res.json();
|
|
628
634
|
const tok = perCall(data.usage, style);
|
|
629
|
-
this.emit({ event: "llm", model
|
|
635
|
+
this.emit({ event: "llm", model, status: "ok", ms: Date.now() - t0, promptTokens: tok.prompt, completionTokens: tok.completion });
|
|
630
636
|
return data;
|
|
631
637
|
}
|
|
632
638
|
catch (e) {
|
|
633
|
-
this.emit({ event: "llm", model
|
|
639
|
+
this.emit({ event: "llm", model, status: "error", ms: Date.now() - t0, promptTokens: 0, completionTokens: 0 });
|
|
634
640
|
throw e;
|
|
635
641
|
}
|
|
636
642
|
}
|
|
@@ -687,29 +693,35 @@ export class Client {
|
|
|
687
693
|
const toolCalls = [];
|
|
688
694
|
const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
|
|
689
695
|
let turns = 0;
|
|
696
|
+
// The model of the last call this run made (a beforeLLM override when one applied) — §8.
|
|
697
|
+
let lastModel = this.opts.model;
|
|
690
698
|
try {
|
|
691
699
|
for (let turn = 0; turn < (this.opts.maxTurns ?? 10); turn++) {
|
|
692
700
|
turns++;
|
|
701
|
+
let turnModel = this.opts.model;
|
|
693
702
|
if (this.opts.hooks?.beforeLLM) {
|
|
694
703
|
const ov = await this.opts.hooks.beforeLLM({ messages, tools, model: this.opts.model, turn });
|
|
695
704
|
if (ov?.messages)
|
|
696
705
|
messages = ov.messages;
|
|
697
706
|
if (ov?.tools)
|
|
698
707
|
tools = ov.tools;
|
|
708
|
+
if (ov?.model)
|
|
709
|
+
turnModel = ov.model;
|
|
699
710
|
}
|
|
711
|
+
lastModel = turnModel;
|
|
700
712
|
const data = await this.llmCallJson(`${this.opts.baseUrl.replace(/\/$/, "")}/chat/completions`, {
|
|
701
713
|
method: "POST",
|
|
702
714
|
headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json", ...this.opts.headers },
|
|
703
|
-
body: JSON.stringify(this.openaiBody(messages, tools, false)),
|
|
704
|
-
}, signal, "openai");
|
|
715
|
+
body: JSON.stringify(this.openaiBody(messages, tools, false, turnModel)),
|
|
716
|
+
}, signal, "openai", turnModel);
|
|
705
717
|
addUsage(usage, data.usage, "openai");
|
|
706
718
|
if (this.opts.hooks?.afterLLM)
|
|
707
|
-
await this.opts.hooks.afterLLM({ response: data, model:
|
|
719
|
+
await this.opts.hooks.afterLLM({ response: data, model: turnModel, turn });
|
|
708
720
|
const msg = data.choices[0].message;
|
|
709
721
|
messages.push(msg);
|
|
710
722
|
const calls = msg.tool_calls ?? [];
|
|
711
723
|
if (calls.length === 0)
|
|
712
|
-
return this.endRun(runStart, msg.content ?? "", messages, toolCalls, turns, usage);
|
|
724
|
+
return this.endRun(runStart, lastModel, msg.content ?? "", messages, toolCalls, turns, usage);
|
|
713
725
|
// execute all tool calls in this turn concurrently (true parallel tool calling)
|
|
714
726
|
const settled = await Promise.all(calls.map(async (call) => {
|
|
715
727
|
let { args, result } = await this.runTool(toolkit, call.function.name, safeJson(call.function.arguments), call.id, turn);
|
|
@@ -729,27 +741,27 @@ export class Client {
|
|
|
729
741
|
toolCalls.push({ name: s.call.function.name, args: s.args, output: s.result.output, isError: s.result.isError, metadata: s.result.metadata });
|
|
730
742
|
messages.push(toolMessage(s.call.id, s.call.function.name, s.result));
|
|
731
743
|
if (s.halted)
|
|
732
|
-
return this.pendingRun(runStart, s.halted, messages, toolCalls, turns, usage);
|
|
744
|
+
return this.pendingRun(runStart, lastModel, s.halted, messages, toolCalls, turns, usage);
|
|
733
745
|
}
|
|
734
746
|
}
|
|
735
|
-
return this.endRun(runStart, lastText(messages), messages, toolCalls, turns, usage, true);
|
|
747
|
+
return this.endRun(runStart, lastModel, lastText(messages), messages, toolCalls, turns, usage, true);
|
|
736
748
|
}
|
|
737
749
|
catch (e) {
|
|
738
|
-
this.emitRunError(runStart, toolCalls, turns, usage, e);
|
|
750
|
+
this.emitRunError(runStart, lastModel, toolCalls, turns, usage, e);
|
|
739
751
|
throw e;
|
|
740
752
|
}
|
|
741
753
|
}
|
|
742
|
-
result(text, messages, toolCalls, turns, usage, exhausted = false) {
|
|
754
|
+
result(model, text, messages, toolCalls, turns, usage, exhausted = false) {
|
|
743
755
|
const incomplete = exhausted && text === "";
|
|
744
|
-
const r = { text, messages, toolCalls, toolCallCount: toolCalls.length, turns, usage, model
|
|
756
|
+
const r = { text, messages, toolCalls, toolCallCount: toolCalls.length, turns, usage, model, status: incomplete ? "incomplete" : "done" };
|
|
745
757
|
if (incomplete)
|
|
746
758
|
r.limit = "maxTurns";
|
|
747
759
|
return r;
|
|
748
760
|
}
|
|
749
761
|
/** §10: a run halted because a tool suspended and no `waitFor` was configured. */
|
|
750
|
-
pendingRun(runStart, request, messages, toolCalls, turns, usage) {
|
|
751
|
-
this.emit({ event: "run", model
|
|
752
|
-
return { text: request.prompt, messages, toolCalls, toolCallCount: toolCalls.length, turns, usage, model
|
|
762
|
+
pendingRun(runStart, model, request, messages, toolCalls, turns, usage) {
|
|
763
|
+
this.emit({ event: "run", model, turns, toolCalls: toolCalls.length, totalTokens: usage.totalTokens, ms: Date.now() - runStart });
|
|
764
|
+
return { text: request.prompt, messages, toolCalls, toolCallCount: toolCalls.length, turns, usage, model, status: "pending", pending: request };
|
|
753
765
|
}
|
|
754
766
|
/**
|
|
755
767
|
* §10: resolve a suspended tool result. Calls `waitFor(request)`, and on `ok` re-executes the
|
|
@@ -788,16 +800,22 @@ export class Client {
|
|
|
788
800
|
const toolCalls = [];
|
|
789
801
|
const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
|
|
790
802
|
let turns = 0;
|
|
803
|
+
// The model of the last call this run made (a beforeLLM override when one applied) — §8.
|
|
804
|
+
let lastModel = this.opts.model;
|
|
791
805
|
try {
|
|
792
806
|
for (let turn = 0; turn < (this.opts.maxTurns ?? 10); turn++) {
|
|
793
807
|
turns++;
|
|
808
|
+
let turnModel = this.opts.model;
|
|
794
809
|
if (this.opts.hooks?.beforeLLM) {
|
|
795
810
|
const ov = await this.opts.hooks.beforeLLM({ messages, tools, model: this.opts.model, turn });
|
|
796
811
|
if (ov?.messages)
|
|
797
812
|
messages = ov.messages;
|
|
798
813
|
if (ov?.tools)
|
|
799
814
|
tools = ov.tools;
|
|
815
|
+
if (ov?.model)
|
|
816
|
+
turnModel = ov.model;
|
|
800
817
|
}
|
|
818
|
+
lastModel = turnModel;
|
|
801
819
|
const data = await this.llmCallJson(endpoint, {
|
|
802
820
|
method: "POST",
|
|
803
821
|
headers: {
|
|
@@ -806,16 +824,16 @@ export class Client {
|
|
|
806
824
|
"Content-Type": "application/json",
|
|
807
825
|
...this.opts.headers,
|
|
808
826
|
},
|
|
809
|
-
body: JSON.stringify(this.anthropicBody(system, messages, tools, false)),
|
|
810
|
-
}, signal, "anthropic");
|
|
827
|
+
body: JSON.stringify(this.anthropicBody(system, messages, tools, false, turnModel)),
|
|
828
|
+
}, signal, "anthropic", turnModel);
|
|
811
829
|
addUsage(usage, data.usage, "anthropic");
|
|
812
830
|
if (this.opts.hooks?.afterLLM)
|
|
813
|
-
await this.opts.hooks.afterLLM({ response: data, model:
|
|
831
|
+
await this.opts.hooks.afterLLM({ response: data, model: turnModel, turn });
|
|
814
832
|
messages.push({ role: "assistant", content: data.content });
|
|
815
833
|
const uses = (data.content ?? []).filter((b) => b.type === "tool_use");
|
|
816
834
|
if (uses.length === 0) {
|
|
817
835
|
const text = (data.content ?? []).filter((b) => b.type === "text").map((b) => b.text).join("");
|
|
818
|
-
return this.endRun(runStart, text, messages, toolCalls, turns, usage);
|
|
836
|
+
return this.endRun(runStart, lastModel, text, messages, toolCalls, turns, usage);
|
|
819
837
|
}
|
|
820
838
|
// execute all tool_use blocks in this turn concurrently (true parallel tool calling)
|
|
821
839
|
const settled = await Promise.all(uses.map(async (use) => {
|
|
@@ -844,12 +862,12 @@ export class Client {
|
|
|
844
862
|
}
|
|
845
863
|
messages.push({ role: "user", content });
|
|
846
864
|
if (halted)
|
|
847
|
-
return this.pendingRun(runStart, halted, messages, toolCalls, turns, usage);
|
|
865
|
+
return this.pendingRun(runStart, lastModel, halted, messages, toolCalls, turns, usage);
|
|
848
866
|
}
|
|
849
|
-
return this.endRun(runStart, "", messages, toolCalls, turns, usage, true);
|
|
867
|
+
return this.endRun(runStart, lastModel, "", messages, toolCalls, turns, usage, true);
|
|
850
868
|
}
|
|
851
869
|
catch (e) {
|
|
852
|
-
this.emitRunError(runStart, toolCalls, turns, usage, e);
|
|
870
|
+
this.emitRunError(runStart, lastModel, toolCalls, turns, usage, e);
|
|
853
871
|
throw e;
|
|
854
872
|
}
|
|
855
873
|
}
|
|
@@ -868,16 +886,22 @@ export class Client {
|
|
|
868
886
|
const toolCalls = [];
|
|
869
887
|
const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
|
|
870
888
|
let turns = 0;
|
|
889
|
+
// The model of the last call this run made (a beforeLLM override when one applied) — §8.
|
|
890
|
+
let lastModel = this.opts.model;
|
|
871
891
|
try {
|
|
872
892
|
for (let turn = 0; turn < (this.opts.maxTurns ?? 10); turn++) {
|
|
873
893
|
turns++;
|
|
894
|
+
let turnModel = this.opts.model;
|
|
874
895
|
if (this.opts.hooks?.beforeLLM) {
|
|
875
896
|
const ov = await this.opts.hooks.beforeLLM({ messages, tools, model: this.opts.model, turn });
|
|
876
897
|
if (ov?.messages)
|
|
877
898
|
messages = ov.messages;
|
|
878
899
|
if (ov?.tools)
|
|
879
900
|
tools = ov.tools;
|
|
901
|
+
if (ov?.model)
|
|
902
|
+
turnModel = ov.model;
|
|
880
903
|
}
|
|
904
|
+
lastModel = turnModel;
|
|
881
905
|
const t0 = Date.now();
|
|
882
906
|
const beforeP = usage.promptTokens, beforeC = usage.completionTokens;
|
|
883
907
|
let content = "";
|
|
@@ -886,7 +910,7 @@ export class Client {
|
|
|
886
910
|
const res = await this.llmFetch(`${this.opts.baseUrl.replace(/\/$/, "")}/chat/completions`, {
|
|
887
911
|
method: "POST",
|
|
888
912
|
headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json", ...this.opts.headers },
|
|
889
|
-
body: JSON.stringify(this.openaiBody(messages, tools, true)),
|
|
913
|
+
body: JSON.stringify(this.openaiBody(messages, tools, true, turnModel)),
|
|
890
914
|
}, signal);
|
|
891
915
|
if (!res.ok || !res.body)
|
|
892
916
|
throw await llmHttpError(res);
|
|
@@ -918,17 +942,17 @@ export class Client {
|
|
|
918
942
|
}
|
|
919
943
|
}
|
|
920
944
|
catch (e) {
|
|
921
|
-
this.emit({ event: "llm", model:
|
|
945
|
+
this.emit({ event: "llm", model: turnModel, status: "error", ms: Date.now() - t0, promptTokens: 0, completionTokens: 0 });
|
|
922
946
|
throw e;
|
|
923
947
|
}
|
|
924
|
-
this.emit({ event: "llm", model:
|
|
948
|
+
this.emit({ event: "llm", model: turnModel, status: "ok", ms: Date.now() - t0, promptTokens: usage.promptTokens - beforeP, completionTokens: usage.completionTokens - beforeC });
|
|
925
949
|
if (this.opts.hooks?.afterLLM)
|
|
926
|
-
await this.opts.hooks.afterLLM({ response: { streamed: true, usage }, model:
|
|
950
|
+
await this.opts.hooks.afterLLM({ response: { streamed: true, usage }, model: turnModel, turn });
|
|
927
951
|
const calls = [...acc.values()];
|
|
928
952
|
if (calls.length === 0) {
|
|
929
953
|
messages.push({ role: "assistant", content });
|
|
930
954
|
yield { type: "usage", usage };
|
|
931
|
-
yield { type: "done", result: this.endRun(runStart, content, messages, toolCalls, turns, usage) };
|
|
955
|
+
yield { type: "done", result: this.endRun(runStart, lastModel, content, messages, toolCalls, turns, usage) };
|
|
932
956
|
return;
|
|
933
957
|
}
|
|
934
958
|
messages.push({ role: "assistant", content: content || null, tool_calls: calls.map((c) => ({ id: c.id, type: "function", function: { name: c.name, arguments: c.args } })) });
|
|
@@ -946,7 +970,7 @@ export class Client {
|
|
|
946
970
|
if (r.halted) {
|
|
947
971
|
toolCalls.push({ name: c.name, args, output: result.output, isError: result.isError, metadata: result.metadata });
|
|
948
972
|
messages.push(toolMessage(c.id, c.name, result));
|
|
949
|
-
yield { type: "done", result: this.pendingRun(runStart, r.halted, messages, toolCalls, turns, usage) };
|
|
973
|
+
yield { type: "done", result: this.pendingRun(runStart, lastModel, r.halted, messages, toolCalls, turns, usage) };
|
|
950
974
|
return;
|
|
951
975
|
}
|
|
952
976
|
}
|
|
@@ -955,10 +979,10 @@ export class Client {
|
|
|
955
979
|
yield { type: "tool_result", id: c.id, name: c.name, output: result.output, isError: result.isError };
|
|
956
980
|
}
|
|
957
981
|
}
|
|
958
|
-
yield { type: "done", result: this.endRun(runStart, lastText(messages), messages, toolCalls, turns, usage, true) };
|
|
982
|
+
yield { type: "done", result: this.endRun(runStart, lastModel, lastText(messages), messages, toolCalls, turns, usage, true) };
|
|
959
983
|
}
|
|
960
984
|
catch (e) {
|
|
961
|
-
this.emitRunError(runStart, toolCalls, turns, usage, e);
|
|
985
|
+
this.emitRunError(runStart, lastModel, toolCalls, turns, usage, e);
|
|
962
986
|
throw e;
|
|
963
987
|
}
|
|
964
988
|
}
|
|
@@ -974,16 +998,22 @@ export class Client {
|
|
|
974
998
|
const toolCalls = [];
|
|
975
999
|
const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
|
|
976
1000
|
let turns = 0;
|
|
1001
|
+
// The model of the last call this run made (a beforeLLM override when one applied) — §8.
|
|
1002
|
+
let lastModel = this.opts.model;
|
|
977
1003
|
try {
|
|
978
1004
|
for (let turn = 0; turn < (this.opts.maxTurns ?? 10); turn++) {
|
|
979
1005
|
turns++;
|
|
1006
|
+
let turnModel = this.opts.model;
|
|
980
1007
|
if (this.opts.hooks?.beforeLLM) {
|
|
981
1008
|
const ov = await this.opts.hooks.beforeLLM({ messages, tools, model: this.opts.model, turn });
|
|
982
1009
|
if (ov?.messages)
|
|
983
1010
|
messages = ov.messages;
|
|
984
1011
|
if (ov?.tools)
|
|
985
1012
|
tools = ov.tools;
|
|
1013
|
+
if (ov?.model)
|
|
1014
|
+
turnModel = ov.model;
|
|
986
1015
|
}
|
|
1016
|
+
lastModel = turnModel;
|
|
987
1017
|
const t0 = Date.now();
|
|
988
1018
|
const beforeP = usage.promptTokens, beforeC = usage.completionTokens;
|
|
989
1019
|
const blocks = new Map();
|
|
@@ -992,7 +1022,7 @@ export class Client {
|
|
|
992
1022
|
const res = await this.llmFetch(endpoint, {
|
|
993
1023
|
method: "POST",
|
|
994
1024
|
headers: { "x-api-key": key, "anthropic-version": "2023-06-01", "Content-Type": "application/json", ...this.opts.headers },
|
|
995
|
-
body: JSON.stringify(this.anthropicBody(system, messages, tools, true)),
|
|
1025
|
+
body: JSON.stringify(this.anthropicBody(system, messages, tools, true, turnModel)),
|
|
996
1026
|
}, signal);
|
|
997
1027
|
if (!res.ok || !res.body)
|
|
998
1028
|
throw await llmHttpError(res);
|
|
@@ -1022,19 +1052,19 @@ export class Client {
|
|
|
1022
1052
|
}
|
|
1023
1053
|
}
|
|
1024
1054
|
catch (e) {
|
|
1025
|
-
this.emit({ event: "llm", model:
|
|
1055
|
+
this.emit({ event: "llm", model: turnModel, status: "error", ms: Date.now() - t0, promptTokens: 0, completionTokens: 0 });
|
|
1026
1056
|
throw e;
|
|
1027
1057
|
}
|
|
1028
|
-
this.emit({ event: "llm", model:
|
|
1058
|
+
this.emit({ event: "llm", model: turnModel, status: "ok", ms: Date.now() - t0, promptTokens: usage.promptTokens - beforeP, completionTokens: usage.completionTokens - beforeC });
|
|
1029
1059
|
if (this.opts.hooks?.afterLLM)
|
|
1030
|
-
await this.opts.hooks.afterLLM({ response: { streamed: true, usage }, model:
|
|
1060
|
+
await this.opts.hooks.afterLLM({ response: { streamed: true, usage }, model: turnModel, turn });
|
|
1031
1061
|
const content = [...blocks.values()].map((b) => b.type === "tool_use" ? { type: "tool_use", id: b.id, name: b.name, input: safeJson(b.json || "{}") } : { type: "text", text: b.text });
|
|
1032
1062
|
messages.push({ role: "assistant", content });
|
|
1033
1063
|
const uses = content.filter((b) => b.type === "tool_use");
|
|
1034
1064
|
if (stopReason !== "tool_use" || uses.length === 0) {
|
|
1035
1065
|
const text = content.filter((b) => b.type === "text").map((b) => b.text).join("");
|
|
1036
1066
|
yield { type: "usage", usage };
|
|
1037
|
-
yield { type: "done", result: this.endRun(runStart, text, messages, toolCalls, turns, usage) };
|
|
1067
|
+
yield { type: "done", result: this.endRun(runStart, lastModel, text, messages, toolCalls, turns, usage) };
|
|
1038
1068
|
return;
|
|
1039
1069
|
}
|
|
1040
1070
|
for (const u of uses)
|
|
@@ -1053,7 +1083,7 @@ export class Client {
|
|
|
1053
1083
|
toolCalls.push({ name: u.name, args, output: result.output, isError: result.isError, metadata: result.metadata });
|
|
1054
1084
|
results.push(toolResultBlock(u.id, result));
|
|
1055
1085
|
messages.push({ role: "user", content: results });
|
|
1056
|
-
yield { type: "done", result: this.pendingRun(runStart, r.halted, messages, toolCalls, turns, usage) };
|
|
1086
|
+
yield { type: "done", result: this.pendingRun(runStart, lastModel, r.halted, messages, toolCalls, turns, usage) };
|
|
1057
1087
|
return;
|
|
1058
1088
|
}
|
|
1059
1089
|
}
|
|
@@ -1063,10 +1093,10 @@ export class Client {
|
|
|
1063
1093
|
}
|
|
1064
1094
|
messages.push({ role: "user", content: results });
|
|
1065
1095
|
}
|
|
1066
|
-
yield { type: "done", result: this.endRun(runStart, "", messages, toolCalls, turns, usage, true) };
|
|
1096
|
+
yield { type: "done", result: this.endRun(runStart, lastModel, "", messages, toolCalls, turns, usage, true) };
|
|
1067
1097
|
}
|
|
1068
1098
|
catch (e) {
|
|
1069
|
-
this.emitRunError(runStart, toolCalls, turns, usage, e);
|
|
1099
|
+
this.emitRunError(runStart, lastModel, toolCalls, turns, usage, e);
|
|
1070
1100
|
throw e;
|
|
1071
1101
|
}
|
|
1072
1102
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -16,3 +16,6 @@ export * from "./toolkit.js";
|
|
|
16
16
|
export * from "./client.js";
|
|
17
17
|
export * from "./classifier.js";
|
|
18
18
|
export * as agents from "./agents/index.js";
|
|
19
|
+
export * as judge from "./judge.js";
|
|
20
|
+
export * from "./batteries.js";
|
|
21
|
+
export { State, context, questionMap, ask, gate, decide, applyGate, applyPolicy, answersOf, staticClassifier, Tape, DEFAULT_BANDS, type NamedQuestion, type Band, type Bands, type Answers, type Rule, type GateOutcome, type Policy, } from "./judge.js";
|
package/dist/index.js
CHANGED
|
@@ -16,3 +16,9 @@ export * from "./toolkit.js";
|
|
|
16
16
|
export * from "./client.js";
|
|
17
17
|
export * from "./classifier.js";
|
|
18
18
|
export * as agents from "./agents/index.js";
|
|
19
|
+
// Simple judgments (§8B, add-judge-adapters). `judge.noul/choice/score` are the NAMED builders
|
|
20
|
+
// (the bare `noul/choice/score` above stay the §8B wire builders); the rest is also top-level.
|
|
21
|
+
export * as judge from "./judge.js";
|
|
22
|
+
// Judge batteries (§8B "Batteries", add-judge-batteries): eight `*Classifier` values + latestUserText.
|
|
23
|
+
export * from "./batteries.js";
|
|
24
|
+
export { State, context, questionMap, ask, gate, decide, applyGate, applyPolicy, answersOf, staticClassifier, Tape, DEFAULT_BANDS, } from "./judge.js";
|
package/dist/judge.d.ts
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Simple judgments (SPEC.md §8B "Simple judgments — ask / gate", change `add-judge-adapters`).
|
|
3
|
+
* A thin layer over any `Classifier`: named question builders, a role-carrying `State`,
|
|
4
|
+
* `ask` (answers by name with bands) and `gate` / `Policy` (rules that escalate instead of
|
|
5
|
+
* deciding when unsure). The wire is unchanged — the builders produce exactly the §8B
|
|
6
|
+
* `evaluate(state, questions)` inputs.
|
|
7
|
+
*
|
|
8
|
+
* A gate never authorises; it only declines to decide.
|
|
9
|
+
*/
|
|
10
|
+
import { Classifier, Decision, type DecisionAnswer, type NoulCriteria, type Question, type RecordedDecision } from "./classifier.js";
|
|
11
|
+
import type { Request } from "./types.js";
|
|
12
|
+
/** One named question of an ordered question list. */
|
|
13
|
+
export interface NamedQuestion {
|
|
14
|
+
name: string;
|
|
15
|
+
question: Question;
|
|
16
|
+
}
|
|
17
|
+
/** `noul` question with a name. Name the state field it judges in `instructions`. */
|
|
18
|
+
export declare function noul(name: string, instructions: string, criteria?: NoulCriteria): NamedQuestion;
|
|
19
|
+
/** `choice` question with a name; `options` maps option -> what picking it MEANS. */
|
|
20
|
+
export declare function choice(name: string, instructions: string, options: Record<string, string>): NamedQuestion;
|
|
21
|
+
/** `score` question with a name; `levels` is the ordered rubric. */
|
|
22
|
+
export declare function score(name: string, instructions: string, levels: string[]): NamedQuestion;
|
|
23
|
+
/** Ordered list -> the §8B questions map. A duplicate name is an error naming it. */
|
|
24
|
+
export declare function questionMap(questions: readonly NamedQuestion[]): Record<string, Question>;
|
|
25
|
+
/** `role` next to the data's fields; a non-object value goes under `data`. */
|
|
26
|
+
export declare function State(role: string, data: unknown): Record<string, unknown>;
|
|
27
|
+
/** Sugar: `{context, message, ...extra}`. */
|
|
28
|
+
export declare function context(ctx: string, message: string, extra?: Record<string, unknown>): Record<string, unknown>;
|
|
29
|
+
export type Band = "yes" | "no" | "uncertain";
|
|
30
|
+
export interface Bands {
|
|
31
|
+
low: number;
|
|
32
|
+
high: number;
|
|
33
|
+
}
|
|
34
|
+
/** Default cut-points; exclusive on the confident side. */
|
|
35
|
+
export declare const DEFAULT_BANDS: Readonly<Bands>;
|
|
36
|
+
/** One answer from `ask`: the §8B answer plus `band` (noul) or `sure` (choice/score). */
|
|
37
|
+
export type Answer = DecisionAnswer & {
|
|
38
|
+
/** noul only. */
|
|
39
|
+
band?: Band;
|
|
40
|
+
/** choice / score only. */
|
|
41
|
+
sure?: boolean;
|
|
42
|
+
/** The one number: noul probability, score value, or choice confidence. */
|
|
43
|
+
value(): number;
|
|
44
|
+
/** The picked option of a choice answer ("" otherwise). JS keeps §8B's `choice` string FIELD on a
|
|
45
|
+
* choice answer (a method of the same name would shadow it), so this is `pick()`. */
|
|
46
|
+
pick(): string;
|
|
47
|
+
};
|
|
48
|
+
export type Answers = Record<string, Answer>;
|
|
49
|
+
/** Band / sure every answer of a decision. */
|
|
50
|
+
export declare function answersOf(decision: Decision, bands?: Bands): Answers;
|
|
51
|
+
/** Evaluate once; answers by name, each with a band (noul) or `sure` (choice/score). */
|
|
52
|
+
export declare function ask(classifier: Classifier, state: unknown, questions: readonly NamedQuestion[], bands?: Bands): Promise<Answers>;
|
|
53
|
+
export interface Rule {
|
|
54
|
+
question: string;
|
|
55
|
+
below?: number;
|
|
56
|
+
at_least?: number;
|
|
57
|
+
is?: string;
|
|
58
|
+
action: string;
|
|
59
|
+
target?: string;
|
|
60
|
+
}
|
|
61
|
+
export interface GateOutcome {
|
|
62
|
+
action: string;
|
|
63
|
+
target: string;
|
|
64
|
+
escalated: boolean;
|
|
65
|
+
/** Present iff escalated: a §10 `input` Request. */
|
|
66
|
+
request?: Request;
|
|
67
|
+
answers: Answers;
|
|
68
|
+
}
|
|
69
|
+
export interface Policy {
|
|
70
|
+
rules: Rule[];
|
|
71
|
+
/** The no-rule-fired action; empty escalates with reason "no rule fired". */
|
|
72
|
+
default?: string;
|
|
73
|
+
bands?: Bands;
|
|
74
|
+
/** Skip a rule whose answer is uncertain instead of escalating on it. */
|
|
75
|
+
skipUncertain?: boolean;
|
|
76
|
+
}
|
|
77
|
+
/** Pure half of `gate`: first-match rules; an unsure or missing answer on rule i escalates and wins. */
|
|
78
|
+
export declare function applyGate(answers: Answers, rules: readonly Rule[]): GateOutcome;
|
|
79
|
+
/** Pure half of a `Policy`. */
|
|
80
|
+
export declare function applyPolicy(answers: Answers, policy: Policy): GateOutcome;
|
|
81
|
+
/** Ask, then apply rules; no rule fired -> empty action (use a `Policy` for a declared default). */
|
|
82
|
+
export declare function gate(classifier: Classifier, state: unknown, questions: readonly NamedQuestion[], rules: readonly Rule[], bands?: Bands): Promise<GateOutcome>;
|
|
83
|
+
/** Ask, then apply a `Policy` (rules, default, bands, skipUncertain). */
|
|
84
|
+
export declare function decide(classifier: Classifier, state: unknown, questions: readonly NamedQuestion[], policy: Policy): Promise<GateOutcome>;
|
|
85
|
+
/** One-line `static` classifier from recorded decisions. */
|
|
86
|
+
export declare function staticClassifier(decisions: RecordedDecision[] | RecordedDecision, model?: string): Classifier;
|
|
87
|
+
/**
|
|
88
|
+
* Records live decisions keyed by a caller-given call name, and replays them with no network.
|
|
89
|
+
* `tape.classifier(name)` returns a Classifier: recording (live given) or replaying (no live).
|
|
90
|
+
*/
|
|
91
|
+
export declare class Tape {
|
|
92
|
+
private readonly live?;
|
|
93
|
+
readonly entries: Record<string, RecordedDecision>;
|
|
94
|
+
constructor(live?: Classifier | undefined, entries?: Record<string, RecordedDecision>);
|
|
95
|
+
/** A replay-only tape from saved entries. */
|
|
96
|
+
static replay(entries: Record<string, RecordedDecision>): Tape;
|
|
97
|
+
classifier(name: string): Classifier;
|
|
98
|
+
}
|