tnega 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +115 -189
- package/dist/agent.js +2907 -0
- package/dist/agent.js.map +7 -0
- package/dist/bin.js +32512 -4419
- package/dist/bin.js.map +4 -4
- package/dist/cli-runtime.js +33404 -0
- package/dist/cli-runtime.js.map +7 -0
- package/dist/coding-agent.js +19088 -0
- package/dist/coding-agent.js.map +7 -0
- package/dist/core.js +1398 -0
- package/dist/core.js.map +7 -0
- package/dist/eval.js +23905 -0
- package/dist/eval.js.map +7 -0
- package/dist/events.js +1215 -0
- package/dist/events.js.map +7 -0
- package/dist/evolve.js +18303 -0
- package/dist/evolve.js.map +7 -0
- package/dist/index.js +33087 -4450
- package/dist/index.js.map +4 -4
- package/dist/llm.js +1016 -0
- package/dist/llm.js.map +7 -0
- package/dist/services.js +1142 -0
- package/dist/services.js.map +7 -0
- package/dist/session.js +1045 -0
- package/dist/session.js.map +7 -0
- package/dist/tools.js +2577 -0
- package/dist/tools.js.map +7 -0
- package/dist/types/packages/agent/src/definition.d.ts +17 -0
- package/dist/types/packages/agent/src/inbox-durable.d.ts +54 -0
- package/dist/types/packages/agent/src/index.d.ts +7 -0
- package/dist/types/packages/agent/src/live.d.ts +115 -0
- package/dist/types/packages/agent/src/llm-service.d.ts +32 -0
- package/dist/types/packages/agent/src/prompt.d.ts +55 -0
- package/dist/types/packages/agent/src/service.d.ts +68 -0
- package/dist/types/packages/agent/src/types.d.ts +215 -0
- package/dist/types/packages/benchmark/src/bigcodebench.d.ts +6 -0
- package/dist/types/packages/benchmark/src/humaneval.d.ts +11 -0
- package/dist/types/packages/benchmark/src/index.d.ts +9 -0
- package/dist/types/packages/benchmark/src/manifest.d.ts +10 -0
- package/dist/types/packages/benchmark/src/mbpp.d.ts +10 -0
- package/dist/types/packages/benchmark/src/parquet.d.ts +2 -0
- package/dist/types/packages/benchmark/src/stdlib.d.ts +3 -0
- package/dist/types/packages/benchmark/src/swebench.d.ts +2 -0
- package/dist/types/packages/benchmark/src/types.d.ts +58 -0
- package/dist/types/packages/cli/src/bin.d.ts +2 -0
- package/dist/types/packages/cli/src/commands.d.ts +146 -0
- package/dist/types/packages/cli/src/config.d.ts +28 -0
- package/dist/types/packages/cli/src/index.d.ts +9 -0
- package/dist/types/packages/cli/src/profile-file.d.ts +26 -0
- package/dist/types/packages/cli/src/profile.d.ts +9 -0
- package/dist/types/packages/cli/src/server.d.ts +14 -0
- package/dist/types/packages/cli/src/store.d.ts +60 -0
- package/dist/types/packages/cli/src/yaml.d.ts +1 -0
- package/dist/types/packages/coding-agent/src/codingAgent.d.ts +25 -0
- package/dist/types/packages/coding-agent/src/index.d.ts +6 -0
- package/dist/types/packages/coding-agent/src/mcp.d.ts +24 -0
- package/dist/types/packages/coding-agent/src/plan.d.ts +13 -0
- package/dist/types/packages/coding-agent/src/skills.d.ts +12 -0
- package/dist/types/packages/coding-agent/src/slash.d.ts +28 -0
- package/dist/types/packages/coding-agent/src/types.d.ts +41 -0
- package/dist/types/packages/core/src/context.d.ts +28 -0
- package/dist/types/packages/core/src/events.d.ts +57 -0
- package/dist/types/packages/core/src/fiber.d.ts +79 -0
- package/dist/types/packages/core/src/index.d.ts +10 -0
- package/dist/types/packages/core/src/invariant.d.ts +45 -0
- package/dist/types/packages/core/src/logger.d.ts +48 -0
- package/dist/types/packages/core/src/reflect.d.ts +42 -0
- package/dist/types/packages/core/src/registry.d.ts +54 -0
- package/dist/types/packages/core/src/service.d.ts +19 -0
- package/dist/types/packages/core/src/utils.d.ts +54 -0
- package/dist/types/packages/eval/src/codingRuntime.d.ts +18 -0
- package/dist/types/packages/eval/src/index.d.ts +5 -0
- package/dist/types/packages/eval/src/policy.d.ts +8 -0
- package/dist/types/packages/eval/src/runner.d.ts +31 -0
- package/dist/types/packages/eval/src/service.d.ts +23 -0
- package/dist/types/packages/eval/src/strategies.d.ts +58 -0
- package/dist/types/packages/eval/src/trace.d.ts +7 -0
- package/dist/types/packages/eval/src/types.d.ts +281 -0
- package/dist/types/packages/eval/src/utils.d.ts +5 -0
- package/dist/types/packages/eval/src/workspace.d.ts +17 -0
- package/dist/types/packages/evolve/src/diagnose.d.ts +9 -0
- package/dist/types/packages/evolve/src/index.d.ts +7 -0
- package/dist/types/packages/evolve/src/llm.d.ts +39 -0
- package/dist/types/packages/evolve/src/log.d.ts +26 -0
- package/dist/types/packages/evolve/src/select.d.ts +3 -0
- package/dist/types/packages/evolve/src/service.d.ts +29 -0
- package/dist/types/packages/evolve/src/types.d.ts +176 -0
- package/dist/types/packages/llm/src/anthropic-messages.d.ts +4 -0
- package/dist/types/packages/llm/src/errors.d.ts +6 -0
- package/dist/types/packages/llm/src/index.d.ts +7 -0
- package/dist/types/packages/llm/src/models.d.ts +29 -0
- package/dist/types/packages/llm/src/openai.d.ts +4 -0
- package/dist/types/packages/llm/src/provider.d.ts +3 -0
- package/dist/types/packages/llm/src/shared.d.ts +15 -0
- package/dist/types/packages/llm/src/types.d.ts +13 -0
- package/dist/types/packages/session/src/index.d.ts +500 -0
- package/dist/types/packages/session/src/invariant.d.ts +33 -0
- package/dist/types/packages/tools/src/builtins.d.ts +25 -0
- package/dist/types/packages/tools/src/calc.d.ts +4 -0
- package/dist/types/packages/tools/src/execution.d.ts +31 -0
- package/dist/types/packages/tools/src/index.d.ts +107 -0
- package/dist/types/packages/tools/src/path.d.ts +4 -0
- package/dist/types/packages/tools/src/policy.d.ts +14 -0
- package/dist/types/src/events.d.ts +1 -0
- package/dist/types/src/index.d.ts +15 -0
- package/dist/types/src/services.d.ts +4 -0
- package/dist/web/assets/index-45pokiBH.js +270 -0
- package/dist/web/assets/index-DCLL69Kz.css +1 -0
- package/dist/web/index.html +13 -0
- package/package.json +92 -35
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { Context } from "../../core/src/index.js";
|
|
2
|
+
import type { LLMAdapter } from './types.js';
|
|
3
|
+
export interface LlmRouteCapacity {
|
|
4
|
+
provider: string;
|
|
5
|
+
model: string;
|
|
6
|
+
contextWindow?: number;
|
|
7
|
+
}
|
|
8
|
+
export declare class LlmService {
|
|
9
|
+
private _adapters;
|
|
10
|
+
private _current;
|
|
11
|
+
private _defaultName;
|
|
12
|
+
register(name: string, adapter: LLMAdapter): () => void;
|
|
13
|
+
setCurrent(name: string): LLMAdapter;
|
|
14
|
+
current(): LLMAdapter | undefined;
|
|
15
|
+
list(): readonly {
|
|
16
|
+
name: string;
|
|
17
|
+
adapter: LLMAdapter;
|
|
18
|
+
}[];
|
|
19
|
+
/**
|
|
20
|
+
* Report provider/model/contextWindow for the current route. The adapter
|
|
21
|
+
* may be registered with an optional capacity provider by callers that own
|
|
22
|
+
* the model catalog (for example the CLI runtime).
|
|
23
|
+
*/
|
|
24
|
+
routeCapacity(): LlmRouteCapacity | undefined;
|
|
25
|
+
private _capacityFor;
|
|
26
|
+
/** Install a catalog-backed contextWindow resolver (optional). */
|
|
27
|
+
setCapacityResolver(resolver: (model: string | undefined) => number | undefined): void;
|
|
28
|
+
}
|
|
29
|
+
export declare const llmService: {
|
|
30
|
+
name: string;
|
|
31
|
+
apply: (ctx: Context) => () => void;
|
|
32
|
+
};
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { Context } from "../../core/src/index.js";
|
|
2
|
+
export interface PromptSection {
|
|
3
|
+
name: string;
|
|
4
|
+
order?: number;
|
|
5
|
+
content: string;
|
|
6
|
+
}
|
|
7
|
+
export interface SystemPromptAssemblyOptions {
|
|
8
|
+
scope?: unknown;
|
|
9
|
+
[key: string]: unknown;
|
|
10
|
+
}
|
|
11
|
+
export interface PromptAssembly {
|
|
12
|
+
sections: readonly PromptSection[];
|
|
13
|
+
text: string;
|
|
14
|
+
context?: SystemPromptAssemblyOptions;
|
|
15
|
+
variables?: Record<string, string>;
|
|
16
|
+
tools?: readonly {
|
|
17
|
+
name: string;
|
|
18
|
+
description: string;
|
|
19
|
+
parameters?: Record<string, unknown>;
|
|
20
|
+
}[];
|
|
21
|
+
}
|
|
22
|
+
export type ToolSchemaSnapshot = {
|
|
23
|
+
name: string;
|
|
24
|
+
description: string;
|
|
25
|
+
parameters?: Record<string, unknown>;
|
|
26
|
+
};
|
|
27
|
+
export type ToolSchemaProvider = (context: SystemPromptAssemblyOptions) => readonly ToolSchemaSnapshot[] | Promise<readonly ToolSchemaSnapshot[]>;
|
|
28
|
+
export type PromptVariableProvider = (context: SystemPromptAssemblyOptions) => string | undefined | Promise<string | undefined>;
|
|
29
|
+
export interface PromptContextProvider {
|
|
30
|
+
name: string;
|
|
31
|
+
order?: number;
|
|
32
|
+
content: PromptVariableProvider;
|
|
33
|
+
}
|
|
34
|
+
export declare class SystemPromptService {
|
|
35
|
+
private _sections;
|
|
36
|
+
private _toolProviders;
|
|
37
|
+
private _variables;
|
|
38
|
+
private _contexts;
|
|
39
|
+
private _ctx;
|
|
40
|
+
constructor(ctx?: Context);
|
|
41
|
+
registerSection(section: PromptSection): () => void;
|
|
42
|
+
sections(): readonly PromptSection[];
|
|
43
|
+
registerTools(provider: ToolSchemaProvider): () => void;
|
|
44
|
+
toolSchemas(options?: SystemPromptAssemblyOptions): Promise<ToolSchemaSnapshot[]>;
|
|
45
|
+
registerVariable(name: string, provider: PromptVariableProvider): () => void;
|
|
46
|
+
registerContext(context: PromptContextProvider): () => void;
|
|
47
|
+
assemble(options?: SystemPromptAssemblyOptions, ctx?: Context): Promise<PromptAssembly>;
|
|
48
|
+
private _resolveVariables;
|
|
49
|
+
private _resolveContexts;
|
|
50
|
+
private _emitChange;
|
|
51
|
+
}
|
|
52
|
+
export declare const systemPrompt: {
|
|
53
|
+
name: string;
|
|
54
|
+
apply: (ctx: Context) => () => void;
|
|
55
|
+
};
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import type { Context } from "../../core/src/index.js";
|
|
2
|
+
import { type SessionLog } from "../../session/src/index.js";
|
|
3
|
+
import type { AgentContextBudget, AgentHooks, AgentInput, AgentRunOptions, AgentRunResult, AgentStreamEvent, LLMAdapter } from './types.js';
|
|
4
|
+
export declare class AgentError extends Error {
|
|
5
|
+
name: string;
|
|
6
|
+
}
|
|
7
|
+
export declare class AgentInbox {
|
|
8
|
+
private _queue;
|
|
9
|
+
private _steerQueue;
|
|
10
|
+
private _injected;
|
|
11
|
+
get size(): number;
|
|
12
|
+
get steerSize(): number;
|
|
13
|
+
push(input: AgentInput): AgentInput;
|
|
14
|
+
followup(input: AgentInput): AgentInput;
|
|
15
|
+
steer(input: AgentInput): AgentInput;
|
|
16
|
+
send(input: AgentInput, options?: {
|
|
17
|
+
mode?: 'followup' | 'steer';
|
|
18
|
+
wake?: boolean;
|
|
19
|
+
}): AgentInput;
|
|
20
|
+
claim(): AgentInput | undefined;
|
|
21
|
+
peek(): AgentInput | undefined;
|
|
22
|
+
inject(key: string, value: unknown): void;
|
|
23
|
+
injected(): ReadonlyMap<string, unknown>;
|
|
24
|
+
}
|
|
25
|
+
export interface AgentConfig {
|
|
26
|
+
llm?: LLMAdapter;
|
|
27
|
+
/** Bind the loop to a specific session instead of the ctx-provided singleton. */
|
|
28
|
+
session?: SessionLog;
|
|
29
|
+
/** Fail fast when the actual request would not be reconstructable from the log. */
|
|
30
|
+
assertReplayable?: boolean;
|
|
31
|
+
maxTurns?: number;
|
|
32
|
+
maxSteps?: number;
|
|
33
|
+
inbox?: AgentInbox;
|
|
34
|
+
hooks?: AgentHooks;
|
|
35
|
+
contextBudget?: AgentContextBudget;
|
|
36
|
+
}
|
|
37
|
+
export declare class AgentService {
|
|
38
|
+
private ctx;
|
|
39
|
+
private config;
|
|
40
|
+
readonly inbox: AgentInbox;
|
|
41
|
+
private _seriesStarted;
|
|
42
|
+
private _persistedRequest;
|
|
43
|
+
constructor(ctx: Context, config?: AgentConfig);
|
|
44
|
+
/** Call when a request begins an explicit new model-message series. */
|
|
45
|
+
startSeries(): void;
|
|
46
|
+
/** Clear any pending series marker after a completed run. */
|
|
47
|
+
endSeriesBoundary(): void;
|
|
48
|
+
/** True when this service instance has already persisted a model-visible step. */
|
|
49
|
+
get hasPersistedRequest(): boolean;
|
|
50
|
+
run(input?: AgentInput, options?: AgentRunOptions): Promise<AgentRunResult>;
|
|
51
|
+
runStream(input?: AgentInput, options?: AgentRunOptions): AsyncGenerator<AgentStreamEvent, AgentRunResult, void>;
|
|
52
|
+
private _stream;
|
|
53
|
+
private _enforceContextBudget;
|
|
54
|
+
private _persistStepInput;
|
|
55
|
+
private _assertReplayable;
|
|
56
|
+
private _initialMessages;
|
|
57
|
+
private _extendMessages;
|
|
58
|
+
private _session;
|
|
59
|
+
private _tools;
|
|
60
|
+
private _resolveAvailableTools;
|
|
61
|
+
private _llm;
|
|
62
|
+
}
|
|
63
|
+
export declare const agent: {
|
|
64
|
+
name: string;
|
|
65
|
+
inject: string[];
|
|
66
|
+
apply: (ctx: Context, config?: AgentConfig) => () => void;
|
|
67
|
+
};
|
|
68
|
+
export declare const name = "index.js";
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
import type { CancelCause, ContextUsage, ModelMessage } from "../../session/src/index.js";
|
|
2
|
+
import type { ToolDefinition, ToolResult } from "../../tools/src/index.js";
|
|
3
|
+
export type AgentCancelCause = CancelCause;
|
|
4
|
+
export type AgentFinishReason = 'stop' | 'tool_calls' | 'length' | 'max_turns' | 'max_steps' | 'error' | 'cancelled';
|
|
5
|
+
export interface LLMToolCall {
|
|
6
|
+
id: string;
|
|
7
|
+
name: string;
|
|
8
|
+
arguments: unknown;
|
|
9
|
+
}
|
|
10
|
+
export interface LLMCompletion {
|
|
11
|
+
content?: string;
|
|
12
|
+
toolCalls?: LLMToolCall[];
|
|
13
|
+
finishReason: AgentFinishReason;
|
|
14
|
+
}
|
|
15
|
+
export interface CompleteOptions {
|
|
16
|
+
maxSteps?: number;
|
|
17
|
+
signal?: AbortSignal;
|
|
18
|
+
provider?: string;
|
|
19
|
+
model?: string;
|
|
20
|
+
temperature?: number;
|
|
21
|
+
}
|
|
22
|
+
export interface LLMMessageStartEvent {
|
|
23
|
+
type: 'message_start';
|
|
24
|
+
id: string;
|
|
25
|
+
model?: string;
|
|
26
|
+
}
|
|
27
|
+
export interface LLMMessageDeltaEvent {
|
|
28
|
+
type: 'message_delta';
|
|
29
|
+
id: string;
|
|
30
|
+
delta: string;
|
|
31
|
+
}
|
|
32
|
+
export interface LLMToolCallStartEvent {
|
|
33
|
+
type: 'toolcall_start';
|
|
34
|
+
id: string;
|
|
35
|
+
index: number;
|
|
36
|
+
name: string;
|
|
37
|
+
}
|
|
38
|
+
export interface LLMToolCallEndEvent {
|
|
39
|
+
type: 'toolcall_end';
|
|
40
|
+
id: string;
|
|
41
|
+
index: number;
|
|
42
|
+
name: string;
|
|
43
|
+
arguments: unknown;
|
|
44
|
+
}
|
|
45
|
+
export interface LLMMessageStopEvent {
|
|
46
|
+
type: 'message_stop';
|
|
47
|
+
id: string;
|
|
48
|
+
finishReason: AgentFinishReason;
|
|
49
|
+
}
|
|
50
|
+
export type LLMStreamEvent = LLMMessageStartEvent | LLMMessageDeltaEvent | LLMToolCallStartEvent | LLMToolCallEndEvent | LLMMessageStopEvent;
|
|
51
|
+
export interface LLMStreamRequestEvent {
|
|
52
|
+
index: number;
|
|
53
|
+
messages: ModelMessage[];
|
|
54
|
+
tools: readonly ToolDefinition[];
|
|
55
|
+
options: CompleteOptions;
|
|
56
|
+
}
|
|
57
|
+
export interface LLMAdapter {
|
|
58
|
+
complete(messages: readonly ModelMessage[], tools: readonly ToolDefinition[], options: CompleteOptions): Promise<LLMCompletion>;
|
|
59
|
+
stream?(messages: readonly ModelMessage[], tools: readonly ToolDefinition[], options: CompleteOptions): AsyncIterable<LLMStreamEvent>;
|
|
60
|
+
}
|
|
61
|
+
export interface AgentInput {
|
|
62
|
+
text?: string;
|
|
63
|
+
messages?: ModelMessage[];
|
|
64
|
+
context?: unknown;
|
|
65
|
+
}
|
|
66
|
+
export interface AgentStep {
|
|
67
|
+
index: number;
|
|
68
|
+
input: readonly ModelMessage[];
|
|
69
|
+
completion: LLMCompletion;
|
|
70
|
+
toolResults: readonly ToolResult[];
|
|
71
|
+
}
|
|
72
|
+
export interface AgentRunResult {
|
|
73
|
+
input: AgentInput;
|
|
74
|
+
output: string;
|
|
75
|
+
finishReason: AgentFinishReason;
|
|
76
|
+
/** Durable turn number this run executed, when a turn was opened. */
|
|
77
|
+
turn?: number;
|
|
78
|
+
steps: readonly AgentStep[];
|
|
79
|
+
messages: readonly ModelMessage[];
|
|
80
|
+
}
|
|
81
|
+
export interface AgentRunOptions {
|
|
82
|
+
maxTurns?: number;
|
|
83
|
+
maxSteps?: number;
|
|
84
|
+
signal?: AbortSignal;
|
|
85
|
+
contextBudget?: AgentContextBudget;
|
|
86
|
+
}
|
|
87
|
+
export interface AgentContextBudget {
|
|
88
|
+
limit?: number;
|
|
89
|
+
compactRatio?: number;
|
|
90
|
+
keepTokens?: number;
|
|
91
|
+
summarize?: AgentContextSummarizer;
|
|
92
|
+
}
|
|
93
|
+
export type AgentContextSummarizer = (messages: readonly ModelMessage[], usage: ContextUsage) => readonly ModelMessage[] | Promise<readonly ModelMessage[]>;
|
|
94
|
+
export interface AgentContextCompactEvent {
|
|
95
|
+
type: 'agent/context-compact';
|
|
96
|
+
messagesBefore: number;
|
|
97
|
+
tokensBefore: number;
|
|
98
|
+
limit: number;
|
|
99
|
+
keepTokens: number;
|
|
100
|
+
messagesAfter: number;
|
|
101
|
+
}
|
|
102
|
+
export interface AgentHooks {
|
|
103
|
+
beforeRun?: (input: AgentInput, options: AgentRunOptions) => void | Promise<void>;
|
|
104
|
+
afterRun?: (result: AgentRunResult, options: AgentRunOptions) => void | Promise<void>;
|
|
105
|
+
}
|
|
106
|
+
export interface AgentStartEvent {
|
|
107
|
+
input: AgentInput;
|
|
108
|
+
options: AgentRunOptions;
|
|
109
|
+
injected: ReadonlyMap<string, unknown>;
|
|
110
|
+
}
|
|
111
|
+
export interface AgentTurnStartEvent {
|
|
112
|
+
input: AgentInput;
|
|
113
|
+
messages: readonly ModelMessage[];
|
|
114
|
+
injected: ReadonlyMap<string, unknown>;
|
|
115
|
+
}
|
|
116
|
+
export interface AgentStepEvent {
|
|
117
|
+
index: number;
|
|
118
|
+
turn?: number;
|
|
119
|
+
step?: number;
|
|
120
|
+
input: readonly ModelMessage[];
|
|
121
|
+
}
|
|
122
|
+
export interface AgentPreStepEvent {
|
|
123
|
+
index: number;
|
|
124
|
+
/** Durable turn that will own the proposed step. */
|
|
125
|
+
turn?: number;
|
|
126
|
+
/** Durable step proposed by the loop. */
|
|
127
|
+
step?: number;
|
|
128
|
+
/** Cancellation signal for the current turn. */
|
|
129
|
+
signal?: AbortSignal;
|
|
130
|
+
messages: ModelMessage[];
|
|
131
|
+
/** Begin a distinct model-message series before this step's admitted messages. */
|
|
132
|
+
startsRequestSeries?: boolean;
|
|
133
|
+
}
|
|
134
|
+
export interface AgentRequestEvent {
|
|
135
|
+
index: number;
|
|
136
|
+
/** Read-only snapshot of the model-visible messages for this request. */
|
|
137
|
+
readonly messages: readonly ModelMessage[];
|
|
138
|
+
tools: readonly ToolDefinition[];
|
|
139
|
+
options: CompleteOptions;
|
|
140
|
+
}
|
|
141
|
+
export interface AgentRequestErrorEvent {
|
|
142
|
+
index: number;
|
|
143
|
+
turn: number;
|
|
144
|
+
step: number;
|
|
145
|
+
messages: readonly ModelMessage[];
|
|
146
|
+
tools: readonly ToolDefinition[];
|
|
147
|
+
options: CompleteOptions;
|
|
148
|
+
attempt: number;
|
|
149
|
+
error: unknown;
|
|
150
|
+
/** Normalized durable failure fact surfaced to recovery listeners. */
|
|
151
|
+
failure: {
|
|
152
|
+
name: string;
|
|
153
|
+
message: string;
|
|
154
|
+
stack?: string;
|
|
155
|
+
};
|
|
156
|
+
provider?: string;
|
|
157
|
+
model?: string;
|
|
158
|
+
signal?: AbortSignal;
|
|
159
|
+
retryPolicy?: {
|
|
160
|
+
maxRetries: number;
|
|
161
|
+
retryDelayMs: number;
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
export type AgentRequestRetryDecision = {
|
|
165
|
+
kind: 'retry';
|
|
166
|
+
delayMs?: number;
|
|
167
|
+
} | undefined;
|
|
168
|
+
export interface AgentTurnStoppingEvent {
|
|
169
|
+
index: number;
|
|
170
|
+
turn?: number;
|
|
171
|
+
steps: readonly AgentStep[];
|
|
172
|
+
finishReason: AgentFinishReason;
|
|
173
|
+
}
|
|
174
|
+
export interface AgentToolCallEvent {
|
|
175
|
+
index: number;
|
|
176
|
+
turn?: number;
|
|
177
|
+
step?: number;
|
|
178
|
+
call: LLMToolCall;
|
|
179
|
+
}
|
|
180
|
+
export interface AgentToolResultEvent {
|
|
181
|
+
index: number;
|
|
182
|
+
turn?: number;
|
|
183
|
+
step?: number;
|
|
184
|
+
call: LLMToolCall;
|
|
185
|
+
result: ToolResult;
|
|
186
|
+
}
|
|
187
|
+
export interface AgentTurnEndEvent {
|
|
188
|
+
input: AgentInput;
|
|
189
|
+
steps: readonly AgentStep[];
|
|
190
|
+
messages: readonly ModelMessage[];
|
|
191
|
+
output: string;
|
|
192
|
+
finishReason: AgentFinishReason;
|
|
193
|
+
}
|
|
194
|
+
export type AgentEndEvent = AgentTurnEndEvent;
|
|
195
|
+
export interface AgentToolStartEvent {
|
|
196
|
+
type: 'tool/start';
|
|
197
|
+
index: number;
|
|
198
|
+
turn?: number;
|
|
199
|
+
step?: number;
|
|
200
|
+
call: LLMToolCall;
|
|
201
|
+
}
|
|
202
|
+
export interface AgentToolEndEvent {
|
|
203
|
+
type: 'tool/end';
|
|
204
|
+
index: number;
|
|
205
|
+
turn?: number;
|
|
206
|
+
step?: number;
|
|
207
|
+
call: LLMToolCall;
|
|
208
|
+
result: ToolResult;
|
|
209
|
+
}
|
|
210
|
+
export interface AgentRunEndEvent {
|
|
211
|
+
type: 'run/end';
|
|
212
|
+
run: AgentRunResult;
|
|
213
|
+
}
|
|
214
|
+
export type AgentStreamEvent = LLMStreamEvent | AgentToolStartEvent | AgentToolEndEvent | AgentRunEndEvent;
|
|
215
|
+
export type AgentLoop = (input?: AgentInput, options?: AgentRunOptions) => Promise<AgentRunResult>;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { BenchmarkImportOptions, ImportedBenchmark } from './types.js';
|
|
2
|
+
export declare function isBigCodeBenchEligible(row: {
|
|
3
|
+
libs?: string;
|
|
4
|
+
test?: string;
|
|
5
|
+
}): boolean;
|
|
6
|
+
export declare function importBigCodeBench(options: BenchmarkImportOptions): Promise<ImportedBenchmark>;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { BenchmarkImportOptions, ImportedBenchmark } from './types.js';
|
|
2
|
+
export interface HumanEvalFixture {
|
|
3
|
+
solution: string;
|
|
4
|
+
test: string;
|
|
5
|
+
}
|
|
6
|
+
export declare function buildHumanEvalFixture(row: {
|
|
7
|
+
prompt: string;
|
|
8
|
+
test: string;
|
|
9
|
+
entry_point: string;
|
|
10
|
+
}): HumanEvalFixture;
|
|
11
|
+
export declare function importHumanEval(options: BenchmarkImportOptions): Promise<ImportedBenchmark>;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export * from './bigcodebench.js';
|
|
2
|
+
export * from './humaneval.js';
|
|
3
|
+
export * from './manifest.js';
|
|
4
|
+
export * from './mbpp.js';
|
|
5
|
+
export * from './parquet.js';
|
|
6
|
+
export * from './stdlib.js';
|
|
7
|
+
export * from './swebench.js';
|
|
8
|
+
export * from './types.js';
|
|
9
|
+
export declare const name = "index.js";
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { BenchmarkManifest } from './types.js';
|
|
2
|
+
export declare function saveManifest(dir: string, manifest: BenchmarkManifest): Promise<string>;
|
|
3
|
+
export declare function loadManifest(file: string): Promise<BenchmarkManifest>;
|
|
4
|
+
export declare function manifestFile(dir: string): string;
|
|
5
|
+
export declare function tasksFile(dir: string): string;
|
|
6
|
+
export declare function writeTasksFile(dir: string, tasks: Array<{
|
|
7
|
+
task: import("../../eval/src/index.js").Task;
|
|
8
|
+
meta: import('./types.js').BenchmarkTaskMeta;
|
|
9
|
+
}>): Promise<string>;
|
|
10
|
+
export declare function ensureDirname(file: string): Promise<string>;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { BenchmarkImportOptions, ImportedBenchmark } from './types.js';
|
|
2
|
+
export interface MbppFixture {
|
|
3
|
+
solution: string;
|
|
4
|
+
test: string;
|
|
5
|
+
}
|
|
6
|
+
export declare function buildMbppFixture(row: {
|
|
7
|
+
test_list: readonly string[];
|
|
8
|
+
test_setup_code?: string;
|
|
9
|
+
}): MbppFixture;
|
|
10
|
+
export declare function importMbpp(options: BenchmarkImportOptions): Promise<ImportedBenchmark>;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import type { Task } from "../../eval/src/index.js";
|
|
2
|
+
export type BenchmarkSource = 'bigcodebench' | 'swebench' | 'humaneval' | 'mbpp';
|
|
3
|
+
export interface BenchmarkImportOptions {
|
|
4
|
+
/** Destination directory for the materialized benchmark. */
|
|
5
|
+
outDir: string;
|
|
6
|
+
/** Number of tasks to import, default source-specific. */
|
|
7
|
+
subset?: number;
|
|
8
|
+
/** Optional comma-separated repo filter for swebench, e.g. "psf/requests,sympy/sympy". */
|
|
9
|
+
repo?: string;
|
|
10
|
+
/** Optional comma-separated instance filter for swebench, e.g. "psf__requests-6028". */
|
|
11
|
+
ids?: string;
|
|
12
|
+
/** Dataset version, default latest known. */
|
|
13
|
+
version?: string;
|
|
14
|
+
/** HuggingFace mirror base URL, default https://hf-mirror.com. */
|
|
15
|
+
mirror?: string;
|
|
16
|
+
/** Force re-download of the raw parquet file. */
|
|
17
|
+
force?: boolean;
|
|
18
|
+
}
|
|
19
|
+
export interface BenchmarkTaskMeta {
|
|
20
|
+
dataset: string;
|
|
21
|
+
instanceId: string;
|
|
22
|
+
version: string;
|
|
23
|
+
entryPoint?: string;
|
|
24
|
+
libs?: string[];
|
|
25
|
+
repo?: string;
|
|
26
|
+
baseCommit?: string;
|
|
27
|
+
failToPass?: string[];
|
|
28
|
+
passToPass?: string[];
|
|
29
|
+
testCount?: number;
|
|
30
|
+
}
|
|
31
|
+
export interface BenchmarkTask {
|
|
32
|
+
task: Task;
|
|
33
|
+
meta: BenchmarkTaskMeta;
|
|
34
|
+
/** Directory containing the fixture workspace. */
|
|
35
|
+
fixtureDir: string;
|
|
36
|
+
}
|
|
37
|
+
export interface BenchmarkManifest {
|
|
38
|
+
source: BenchmarkSource;
|
|
39
|
+
version: string;
|
|
40
|
+
importedAt: number;
|
|
41
|
+
total: number;
|
|
42
|
+
tasks: Array<{
|
|
43
|
+
id: string;
|
|
44
|
+
fixtureDir: string;
|
|
45
|
+
taskFile: string;
|
|
46
|
+
dataset: BenchmarkSource;
|
|
47
|
+
}>;
|
|
48
|
+
sources?: Record<string, {
|
|
49
|
+
version: string;
|
|
50
|
+
total: number;
|
|
51
|
+
}>;
|
|
52
|
+
}
|
|
53
|
+
export interface ImportedBenchmark {
|
|
54
|
+
manifest: BenchmarkManifest;
|
|
55
|
+
tasksFile: string;
|
|
56
|
+
manifestFile: string;
|
|
57
|
+
dir: string;
|
|
58
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { Context, type Plugin } from "../../core/src/index.js";
|
|
2
|
+
import type { AgentProfile } from './profile.js';
|
|
3
|
+
import { type AgentContextBudget, type AgentDefinition, type AgentInbox, type AgentRunResult, type LLMAdapter } from "../../agent/src/index.js";
|
|
4
|
+
import { type BuiltinToolsConfig, type ToolPolicy } from "../../tools/src/index.js";
|
|
5
|
+
import { type CompareResult, type EvalRun, type RunBudget, type Task } from "../../eval/src/index.js";
|
|
6
|
+
import { type ImportedBenchmark } from "../../benchmark/src/index.js";
|
|
7
|
+
import { type EvolveStepResult, type SelectionPolicy } from "../../evolve/src/index.js";
|
|
8
|
+
export interface TasksFile {
|
|
9
|
+
tasks: Task[];
|
|
10
|
+
candidates?: Record<string, unknown>;
|
|
11
|
+
defaultCandidate?: string;
|
|
12
|
+
strategyNames?: string[];
|
|
13
|
+
outputDir?: string;
|
|
14
|
+
evolve?: EvolveFileConfig;
|
|
15
|
+
}
|
|
16
|
+
export interface EvolveFileConfig {
|
|
17
|
+
maxIterations?: number;
|
|
18
|
+
maxRuns?: number;
|
|
19
|
+
baseSystem?: string;
|
|
20
|
+
policy?: SelectionPolicy;
|
|
21
|
+
budget?: RunBudget;
|
|
22
|
+
}
|
|
23
|
+
export interface RunCommandOptions {
|
|
24
|
+
tasksFile: string;
|
|
25
|
+
candidateName?: string;
|
|
26
|
+
taskFilter?: string;
|
|
27
|
+
cache?: boolean;
|
|
28
|
+
budget?: RunBudget;
|
|
29
|
+
outputFile?: string;
|
|
30
|
+
cwd?: string;
|
|
31
|
+
model?: string;
|
|
32
|
+
baseUrl?: string;
|
|
33
|
+
maxTokens?: number;
|
|
34
|
+
temperature?: number;
|
|
35
|
+
maxTurns?: number;
|
|
36
|
+
maxSteps?: number;
|
|
37
|
+
timeoutMs?: number;
|
|
38
|
+
maxRetries?: number;
|
|
39
|
+
retryDelayMs?: number;
|
|
40
|
+
}
|
|
41
|
+
export interface CompareCommandOptions {
|
|
42
|
+
base: string;
|
|
43
|
+
head: string;
|
|
44
|
+
outputDir?: string;
|
|
45
|
+
cwd?: string;
|
|
46
|
+
}
|
|
47
|
+
export interface RunAgentCommandOptions {
|
|
48
|
+
prompt: string;
|
|
49
|
+
cwd?: string;
|
|
50
|
+
sessionFile?: string;
|
|
51
|
+
configFile?: string;
|
|
52
|
+
profile?: string;
|
|
53
|
+
model?: string;
|
|
54
|
+
baseUrl?: string;
|
|
55
|
+
maxTokens?: number;
|
|
56
|
+
temperature?: number;
|
|
57
|
+
maxTurns?: number;
|
|
58
|
+
maxSteps?: number;
|
|
59
|
+
allowNetwork?: boolean;
|
|
60
|
+
allowShell?: boolean;
|
|
61
|
+
timeoutMs?: number;
|
|
62
|
+
maxRetries?: number;
|
|
63
|
+
retryDelayMs?: number;
|
|
64
|
+
}
|
|
65
|
+
export interface RunAgentCommandResult {
|
|
66
|
+
run: AgentRunResult;
|
|
67
|
+
sessionFile: string;
|
|
68
|
+
}
|
|
69
|
+
export interface RunEvolveCommandOptions {
|
|
70
|
+
tasksFile: string;
|
|
71
|
+
cwd?: string;
|
|
72
|
+
outputDir?: string;
|
|
73
|
+
configFile?: string;
|
|
74
|
+
maxIterations?: number;
|
|
75
|
+
maxRuns?: number;
|
|
76
|
+
baseSystem?: string;
|
|
77
|
+
model?: string;
|
|
78
|
+
baseUrl?: string;
|
|
79
|
+
maxTokens?: number;
|
|
80
|
+
temperature?: number;
|
|
81
|
+
maxTurns?: number;
|
|
82
|
+
maxSteps?: number;
|
|
83
|
+
budget?: RunBudget;
|
|
84
|
+
cache?: boolean;
|
|
85
|
+
policy?: SelectionPolicy;
|
|
86
|
+
timeoutMs?: number;
|
|
87
|
+
maxRetries?: number;
|
|
88
|
+
retryDelayMs?: number;
|
|
89
|
+
}
|
|
90
|
+
export interface RunEvolveCommandResult {
|
|
91
|
+
prime: EvolveStepResult;
|
|
92
|
+
result: EvolveStepResult;
|
|
93
|
+
logFile: string;
|
|
94
|
+
runDir: string;
|
|
95
|
+
}
|
|
96
|
+
export interface ImportBenchmarkCommandOptions {
|
|
97
|
+
source: string;
|
|
98
|
+
outDir?: string;
|
|
99
|
+
subset?: number;
|
|
100
|
+
repo?: string;
|
|
101
|
+
ids?: string;
|
|
102
|
+
version?: string;
|
|
103
|
+
mirror?: string;
|
|
104
|
+
force?: boolean;
|
|
105
|
+
}
|
|
106
|
+
export interface LlmEnvConfig {
|
|
107
|
+
apiKey?: string;
|
|
108
|
+
baseUrl?: string;
|
|
109
|
+
model?: string;
|
|
110
|
+
}
|
|
111
|
+
export { resolveLlmEnv } from './config.js';
|
|
112
|
+
export interface AgentRuntimeOptions {
|
|
113
|
+
cwd: string;
|
|
114
|
+
sessionFile: string;
|
|
115
|
+
profile?: AgentProfile;
|
|
116
|
+
llm?: LLMAdapter;
|
|
117
|
+
inbox?: AgentInbox;
|
|
118
|
+
allowNetwork?: boolean;
|
|
119
|
+
allowShell?: boolean;
|
|
120
|
+
maxTurns?: number;
|
|
121
|
+
maxSteps?: number;
|
|
122
|
+
contextBudget?: AgentContextBudget;
|
|
123
|
+
agent?: AgentDefinition;
|
|
124
|
+
toolPolicy?: ToolPolicy;
|
|
125
|
+
builtinTools?: false | BuiltinToolsConfig;
|
|
126
|
+
plugins?: readonly Plugin[];
|
|
127
|
+
}
|
|
128
|
+
export interface AgentRuntime {
|
|
129
|
+
root: Context;
|
|
130
|
+
dispose: () => Promise<void>;
|
|
131
|
+
}
|
|
132
|
+
export declare class CliError extends Error {
|
|
133
|
+
name: string;
|
|
134
|
+
}
|
|
135
|
+
export declare function loadTasksFile(file: string): TasksFile;
|
|
136
|
+
export declare function runCommand(options: RunCommandOptions): Promise<EvalRun>;
|
|
137
|
+
export declare function formatRun(run: EvalRun): string;
|
|
138
|
+
export declare function formatCompare(result: CompareResult): string;
|
|
139
|
+
export declare function compareCommand(options: CompareCommandOptions): Promise<CompareResult>;
|
|
140
|
+
export declare function runAgentCommand(options: RunAgentCommandOptions): Promise<RunAgentCommandResult>;
|
|
141
|
+
export declare function createAgentRuntime(options: AgentRuntimeOptions): Promise<AgentRuntime>;
|
|
142
|
+
export declare function formatAgentRun(result: RunAgentCommandResult): string;
|
|
143
|
+
export declare function runEvolveCommand(options: RunEvolveCommandOptions): Promise<RunEvolveCommandResult>;
|
|
144
|
+
export declare function importBenchmarkCommand(options: ImportBenchmarkCommandOptions): Promise<ImportedBenchmark>;
|
|
145
|
+
export declare function formatBenchmarkImport(result: ImportedBenchmark): string;
|
|
146
|
+
export declare function formatEvolveResult(result: RunEvolveCommandResult): string;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
export interface LlmEnvConfig {
|
|
2
|
+
apiKey?: string;
|
|
3
|
+
baseUrl?: string;
|
|
4
|
+
model?: string;
|
|
5
|
+
}
|
|
6
|
+
export interface SystemConfig {
|
|
7
|
+
apiKey?: string;
|
|
8
|
+
baseUrl?: string;
|
|
9
|
+
model?: string;
|
|
10
|
+
protocol?: 'anthropic' | 'openai';
|
|
11
|
+
temperature?: number;
|
|
12
|
+
workspaces?: string[];
|
|
13
|
+
}
|
|
14
|
+
export interface EffectiveLlmConfig {
|
|
15
|
+
apiKeySet: boolean;
|
|
16
|
+
baseUrl: string;
|
|
17
|
+
model: string;
|
|
18
|
+
protocol?: 'anthropic' | 'openai';
|
|
19
|
+
temperature?: number;
|
|
20
|
+
}
|
|
21
|
+
export declare function systemConfigPath(): string;
|
|
22
|
+
export declare function readSystemConfig(file?: string): Promise<SystemConfig>;
|
|
23
|
+
export declare function writeSystemConfig(config: SystemConfig, file?: string): Promise<void>;
|
|
24
|
+
export declare function updateSystemConfig(patch: SystemConfig, file?: string): Promise<SystemConfig>;
|
|
25
|
+
export declare function effectiveLlmConfig(config: SystemConfig, env?: NodeJS.ProcessEnv): EffectiveLlmConfig;
|
|
26
|
+
export declare function toLlmEnvConfig(config: SystemConfig): LlmEnvConfig;
|
|
27
|
+
export declare function effectiveApiKey(config: SystemConfig, env?: NodeJS.ProcessEnv): string | undefined;
|
|
28
|
+
export declare function resolveLlmEnv(env?: NodeJS.ProcessEnv): LlmEnvConfig;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { CompareResult, EvalRun } from "../../eval/src/index.js";
|
|
2
|
+
export * from './profile.js';
|
|
3
|
+
export * from './profile-file.js';
|
|
4
|
+
export type { AgentRuntime, AgentRuntimeOptions, EvolveFileConfig, ImportBenchmarkCommandOptions, LlmEnvConfig, RunAgentCommandOptions, RunAgentCommandResult, RunEvolveCommandOptions, RunEvolveCommandResult, TasksFile, } from './commands.js';
|
|
5
|
+
export { CliError, compareCommand, createAgentRuntime, formatAgentRun, formatBenchmarkImport, formatCompare, formatEvolveResult, formatRun, importBenchmarkCommand, loadTasksFile, resolveLlmEnv, runAgentCommand, runCommand, runEvolveCommand, } from './commands.js';
|
|
6
|
+
export { parseYaml } from './yaml.js';
|
|
7
|
+
export type { CompareResult, EvalRun };
|
|
8
|
+
export declare function main(argv: readonly string[]): Promise<number>;
|
|
9
|
+
export declare const name = "index.js";
|