windmill-client 1.795.0 → 1.796.0

package/dist/types.gen.d.ts

@@ -1479,6 +1479,249 @@ export type JwksResponse = {
         [key: string]: unknown;
     }>;
 };
+/**
+ * What an eval run is executed against.
+ */
+export type EvalSubject = {
+    /**
+     * `agent` runs the ai_agent resource as it is deployed when the run opens, `agent_draft` the caller's unsaved edits of it as the editor holds them (carried in `draft`), and `agent_version` one past version named by `version`. The first and last are read server-side; all three are inlined into the run, so every case of a run executes one configuration: a deploy part-way through changes what the next run measures, never this one.
+     *
+     */
+    kind: 'agent' | 'agent_draft' | 'agent_version';
+    /**
+     * Path of the ai_agent resource.
+     */
+    path: string;
+    /**
+     * The agent's per-path version number when the run opened: how many times the resource had been saved, not a resource_version row id. For `agent` and `agent_draft` it names the configuration the run read and every case executed. For `agent_version` it is the request's own, says which version to inline, and is required.
+     *
+     */
+    version?: number | null;
+    draft?: AgentDraft;
+    /**
+     * Hash of the configuration a draft run executed, stamped server-side. A draft moves without the version moving, so this is what dates a run of one. It is also what recognises a draft run whose configuration was later deployed: when it matches the agent as deployed, the run's kind and version are rewritten to that version, once, and the hash is kept as what the resolution rests on.
+     *
+     */
+    draft_hash?: string;
+};
+/**
+ * The brain and tools of an agent, as the flow editor holds them. Carried by the request and present exactly when the subject kind is `agent_draft` — the edits exist only in the editor — where it is the whole definition of what ran: the run goes through the same unlinked branch of the agent executor the editor's own test uses.
+ *
+ */
+export type AgentDraft = {
+    /**
+     * The agent's input transforms: provider, system prompt, output type and the rest. The message and attachments come from the case and override anything named here.
+     *
+     */
+    input_transforms?: {
+        [key: string]: unknown;
+    };
+    tools?: Array<{
+        [key: string]: unknown;
+    }>;
+};
+export type EvalDataset = {
+    path: string;
+    summary?: string;
+    /**
+     * The columns of the results table, in display order.
+     */
+    scorers?: Array<Scorer>;
+    created_at: string;
+    created_by: string;
+    edited_at: string;
+    edited_by: string;
+};
+/**
+ * The inputs a standalone run feeds the agent.
+ */
+export type EvalCaseInput = {
+    user_message?: string;
+    user_attachments?: Array<{
+        [key: string]: unknown;
+    }>;
+};
+export type NewEvalCase = {
+    input?: EvalCaseInput;
+    /**
+     * Reference output a scorer compares a rerun against.
+     */
+    expected?: unknown;
+};
+export type SaveEvalCase = {
+    /**
+     * Absent for a case the dataset does not hold yet.
+     */
+    id?: string;
+} & NewEvalCase;
+export type EvalCase = {
+    id: string;
+    created_at: string;
+    created_by: string;
+} & NewEvalCase;
+/**
+ * A scorer is a column of the results table, and it is always a runnable: an ai_agent resource sent the run to grade, or a script handed the run as an argument. `id` is assigned when the scorer is added to a dataset and never reused: it is what makes a column the same column across experiments when the scorer is renamed, and a delta is only ever computed between two scores carrying the same id. A scorer sent without an id is given one.
+ *
+ */
+export type Scorer = {
+    id?: string;
+    /**
+     * Column header. Defaults to the last segment of the path.
+     */
+    name?: string;
+    /**
+     * A score at or above this counts as a pass, and the column reports a pass rate beside its mean. Applied when results are read rather than when they are produced, so moving the line re-reads every score already recorded instead of invalidating them.
+     *
+     */
+    pass_if?: number;
+    kind: 'script' | 'agent';
+    /**
+     * The script, or the ai_agent resource used as a judge.
+     */
+    path: string;
+};
+/**
+ * One run of a dataset: written once when the dataset is run, and only ever read afterwards. The case set it executed is returned by the results endpoint, not here: a listing would otherwise send the whole dataset back once per experiment.
+ */
+export type EvalExperiment = {
+    id: string;
+    dataset: string;
+    subject: EvalSubject;
+    /**
+     * This agent's nth run of this dataset, allocated once and never reused. What a run is called. Numbered per agent rather than per subject kind: runs of what is deployed and runs of its draft are the same agent's history.
+     *
+     */
+    run_number: number;
+    /**
+     * The flow executing the run: one job holding every case and its scores.
+     *
+     */
+    run_job_id: string;
+    case_count: number;
+    /**
+     * What the run scored, one entry per scorer that produced a number. Carried on the run itself so a list of runs can say what each one scored without reading every cell of every one of them. Empty on a run whose scores have not been read yet.
+     *
+     */
+    scores?: Array<ExperimentScore>;
+    /**
+     * Whether the flow executing this run is still going. What makes a list of runs worth watching rather than worth reloading.
+     *
+     */
+    running?: boolean;
+    created_at: string;
+    created_by: string;
+};
+/**
+ * One scorer's headline for one run: the two numbers a column reports, over that run's cells.
+ */
+export type ExperimentScore = {
+    scorer_id: string;
+    /**
+     * What the column is called in the dataset that ran it, resolved server-side because a list of runs spanning datasets cannot hold every dataset's scorers to look it up.
+     *
+     */
+    name: string;
+    kind: 'agent' | 'script';
+    mean?: number;
+    /**
+     * The share of scored cells at or above the column's threshold, for a column that has one. Absent where the column has no threshold and the mean is the whole headline.
+     *
+     */
+    pass_rate?: number;
+    scored: number;
+    /**
+     * How many of the run's cells the column failed on. A column that failed on all of them has no number to report and is still one of the columns that ran.
+     *
+     */
+    failed: number;
+};
+/**
+ * One scorer's verdict on one run, and how it compares with the baseline.
+ */
+export type CellScore = {
+    scorer_id: string;
+    score?: number;
+    reason?: string;
+    checks?: unknown;
+    error?: string;
+    /**
+     * The scorer read this case and had nothing to measure on it. Left out of the column's mean and pass rate rather than counted as a zero.
+     *
+     */
+    not_applicable?: boolean;
+    /**
+     * A scoring job is still running for this cell.
+     */
+    pending: boolean;
+    /**
+     * Which side of the scorer's `pass_if` threshold the score fell on. Absent when the column has no threshold, or has no score yet.
+     *
+     */
+    passed?: boolean;
+    /**
+     * The same scorer's number on the baseline experiment.
+     */
+    baseline?: number;
+    /**
+     * The baseline's score came from a different definition of this scorer, so the delta is a change of scorer as much as a change of agent.
+     *
+     */
+    definition_changed: boolean;
+};
+export type ExperimentRow = {
+    case_id: string;
+    input: EvalCaseInput;
+    expected?: unknown;
+    /**
+     * The iteration that ran this case. Absent between a run being recorded and its flow reaching this case, which reads as a case still to run.
+     *
+     */
+    job_id?: string;
+    /**
+     * The case's status; `running` until its iteration completes, and `unavailable` for a case whose job was retained away before anything read what it produced.
+     *
+     */
+    status: 'running' | 'success' | 'failure' | 'canceled' | 'skipped' | 'unavailable';
+    /**
+     * The agent's answer. The full trajectory stays reachable through job_id.
+     */
+    output?: string;
+    /**
+     * The agent version this cell ran against. Cells of one experiment can differ, which the table says rather than averaging two versions silently.
+     *
+     */
+    subject_version?: number;
+    /**
+     * For a run of unsaved edits, the hash of the configuration this cell ran. Edits move without a version changing, so this is what identifies what ran, and what recognises a run whose edits were later saved as a run of that version.
+     *
+     */
+    subject_draft_hash?: string;
+    /**
+     * One entry per scorer of the dataset, in column order.
+     */
+    scores: Array<CellScore>;
+};
+/**
+ * A column's summary. There is no single number for a dataset: averaging a judge with an exact match would invent one.
+ *
+ */
+export type ScorerMean = {
+    scorer_id: string;
+    mean?: number;
+    baseline_mean?: number;
+    /**
+     * The share of scored cells that passed, for a column with a threshold. Reported beside the mean rather than instead of it: a pass rate says how many cases are good enough, a mean says by how much, and neither answers the other's question.
+     *
+     */
+    pass_rate?: number;
+    baseline_pass_rate?: number;
+    scored: number;
+    /**
+     * Cells the baseline has no score for, so a column the baseline never ran shows as unscored rather than as a spurious difference.
+     */
+    missing_in_baseline: number;
+    definition_changed: boolean;
+};
 export type FlowConversation = {
     /**
      * Unique identifier for the conversation
@@ -1611,6 +1854,46 @@ export type AIConfig = {
     max_tokens_per_model?: {
         [key: string]: (number);
     };
+    model_pricing?: {
+        [key: string]: ModelPriceOverride;
+    };
+};
+/**
+ * negotiated rates in USD per million tokens, keyed `provider:model`
+ */
+export type ModelPriceOverride = {
+    input: number;
+    output: number;
+    cache_read?: number;
+    cache_write?: number;
+};
+export type AITokenUsageEvent = {
+    provider: AIProvider;
+    model: string;
+    session_id?: string;
+    input_tokens?: number;
+    cache_read_tokens?: number;
+    cache_write_tokens?: number;
+    output_tokens?: number;
+    /**
+     * only set by providers that bill back an exact figure
+     */
+    reported_cost_nano_usd?: number;
+    requests?: number;
+};
+export type AITokenUsageBucket = {
+    /**
+     * the grouped dimension's value; empty when grouping by model
+     */
+    key: string;
+    provider: string;
+    model: string;
+    input_tokens: number;
+    cache_read_tokens: number;
+    cache_write_tokens: number;
+    output_tokens: number;
+    reported_cost_nano_usd?: number;
+    requests: number;
 };
 export type InstanceAIProviderSummary = {
     provider: AIProvider;
@@ -2588,7 +2871,14 @@ export type Resource = {
     ws_specific?: boolean;
 };
 export type ResourceVersion = {
+    /**
+     * How this version is addressed. Unique across every resource, so it says nothing about how many times this one has been saved.
+     */
     id: number;
+    /**
+     * Which version of this resource it is, counted from its first. What a version is called.
+     */
+    version: number;
     created_at: string;
     created_by?: string;
 };
@@ -9497,7 +9787,10 @@ export type ClearResourceHistoryData = {
 };
 export type ClearResourceHistoryResponse = string;
 export type GetResourceVersionData = {
-    version: number;
+    /**
+     * The version's id, not its number.
+     */
+    id: number;
     workspace: string;
 };
 export type GetResourceVersionResponse = ResourceVersion & {
@@ -9505,7 +9798,10 @@ export type GetResourceVersionResponse = ResourceVersion & {
     missing_references: Array<(string)>;
 };
 export type RestoreResourceVersionData = {
-    version: number;
+    /**
+     * The version's id, not its number.
+     */
+    id: number;
     workspace: string;
 };
 export type RestoreResourceVersionResponse = string;
@@ -13890,68 +14186,276 @@ export type ListConversationMessagesData = {
     workspace: string;
 };
 export type ListConversationMessagesResponse = Array<FlowConversationMessage>;
-export type ListPathAutocompletePathsData = {
+export type ListEvalDatasetsData = {
+    workspace: string;
+};
+export type ListEvalDatasetsResponse = Array<EvalDataset>;
+export type CreateEvalDatasetData = {
     /**
-     * bypass the server-side cache and re-query the DB, refreshing the
-     * cache. Used right after a deploy so the new path appears immediately.
-     *
+     * new eval dataset
      */
-    force?: boolean;
+    requestBody: {
+        path: string;
+        summary?: string;
+        scorers?: Array<Scorer>;
+        /**
+         * The cases to create the dataset holding, so one can be assembled in a single act rather than created empty and filled in afterwards.
+         */
+        cases?: Array<NewEvalCase>;
+    };
     workspace: string;
 };
-export type ListPathAutocompletePathsResponse = {
-    paths: Array<(string)>;
+export type CreateEvalDatasetResponse = string;
+export type GetEvalDatasetData = {
+    path: string;
+    workspace: string;
 };
-export type ListRawAppsData = {
-    /**
-     * filter by exact matching user creator. Supports comma-separated list (e.g. 'alice,bob') and negation by prefixing all values with '!' (e.g. '!alice,!bob')
-     */
-    createdBy?: string;
-    /**
-     * When true, append per-user draft rows whose path has no
-     * deployed counterpart. Synthesized rows carry `draft_only: true`
-     * so the home page can render a "Draft" badge. Gated to
-     * non-operators + page 0 + no narrowing filters on the backend so
-     * picker callers stay deployed-only and pagination stays clean.
-     *
-     */
-    includeDraftOnly?: boolean;
-    /**
-     * Filter by label
-     */
-    label?: string;
+export type GetEvalDatasetResponse = EvalDataset;
+export type UpdateEvalDatasetData = {
+    path: string;
     /**
-     * order by desc order (default true)
+     * updated eval dataset
      */
-    orderDesc?: boolean;
+    requestBody: {
+        /**
+         * Renames the dataset. Its cases and experiments follow through the foreign keys, so a rename keeps the history it already has.
+         *
+         */
+        path?: string;
+        /**
+         * Left out to keep the stored summary; sent as "" to clear it.
+         */
+        summary?: string;
+        /**
+         * Left out to keep the dataset's columns as they are; sent to replace them wholesale.
+         *
+         */
+        scorers?: Array<Scorer>;
+        /**
+         * The cases as they should stand afterwards: all of them, each carrying its id if the dataset already has it. Sent with the rest of an edit so that a rename the dataset refuses refuses the case edits with it.
+         *
+         */
+        cases?: Array<SaveEvalCase>;
+    };
+    workspace: string;
+};
+export type UpdateEvalDatasetResponse = string;
+export type DeleteEvalDatasetData = {
+    path: string;
+    workspace: string;
+};
+export type DeleteEvalDatasetResponse = string;
+export type ListEvalCasesData = {
     /**
      * which page to return (start at 1, default 1)
      */
     page?: number;
+    path: string;
     /**
-     * mask to filter exact matching path
+     * number of items to return for a given page (default 30, max 100)
      */
-    pathExact?: string;
+    perPage?: number;
+    workspace: string;
+};
+export type ListEvalCasesResponse = {
+    cases: Array<EvalCase>;
+};
+export type EvalSubjectStateData = {
+    path: string;
+    workspace: string;
+};
+export type EvalSubjectStateResponse = {
+    version?: number;
+};
+export type EvalRunPayloadData = {
     /**
-     * mask to filter matching starting path
+     * The flow job that answered the case.
      */
-    pathStart?: string;
+    jobId: string;
+    workspace: string;
+};
+export type EvalRunPayloadResponse = {
     /**
-     * number of items to return for a given page (default 30, max 100)
+     * The case, the answer, and every tool call the agent made.
      */
-    perPage?: number;
+    run: {
+        [key: string]: unknown;
+    };
     /**
-     * (default false)
-     * show only the starred items
-     *
+     * The same run as a judge agent is shown it.
      */
-    starredOnly?: boolean;
+    rendered: string;
+};
+export type ScorerDefaultsData = {
     workspace: string;
 };
-export type ListRawAppsResponse = Array<ListableRawApp>;
-export type ResumeSuspendedTriggerJobsData = {
+export type ScorerDefaultsResponse = {
     /**
-     * Optional list of job IDs to reassign
+     * The system prompt a judge agent is created with.
+     */
+    judge_prompt: string;
+    script_template: string;
+};
+export type RecentScorersData = {
+    /**
+     * only scorers of this kind
+     */
+    kind?: 'script' | 'agent';
+    workspace: string;
+};
+export type RecentScorersResponse = Array<(Scorer & {
+    /**
+     * The dataset it is a column of.
+     */
+    dataset: string;
+})>;
+export type RunExperimentData = {
+    /**
+     * what to run
+     */
+    requestBody: {
+        dataset: string;
+        subject: EvalSubject;
+    };
+    workspace: string;
+};
+export type RunExperimentResponse = string;
+export type CollectExperimentData = {
+    id: string;
+    workspace: string;
+};
+export type CollectExperimentResponse = number;
+export type ListAllExperimentsData = {
+    /**
+     * Restrict to one agent's runs, which is what makes the list a history rather than a log. Runs of what is deployed, of a past version, and of the edits waiting on top are all that agent's, so this does not discriminate by kind.
+     *
+     */
+    subjectPath?: string;
+    workspace: string;
+};
+export type ListAllExperimentsResponse = Array<EvalExperiment>;
+export type ExperimentResultsData = {
+    /**
+     * The experiment every column is compared against. A delta is only computed between two scores of the same scorer id, and a column the baseline was never scored with reports it rather than showing a difference.
+     *
+     */
+    baseline?: string;
+    /**
+     * the experiment to read
+     */
+    id: string;
+    path: string;
+    workspace: string;
+};
+export type ExperimentResultsResponse = {
+    experiment: EvalExperiment;
+    baseline?: EvalExperiment;
+    /**
+     * The columns, which belong to the dataset rather than the experiment.
+     */
+    scorers: Array<Scorer>;
+    rows: Array<ExperimentRow>;
+    means: Array<ScorerMean>;
+    /**
+     * Cells scoring lower than the baseline, across every column.
+     */
+    regressed: number;
+    /**
+     * The version the subject is on now. A row that ran against an earlier one describes an agent that no longer exists.
+     *
+     */
+    subject_current_version?: number;
+    /**
+     * What the agent hashes to as deployed. A run of unsaved edits carrying this hash ran exactly what is deployed now — the edits were saved — so it is a run of that version rather than of edits.
+     *
+     */
+    subject_deployed_hash?: string;
+};
+export type ListPathAutocompletePathsData = {
+    /**
+     * bypass the server-side cache and re-query the DB, refreshing the
+     * cache. Used right after a deploy so the new path appears immediately.
+     *
+     */
+    force?: boolean;
+    workspace: string;
+};
+export type ListPathAutocompletePathsResponse = {
+    paths: Array<(string)>;
+};
+export type ListRawAppsData = {
+    /**
+     * filter by exact matching user creator. Supports comma-separated list (e.g. 'alice,bob') and negation by prefixing all values with '!' (e.g. '!alice,!bob')
+     */
+    createdBy?: string;
+    /**
+     * When true, append per-user draft rows whose path has no
+     * deployed counterpart. Synthesized rows carry `draft_only: true`
+     * so the home page can render a "Draft" badge. Gated to
+     * non-operators + page 0 + no narrowing filters on the backend so
+     * picker callers stay deployed-only and pagination stays clean.
+     *
+     */
+    includeDraftOnly?: boolean;
+    /**
+     * Filter by label
+     */
+    label?: string;
+    /**
+     * order by desc order (default true)
+     */
+    orderDesc?: boolean;
+    /**
+     * which page to return (start at 1, default 1)
+     */
+    page?: number;
+    /**
+     * mask to filter exact matching path
+     */
+    pathExact?: string;
+    /**
+     * mask to filter matching starting path
+     */
+    pathStart?: string;
+    /**
+     * number of items to return for a given page (default 30, max 100)
+     */
+    perPage?: number;
+    /**
+     * (default false)
+     * show only the starred items
+     *
+     */
+    starredOnly?: boolean;
+    workspace: string;
+};
+export type ListRawAppsResponse = Array<ListableRawApp>;
+export type RecordAiUsageData = {
+    requestBody: {
+        events: Array<AITokenUsageEvent>;
+    };
+    workspace: string;
+};
+export type RecordAiUsageResponse = void;
+export type ListAiUsageData = {
+    days?: number;
+    groupBy?: 'day' | 'user' | 'model';
+    /**
+     * workspace-wide usage (admin only) or the calling user's own
+     */
+    scope?: 'workspace' | 'self';
+    workspace: string;
+};
+export type ListAiUsageResponse = {
+    buckets: Array<AITokenUsageBucket>;
+    /**
+     * more buckets matched than were returned, so summing them under-reports
+     */
+    truncated: boolean;
+};
+export type ResumeSuspendedTriggerJobsData = {
+    /**
+     * Optional list of job IDs to reassign
      */
     requestBody?: {
         /**
@@ -22471,10 +22975,13 @@ export type $OpenApiTs = {
             };
         };
     };
-    '/w/{workspace}/resources/history/v/{version}': {
+    '/w/{workspace}/resources/history/v/{id}': {
         get: {
             req: {
-                version: number;
+                /**
+                 * The version's id, not its number.
+                 */
+                id: number;
                 workspace: string;
             };
             res: {
@@ -22488,10 +22995,13 @@ export type $OpenApiTs = {
             };
         };
     };
-    '/w/{workspace}/resources/history/restore/v/{version}': {
+    '/w/{workspace}/resources/history/restore/v/{id}': {
         post: {
             req: {
-                version: number;
+                /**
+                 * The version's id, not its number.
+                 */
+                id: number;
                 workspace: string;
             };
             res: {
@@ -29094,6 +29604,318 @@ export type $OpenApiTs = {
             };
         };
     };
+    '/w/{workspace}/ai_evals/datasets/list': {
+        get: {
+            req: {
+                workspace: string;
+            };
+            res: {
+                /**
+                 * eval datasets list
+                 */
+                200: Array<EvalDataset>;
+            };
+        };
+    };
+    '/w/{workspace}/ai_evals/datasets/create': {
+        post: {
+            req: {
+                /**
+                 * new eval dataset
+                 */
+                requestBody: {
+                    path: string;
+                    summary?: string;
+                    scorers?: Array<Scorer>;
+                    /**
+                     * The cases to create the dataset holding, so one can be assembled in a single act rather than created empty and filled in afterwards.
+                     */
+                    cases?: Array<NewEvalCase>;
+                };
+                workspace: string;
+            };
+            res: {
+                /**
+                 * eval dataset created
+                 */
+                200: string;
+            };
+        };
+    };
+    '/w/{workspace}/ai_evals/datasets/get/{path}': {
+        get: {
+            req: {
+                path: string;
+                workspace: string;
+            };
+            res: {
+                /**
+                 * eval dataset
+                 */
+                200: EvalDataset;
+            };
+        };
+    };
+    '/w/{workspace}/ai_evals/datasets/update/{path}': {
+        post: {
+            req: {
+                path: string;
+                /**
+                 * updated eval dataset
+                 */
+                requestBody: {
+                    /**
+                     * Renames the dataset. Its cases and experiments follow through the foreign keys, so a rename keeps the history it already has.
+                     *
+                     */
+                    path?: string;
+                    /**
+                     * Left out to keep the stored summary; sent as "" to clear it.
+                     */
+                    summary?: string;
+                    /**
+                     * Left out to keep the dataset's columns as they are; sent to replace them wholesale.
+                     *
+                     */
+                    scorers?: Array<Scorer>;
+                    /**
+                     * The cases as they should stand afterwards: all of them, each carrying its id if the dataset already has it. Sent with the rest of an edit so that a rename the dataset refuses refuses the case edits with it.
+                     *
+                     */
+                    cases?: Array<SaveEvalCase>;
+                };
+                workspace: string;
+            };
+            res: {
+                /**
+                 * eval dataset updated
+                 */
+                200: string;
+            };
+        };
+    };
+    '/w/{workspace}/ai_evals/datasets/delete/{path}': {
+        post: {
+            req: {
+                path: string;
+                workspace: string;
+            };
+            res: {
+                /**
+                 * eval dataset deleted
+                 */
+                200: string;
+            };
+        };
+    };
+    '/w/{workspace}/ai_evals/cases/list/{path}': {
+        get: {
+            req: {
+                /**
+                 * which page to return (start at 1, default 1)
+                 */
+                page?: number;
+                path: string;
+                /**
+                 * number of items to return for a given page (default 30, max 100)
+                 */
+                perPage?: number;
+                workspace: string;
+            };
+            res: {
+                /**
+                 * eval cases
+                 */
+                200: {
+                    cases: Array<EvalCase>;
+                };
+            };
+        };
+    };
+    '/w/{workspace}/ai_evals/subject_state': {
+        get: {
+            req: {
+                path: string;
+                workspace: string;
+            };
+            res: {
+                /**
+                 * the subject as it is now
+                 */
+                200: {
+                    version?: number;
+                };
+            };
+        };
+    };
+    '/w/{workspace}/ai_evals/run_payload': {
+        get: {
+            req: {
+                /**
+                 * The flow job that answered the case.
+                 */
+                jobId: string;
+                workspace: string;
+            };
+            res: {
+                /**
+                 * the run and its rendering
+                 */
+                200: {
+                    /**
+                     * The case, the answer, and every tool call the agent made.
+                     */
+                    run: {
+                        [key: string]: unknown;
+                    };
+                    /**
+                     * The same run as a judge agent is shown it.
+                     */
+                    rendered: string;
+                };
+            };
+        };
+    };
+    '/w/{workspace}/ai_evals/scorer_defaults': {
+        get: {
+            req: {
+                workspace: string;
+            };
+            res: {
+                /**
+                 * scorer defaults
+                 */
+                200: {
+                    /**
+                     * The system prompt a judge agent is created with.
+                     */
+                    judge_prompt: string;
+                    script_template: string;
+                };
+            };
+        };
+    };
+    '/w/{workspace}/ai_evals/scorers/recent': {
+        get: {
+            req: {
+                /**
+                 * only scorers of this kind
+                 */
+                kind?: 'script' | 'agent';
+                workspace: string;
+            };
+            res: {
+                /**
+                 * recently used scorers
+                 */
+                200: Array<(Scorer & {
+                    /**
+                     * The dataset it is a column of.
+                     */
+                    dataset: string;
+                })>;
+            };
+        };
+    };
+    '/w/{workspace}/ai_evals/experiments/run': {
+        post: {
+            req: {
+                /**
+                 * what to run
+                 */
+                requestBody: {
+                    dataset: string;
+                    subject: EvalSubject;
+                };
+                workspace: string;
+            };
+            res: {
+                /**
+                 * id of the created experiment
+                 */
+                200: string;
+            };
+        };
+    };
+    '/w/{workspace}/ai_evals/experiments/collect': {
+        post: {
+            req: {
+                id: string;
+                workspace: string;
+            };
+            res: {
+                /**
+                 * how many of the run's cases are recorded
+                 */
+                200: number;
+            };
+        };
+    };
+    '/w/{workspace}/ai_evals/experiments/list_all': {
+        get: {
+            req: {
+                /**
+                 * Restrict to one agent's runs, which is what makes the list a history rather than a log. Runs of what is deployed, of a past version, and of the edits waiting on top are all that agent's, so this does not discriminate by kind.
+                 *
+                 */
+                subjectPath?: string;
+                workspace: string;
+            };
+            res: {
+                /**
+                 * The 100 newest experiments, each naming the dataset it is of. Restricted to datasets the caller can read.
+                 *
+                 */
+                200: Array<EvalExperiment>;
+            };
+        };
+    };
+    '/w/{workspace}/ai_evals/experiments/results/{path}': {
+        get: {
+            req: {
+                /**
+                 * The experiment every column is compared against. A delta is only computed between two scores of the same scorer id, and a column the baseline was never scored with reports it rather than showing a difference.
+                 *
+                 */
+                baseline?: string;
+                /**
+                 * the experiment to read
+                 */
+                id: string;
+                path: string;
+                workspace: string;
+            };
+            res: {
+                /**
+                 * experiment results
+                 */
+                200: {
+                    experiment: EvalExperiment;
+                    baseline?: EvalExperiment;
+                    /**
+                     * The columns, which belong to the dataset rather than the experiment.
+                     */
+                    scorers: Array<Scorer>;
+                    rows: Array<ExperimentRow>;
+                    means: Array<ScorerMean>;
+                    /**
+                     * Cells scoring lower than the baseline, across every column.
+                     */
+                    regressed: number;
+                    /**
+                     * The version the subject is on now. A row that ran against an earlier one describes an agent that no longer exists.
+                     *
+                     */
+                    subject_current_version?: number;
+                    /**
+                     * What the agent hashes to as deployed. A run of unsaved edits carrying this hash ran exactly what is deployed now — the edits were saved — so it is a run of that version rather than of edits.
+                     *
+                     */
+                    subject_deployed_hash?: string;
+                };
+            };
+        };
+    };
     '/w/{workspace}/path_autocomplete/list_paths': {
         get: {
             req: {
@@ -29171,6 +29993,45 @@ export type $OpenApiTs = {
             };
         };
     };
+    '/w/{workspace}/ai/usage': {
+        post: {
+            req: {
+                requestBody: {
+                    events: Array<AITokenUsageEvent>;
+                };
+                workspace: string;
+            };
+            res: {
+                /**
+                 * usage recorded
+                 */
+                204: void;
+            };
+        };
+        get: {
+            req: {
+                days?: number;
+                groupBy?: 'day' | 'user' | 'model';
+                /**
+                 * workspace-wide usage (admin only) or the calling user's own
+                 */
+                scope?: 'workspace' | 'self';
+                workspace: string;
+            };
+            res: {
+                /**
+                 * usage buckets
+                 */
+                200: {
+                    buckets: Array<AITokenUsageBucket>;
+                    /**
+                     * more buckets matched than were returned, so summing them under-reports
+                     */
+                    truncated: boolean;
+                };
+            };
+        };
+    };
     '/w/{workspace}/trigger/{trigger_kind}/resume_suspended_trigger_jobs/{trigger_path}': {
         post: {
             req: {