td-barrage 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -4
- package/dist/client.d.ts +3 -1
- package/dist/client.js +4 -2
- package/dist/deps.d.ts +8 -1
- package/dist/deps.js +6 -5
- package/dist/git.d.ts +28 -0
- package/dist/git.js +47 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +12 -4
- package/dist/log.d.ts +16 -1
- package/dist/log.js +117 -74
- package/dist/orchestrator.d.ts +7 -2
- package/dist/orchestrator.js +79 -26
- package/dist/pause.d.ts +34 -0
- package/dist/pause.js +119 -0
- package/dist/schema.d.ts +0 -13
- package/dist/schema.js +0 -1
- package/dist/server.js +4 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -69,7 +69,6 @@ Supported task fields:
|
|
|
69
69
|
- `maxAttempts`: per-task retry limit
|
|
70
70
|
- `timeoutMs`: per-task prompt timeout
|
|
71
71
|
- `cwd`: per-task working directory
|
|
72
|
-
- `resultPath`: optional metadata field
|
|
73
72
|
- `success`: `default`, `file_exists`, or `result_json`
|
|
74
73
|
- `recoveryNotes`, `error`, `startedAt`, `finishedAt`: runtime metadata
|
|
75
74
|
|
|
@@ -77,7 +76,7 @@ Supported task fields:
|
|
|
77
76
|
|
|
78
77
|
```sh
|
|
79
78
|
npm run dev -- --tasks queue.json
|
|
80
|
-
npm run dev -- queue.json --max-attempts 3 --cwd /path/to/work
|
|
79
|
+
npm run dev -- queue.json --max-attempts 3 --cwd /path/to/work
|
|
81
80
|
npm run dev -- --tasks queue.json --dry-run
|
|
82
81
|
npm run dev -- --tasks queue.json --json
|
|
83
82
|
npm run dev -- --tasks queue.json --verbose --timestamps
|
|
@@ -88,8 +87,8 @@ Options:
|
|
|
88
87
|
- `--tasks, -t <path>`: task file (also accepted as a positional argument)
|
|
89
88
|
- `--max-attempts <n>`: global retry limit (per-task `maxAttempts` overrides it)
|
|
90
89
|
- `--cwd <path>`: working directory for tasks that do not set their own
|
|
91
|
-
- `--results-dir <path>`: where result files are expected, default `.queue/results`
|
|
92
90
|
- `--dry-run`: print the topological plan (one id per line) and exit without running
|
|
91
|
+
- `--no-commit`: skip the git commit stage that otherwise runs after each successful task
|
|
93
92
|
- `--json`: emit newline-delimited JSON events instead of human output
|
|
94
93
|
|
|
95
94
|
Exit codes:
|
|
@@ -97,11 +96,32 @@ Exit codes:
|
|
|
97
96
|
- `0`: all reachable tasks completed
|
|
98
97
|
- `1`: at least one task failed or became blocked
|
|
99
98
|
- `2`: CLI usage or missing task-file error
|
|
99
|
+
- `75`: the run paused on a provider quota / rate-limit wall (see below)
|
|
100
|
+
|
|
101
|
+
## Pausing on quota / rate limits
|
|
102
|
+
|
|
103
|
+
Running out of provider tokens is treated as a property of the run, not of any
|
|
104
|
+
task. When a prompt fails with a quota, rate-limit, or overload signal (HTTP
|
|
105
|
+
`402`/`429`/`503`/`529`, or matching error wording), the runner:
|
|
106
|
+
|
|
107
|
+
- rolls back the attempt it charged — the task did not really run, so its retry
|
|
108
|
+
budget is untouched,
|
|
109
|
+
- leaves the task `pending` (exactly as crash recovery does),
|
|
110
|
+
- stops the rest of the sweep immediately, since every ready task would hit the
|
|
111
|
+
same wall, and
|
|
112
|
+
- exits `75` (`EX_TEMPFAIL`) with the reason on stderr.
|
|
113
|
+
|
|
114
|
+
Because state is persisted atomically after every transition, resuming is just
|
|
115
|
+
re-running the same command once tokens are back — work picks up where it left
|
|
116
|
+
off, with no failed or blocked tasks recorded from the pause. The distinct exit
|
|
117
|
+
code lets a wrapper or scheduler tell "try again later" apart from `1` ("something
|
|
118
|
+
is broken"). A `Retry-After` header, when present, is surfaced as a suggested
|
|
119
|
+
wait in the pause message.
|
|
100
120
|
|
|
101
121
|
## Telemetry & output
|
|
102
122
|
|
|
103
123
|
By default the runner is deliberately verbose, narrating progress as it happens:
|
|
104
|
-
a startup header (version, task file, cwd,
|
|
124
|
+
a startup header (version, task file, cwd, limits), the resolved
|
|
105
125
|
execution plan, a line as each task starts, and a line as each task finishes with
|
|
106
126
|
its duration and overall progress (`2/5 complete`). Failures, retries, and blocks
|
|
107
127
|
are always reported, along with a closing summary and total elapsed time.
|
|
@@ -128,6 +148,14 @@ stays easy to pipe.
|
|
|
128
148
|
|
|
129
149
|
`result_json` succeeds when the configured JSON file parses and contains `"status": "ok"`.
|
|
130
150
|
|
|
151
|
+
## Commits
|
|
152
|
+
|
|
153
|
+
After a task passes its success check, the orchestrator stages everything with
|
|
154
|
+
`git add -A` and commits it in the task's cwd with the message `feat: <id>`. A
|
|
155
|
+
clean working tree (nothing to commit) is a quiet no-op, and a commit that fails
|
|
156
|
+
outright is reported as a warning but never flips the task back to failed — the
|
|
157
|
+
task already succeeded. Pass `--no-commit` to skip this stage entirely.
|
|
158
|
+
|
|
131
159
|
## Recovery
|
|
132
160
|
|
|
133
161
|
On startup, any task persisted as `running` is reset to `pending`, annotated in `recoveryNotes`, and retried. The `attempts` counter is preserved, so a task that crashed mid-run still counts that attempt.
|
package/dist/client.d.ts
CHANGED
|
@@ -6,4 +6,6 @@ export interface ClientOptions {
|
|
|
6
6
|
timeout?: number;
|
|
7
7
|
config?: unknown;
|
|
8
8
|
}
|
|
9
|
-
export
|
|
9
|
+
/** Imports an SDK module by specifier. Injectable so the export-selection logic is testable. */
|
|
10
|
+
export type SdkImport = (specifier: string) => Promise<unknown>;
|
|
11
|
+
export declare function createOpenCodeClient(options?: ClientOptions, importSdk?: SdkImport): Promise<OpenCodeClient>;
|
package/dist/client.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
// A `new Function` indirection keeps TypeScript from rewriting this dynamic
|
|
2
|
+
// import into a require during transpile, so the specifier resolves at runtime.
|
|
3
|
+
const defaultImport = new Function("specifier", "return import(specifier)");
|
|
4
|
+
export async function createOpenCodeClient(options = {}, importSdk = defaultImport) {
|
|
3
5
|
const sdk = (await importSdkWithFallback(importSdk));
|
|
4
6
|
// A bare client only works when we already know where the server lives.
|
|
5
7
|
// Without a baseUrl it would issue requests against relative URLs like
|
package/dist/deps.d.ts
CHANGED
|
@@ -1,4 +1,11 @@
|
|
|
1
1
|
import type { Task } from "./schema.js";
|
|
2
|
-
export type DependencyState =
|
|
2
|
+
export type DependencyState = {
|
|
3
|
+
state: "ready";
|
|
4
|
+
} | {
|
|
5
|
+
state: "waiting";
|
|
6
|
+
} | {
|
|
7
|
+
state: "blocked";
|
|
8
|
+
blockedBy: string;
|
|
9
|
+
};
|
|
3
10
|
export declare function dependencyState(task: Task, tasks: Task[]): DependencyState;
|
|
4
11
|
export declare function topologicalOrder(tasks: Task[]): Task[];
|
package/dist/deps.js
CHANGED
|
@@ -3,13 +3,14 @@ export function dependencyState(task, tasks) {
|
|
|
3
3
|
for (const depId of task.dependsOn) {
|
|
4
4
|
const dep = byId.get(depId);
|
|
5
5
|
if (!dep)
|
|
6
|
-
return "blocked";
|
|
7
|
-
if (dep.status === "failed" || dep.status === "blocked")
|
|
8
|
-
return "blocked";
|
|
6
|
+
return { state: "blocked", blockedBy: `${depId} (missing)` };
|
|
7
|
+
if (dep.status === "failed" || dep.status === "blocked") {
|
|
8
|
+
return { state: "blocked", blockedBy: `${depId} (${dep.status})` };
|
|
9
|
+
}
|
|
9
10
|
if (dep.status !== "done")
|
|
10
|
-
return "waiting";
|
|
11
|
+
return { state: "waiting" };
|
|
11
12
|
}
|
|
12
|
-
return "ready";
|
|
13
|
+
return { state: "ready" };
|
|
13
14
|
}
|
|
14
15
|
export function topologicalOrder(tasks) {
|
|
15
16
|
const byId = new Map(tasks.map((task) => [task.id, task]));
|
package/dist/git.d.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
export interface GitExecResult {
|
|
2
|
+
code: number;
|
|
3
|
+
stdout: string;
|
|
4
|
+
stderr: string;
|
|
5
|
+
}
|
|
6
|
+
/** Runs `git <args>` in `cwd` and resolves with its exit code and output (never rejects). */
|
|
7
|
+
export type GitExec = (args: string[], cwd: string) => Promise<GitExecResult>;
|
|
8
|
+
export interface CommitInput {
|
|
9
|
+
cwd: string;
|
|
10
|
+
message: string;
|
|
11
|
+
}
|
|
12
|
+
export interface CommitResult {
|
|
13
|
+
/** "committed" — a new commit was made; "nothing" — clean tree, nothing staged; "error" — git failed. */
|
|
14
|
+
status: "committed" | "nothing" | "error";
|
|
15
|
+
/** Short SHA of the new commit, when status is "committed". */
|
|
16
|
+
commit?: string;
|
|
17
|
+
/** Human-readable detail, primarily for the "error" status. */
|
|
18
|
+
detail?: string;
|
|
19
|
+
}
|
|
20
|
+
export interface GitCommitter {
|
|
21
|
+
commit(input: CommitInput): Promise<CommitResult>;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Stages every change with `git add -A` and commits it. A clean working tree
|
|
25
|
+
* (nothing to commit) is reported as "nothing" rather than an error, since a
|
|
26
|
+
* task can legitimately succeed without touching tracked files.
|
|
27
|
+
*/
|
|
28
|
+
export declare function createGitCommitter(exec?: GitExec): GitCommitter;
|
package/dist/git.js
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
/**
|
|
3
|
+
* Stages every change with `git add -A` and commits it. A clean working tree
|
|
4
|
+
* (nothing to commit) is reported as "nothing" rather than an error, since a
|
|
5
|
+
* task can legitimately succeed without touching tracked files.
|
|
6
|
+
*/
|
|
7
|
+
export function createGitCommitter(exec = defaultExec) {
|
|
8
|
+
return {
|
|
9
|
+
async commit({ cwd, message }) {
|
|
10
|
+
const add = await exec(["add", "-A"], cwd);
|
|
11
|
+
if (add.code !== 0) {
|
|
12
|
+
return { status: "error", detail: detailOf(add) || "git add failed" };
|
|
13
|
+
}
|
|
14
|
+
const commit = await exec(["commit", "-m", message], cwd);
|
|
15
|
+
if (commit.code === 0) {
|
|
16
|
+
const head = await exec(["rev-parse", "--short", "HEAD"], cwd);
|
|
17
|
+
return { status: "committed", commit: head.code === 0 ? head.stdout.trim() : undefined };
|
|
18
|
+
}
|
|
19
|
+
if (isNothingToCommit(commit)) {
|
|
20
|
+
return { status: "nothing" };
|
|
21
|
+
}
|
|
22
|
+
return { status: "error", detail: detailOf(commit) || `git commit exited with code ${commit.code}` };
|
|
23
|
+
},
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
// `git commit` exits non-zero with this phrasing when the index has no staged
|
|
27
|
+
// changes — a no-op we treat as success, not failure.
|
|
28
|
+
function isNothingToCommit(result) {
|
|
29
|
+
const text = `${result.stdout}\n${result.stderr}`;
|
|
30
|
+
return /nothing to commit|no changes added to commit|nothing added to commit/i.test(text);
|
|
31
|
+
}
|
|
32
|
+
function detailOf(result) {
|
|
33
|
+
return (result.stderr.trim() || result.stdout.trim()).split("\n")[0] ?? "";
|
|
34
|
+
}
|
|
35
|
+
const defaultExec = (args, cwd) => new Promise((resolve) => {
|
|
36
|
+
execFile("git", args, { cwd }, (error, stdout, stderr) => {
|
|
37
|
+
if (!error) {
|
|
38
|
+
resolve({ code: 0, stdout: String(stdout), stderr: String(stderr) });
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
// A process that exits non-zero surfaces a numeric `code`; a spawn
|
|
42
|
+
// failure (e.g. git not installed) surfaces a string code like "ENOENT",
|
|
43
|
+
// in which case we fall back to 1 and lean on the error message.
|
|
44
|
+
const code = typeof error.code === "number" ? error.code : 1;
|
|
45
|
+
resolve({ code, stdout: String(stdout), stderr: String(stderr) || error.message });
|
|
46
|
+
});
|
|
47
|
+
});
|
package/dist/index.d.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import fs from "node:fs/promises";
|
|
3
|
+
import { type GitCommitter } from "./git.js";
|
|
3
4
|
import { type OpenCodeClient } from "./orchestrator.js";
|
|
4
5
|
export interface MainDeps {
|
|
5
6
|
fs?: typeof fs;
|
|
6
7
|
client?: OpenCodeClient;
|
|
8
|
+
git?: GitCommitter;
|
|
7
9
|
stdout?: Pick<NodeJS.WritableStream, "write">;
|
|
8
10
|
stderr?: Pick<NodeJS.WritableStream, "write">;
|
|
9
11
|
}
|
package/dist/index.js
CHANGED
|
@@ -7,8 +7,9 @@ import { pathToFileURL } from "node:url";
|
|
|
7
7
|
import { parseArgs } from "node:util";
|
|
8
8
|
import { createOpenCodeClient } from "./client.js";
|
|
9
9
|
import { topologicalOrder } from "./deps.js";
|
|
10
|
+
import { createGitCommitter } from "./git.js";
|
|
10
11
|
import { createLogger } from "./log.js";
|
|
11
|
-
import {
|
|
12
|
+
import { runOrchestrator } from "./orchestrator.js";
|
|
12
13
|
import { loadTasks } from "./tasks.js";
|
|
13
14
|
export async function main(argv = process.argv.slice(2), deps = {}) {
|
|
14
15
|
const stdout = deps.stdout ?? process.stdout;
|
|
@@ -20,9 +21,9 @@ export async function main(argv = process.argv.slice(2), deps = {}) {
|
|
|
20
21
|
tasks: { type: "string", short: "t" },
|
|
21
22
|
"max-attempts": { type: "string" },
|
|
22
23
|
cwd: { type: "string" },
|
|
23
|
-
"results-dir": { type: "string" },
|
|
24
24
|
json: { type: "boolean", default: false },
|
|
25
25
|
"dry-run": { type: "boolean", default: false },
|
|
26
|
+
"no-commit": { type: "boolean", default: false },
|
|
26
27
|
verbose: { type: "boolean", short: "v", default: false },
|
|
27
28
|
quiet: { type: "boolean", short: "q", default: false },
|
|
28
29
|
color: { type: "boolean", default: false },
|
|
@@ -49,7 +50,6 @@ export async function main(argv = process.argv.slice(2), deps = {}) {
|
|
|
49
50
|
timestamps: Boolean(parsed.values.timestamps),
|
|
50
51
|
}, stdout);
|
|
51
52
|
const maxAttempts = parsed.values["max-attempts"] ? Number(parsed.values["max-attempts"]) : undefined;
|
|
52
|
-
const resultsDir = parsed.values["results-dir"] ?? defaultResultsDir(cwd);
|
|
53
53
|
if (maxAttempts !== undefined && (!Number.isInteger(maxAttempts) || maxAttempts < 1)) {
|
|
54
54
|
stderr.write("--max-attempts must be a positive integer.\n");
|
|
55
55
|
return 2;
|
|
@@ -62,16 +62,24 @@ export async function main(argv = process.argv.slice(2), deps = {}) {
|
|
|
62
62
|
return 0;
|
|
63
63
|
}
|
|
64
64
|
const client = deps.client ?? (await createOpenCodeClient());
|
|
65
|
+
const git = parsed.values["no-commit"] ? undefined : deps.git ?? createGitCommitter();
|
|
65
66
|
const summary = await runOrchestrator({
|
|
66
67
|
tasksPath,
|
|
67
68
|
client,
|
|
68
69
|
fs: queueFs,
|
|
69
70
|
log: logger,
|
|
70
71
|
cwd,
|
|
71
|
-
resultsDir,
|
|
72
72
|
maxAttempts,
|
|
73
|
+
git,
|
|
73
74
|
version: readVersion(),
|
|
74
75
|
});
|
|
76
|
+
// 75 = EX_TEMPFAIL: the run hit a quota/rate-limit wall and stopped cleanly
|
|
77
|
+
// with work still pending. Distinct from 1 (genuine failures) so a wrapper
|
|
78
|
+
// or scheduler can tell "try again later" apart from "something is broken".
|
|
79
|
+
if (summary.paused) {
|
|
80
|
+
stderr.write(`Paused: ${summary.pauseReason ?? "out of provider tokens"} Re-run to resume.\n`);
|
|
81
|
+
return 75;
|
|
82
|
+
}
|
|
75
83
|
return summary.failed > 0 || summary.blocked > 0 ? 1 : 0;
|
|
76
84
|
}
|
|
77
85
|
catch (error) {
|
package/dist/log.d.ts
CHANGED
|
@@ -6,7 +6,6 @@ export interface RunInfo {
|
|
|
6
6
|
maxAttempts: number;
|
|
7
7
|
timeoutMs: number;
|
|
8
8
|
cwd: string;
|
|
9
|
-
resultsDir: string;
|
|
10
9
|
}
|
|
11
10
|
export type LogEvent = {
|
|
12
11
|
type: "run_started";
|
|
@@ -36,6 +35,17 @@ export type LogEvent = {
|
|
|
36
35
|
durationMs: number;
|
|
37
36
|
done: number;
|
|
38
37
|
total: number;
|
|
38
|
+
} | {
|
|
39
|
+
type: "task_committed";
|
|
40
|
+
taskId: string;
|
|
41
|
+
message: string;
|
|
42
|
+
status: "committed" | "nothing";
|
|
43
|
+
commit?: string;
|
|
44
|
+
} | {
|
|
45
|
+
type: "task_commit_failed";
|
|
46
|
+
taskId: string;
|
|
47
|
+
message: string;
|
|
48
|
+
detail: string;
|
|
39
49
|
} | {
|
|
40
50
|
type: "task_failed";
|
|
41
51
|
taskId: string;
|
|
@@ -49,6 +59,11 @@ export type LogEvent = {
|
|
|
49
59
|
taskId: string;
|
|
50
60
|
reason: string;
|
|
51
61
|
blockedBy?: string;
|
|
62
|
+
} | {
|
|
63
|
+
type: "run_paused";
|
|
64
|
+
taskId: string;
|
|
65
|
+
reason: string;
|
|
66
|
+
resumeAfterMs?: number;
|
|
52
67
|
} | {
|
|
53
68
|
type: "summary";
|
|
54
69
|
summary: Summary;
|
package/dist/log.js
CHANGED
|
@@ -8,8 +8,11 @@ const EVENT_LEVEL = {
|
|
|
8
8
|
task_session: 2,
|
|
9
9
|
task_check: 2,
|
|
10
10
|
task_done: 1,
|
|
11
|
+
task_committed: 1,
|
|
12
|
+
task_commit_failed: 0,
|
|
11
13
|
task_failed: 0,
|
|
12
14
|
task_blocked: 0,
|
|
15
|
+
run_paused: 0,
|
|
13
16
|
summary: 0,
|
|
14
17
|
dry_run: 0,
|
|
15
18
|
server: 1,
|
|
@@ -61,42 +64,31 @@ function createPrettyRenderer(output, s) {
|
|
|
61
64
|
return (event) => {
|
|
62
65
|
switch (event.type) {
|
|
63
66
|
case "run_started": {
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
const details = [
|
|
67
|
-
`${s.dim("tasks ")} ${i.tasksPath} ${s.dim(`(${i.total} ${plural(i.total, "task")})`)}`,
|
|
68
|
-
`${s.dim("cwd ")} ${i.cwd}`,
|
|
69
|
-
`${s.dim("results")} ${i.resultsDir}`,
|
|
70
|
-
`${s.dim("limits ")} ${i.maxAttempts} ${plural(i.maxAttempts, "attempt")} ${s.dim("·")} ${fmtDuration(i.timeoutMs)} timeout`,
|
|
71
|
-
].join("\n");
|
|
72
|
-
note(details, s.dim("run details"), opts);
|
|
67
|
+
intro(s.bold(`td-barrage ${event.info.version}`), opts);
|
|
68
|
+
note(runDetailLines(event.info, s).join("\n"), s.dim("run details"), opts);
|
|
73
69
|
return;
|
|
74
70
|
}
|
|
75
|
-
case "plan":
|
|
76
|
-
|
|
77
|
-
? s.dim("(no tasks)")
|
|
78
|
-
: event.taskIds.map((id) => s.cyan(id)).join(s.dim(" → "));
|
|
79
|
-
clack.step(`${s.dim("plan")} ${body}`, opts);
|
|
71
|
+
case "plan":
|
|
72
|
+
clack.step(`${s.dim("plan")} ${planBody(event.taskIds, s)}`, opts);
|
|
80
73
|
return;
|
|
81
|
-
}
|
|
82
74
|
case "task_started":
|
|
83
75
|
active = spinner(opts);
|
|
84
76
|
active.start(`${s.bold(event.taskId)} ${s.dim(`running (attempt ${event.attempt}/${event.maxAttempts})`)}`);
|
|
85
77
|
return;
|
|
86
78
|
case "task_session":
|
|
87
79
|
if (active)
|
|
88
|
-
active.message(s.dim(`${event.taskId} ·
|
|
80
|
+
active.message(s.dim(`${event.taskId} · ${sessionText(event.sessionId)}`));
|
|
89
81
|
else
|
|
90
|
-
clack.message(s.dim(`↳
|
|
82
|
+
clack.message(s.dim(`↳ ${sessionText(event.sessionId)}`), opts);
|
|
91
83
|
return;
|
|
92
84
|
case "task_check":
|
|
93
85
|
if (active)
|
|
94
|
-
active.message(s.dim(`${event.taskId} ·
|
|
86
|
+
active.message(s.dim(`${event.taskId} · ${checkText(event.strategy, event.ok)}`));
|
|
95
87
|
else
|
|
96
|
-
clack.message(s.dim(`↳
|
|
88
|
+
clack.message(s.dim(`↳ ${checkText(event.strategy, event.ok)}`), opts);
|
|
97
89
|
return;
|
|
98
90
|
case "task_done": {
|
|
99
|
-
const msg =
|
|
91
|
+
const msg = doneMessage(event, s);
|
|
100
92
|
if (active) {
|
|
101
93
|
active.stop(msg);
|
|
102
94
|
active = null;
|
|
@@ -105,11 +97,14 @@ function createPrettyRenderer(output, s) {
|
|
|
105
97
|
clack.success(msg, opts);
|
|
106
98
|
return;
|
|
107
99
|
}
|
|
100
|
+
case "task_committed":
|
|
101
|
+
clack.message(committedDetail(event, s), opts);
|
|
102
|
+
return;
|
|
103
|
+
case "task_commit_failed":
|
|
104
|
+
clack.warn(`${s.bold(event.taskId)} ${s.yellow("commit failed")} ${s.dim(event.detail)}`, opts);
|
|
105
|
+
return;
|
|
108
106
|
case "task_failed": {
|
|
109
|
-
const
|
|
110
|
-
? s.yellow(`will retry (attempt ${event.attempt}/${event.maxAttempts})`)
|
|
111
|
-
: s.red(`gave up after ${event.attempt} ${plural(event.attempt, "attempt")}`);
|
|
112
|
-
const msg = `${s.bold(event.taskId)} ${s.red("failed")} ${s.dim(`in ${fmtDuration(event.durationMs)}`)}: ${event.error} ${tail}`;
|
|
107
|
+
const msg = failureMessage(event, s);
|
|
113
108
|
if (active) {
|
|
114
109
|
// cancel() finalizes the spinner with a red ■ — distinct from the
|
|
115
110
|
// ▲ clack uses for a blocked task, and matching clack.error's glyph
|
|
@@ -121,9 +116,15 @@ function createPrettyRenderer(output, s) {
|
|
|
121
116
|
clack.error(msg, opts);
|
|
122
117
|
return;
|
|
123
118
|
}
|
|
124
|
-
case "task_blocked":
|
|
125
|
-
|
|
126
|
-
|
|
119
|
+
case "task_blocked":
|
|
120
|
+
clack.warn(`${s.bold(event.taskId)} ${s.yellow("blocked")}${blockedSuffix(event, s)} ${s.dim(event.reason)}`, opts);
|
|
121
|
+
return;
|
|
122
|
+
case "run_paused": {
|
|
123
|
+
if (active) {
|
|
124
|
+
active.stop();
|
|
125
|
+
active = null;
|
|
126
|
+
}
|
|
127
|
+
clack.warn(pausedMessage(event, s), opts);
|
|
127
128
|
return;
|
|
128
129
|
}
|
|
129
130
|
case "summary": {
|
|
@@ -131,16 +132,8 @@ function createPrettyRenderer(output, s) {
|
|
|
131
132
|
active.stop();
|
|
132
133
|
active = null;
|
|
133
134
|
}
|
|
134
|
-
const
|
|
135
|
-
|
|
136
|
-
s.green(`${sm.done} done`),
|
|
137
|
-
count(sm.failed, "failed", s.red, s),
|
|
138
|
-
count(sm.blocked, "blocked", s.yellow, s),
|
|
139
|
-
count(sm.pending, "pending", s.yellow, s),
|
|
140
|
-
];
|
|
141
|
-
const ok = sm.failed === 0 && sm.blocked === 0;
|
|
142
|
-
const head = ok ? s.green("done") : s.red("finished with issues");
|
|
143
|
-
outro(`${head} ${parts.join(s.dim(" · "))} ${s.dim(`(${fmtDuration(event.durationMs)})`)}`, opts);
|
|
135
|
+
const head = summaryHead(event.summary, s, false);
|
|
136
|
+
outro(`${head} ${summaryParts(event.summary, s).join(s.dim(" · "))} ${s.dim(`(${fmtDuration(event.durationMs)})`)}`, opts);
|
|
144
137
|
return;
|
|
145
138
|
}
|
|
146
139
|
case "server":
|
|
@@ -158,50 +151,36 @@ function createPrettyRenderer(output, s) {
|
|
|
158
151
|
}
|
|
159
152
|
function formatHuman(event, s) {
|
|
160
153
|
switch (event.type) {
|
|
161
|
-
case "run_started":
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
s.
|
|
165
|
-
|
|
166
|
-
` ${s.dim("cwd ")} ${i.cwd}`,
|
|
167
|
-
` ${s.dim("results")} ${i.resultsDir}`,
|
|
168
|
-
` ${s.dim("limits ")} ${i.maxAttempts} ${plural(i.maxAttempts, "attempt")} ${s.dim("·")} ${fmtDuration(i.timeoutMs)} timeout`,
|
|
169
|
-
];
|
|
170
|
-
return lines.join("\n");
|
|
171
|
-
}
|
|
154
|
+
case "run_started":
|
|
155
|
+
return [
|
|
156
|
+
s.bold(`▶ td-barrage ${event.info.version}`),
|
|
157
|
+
...runDetailLines(event.info, s).map((line) => ` ${line}`),
|
|
158
|
+
].join("\n");
|
|
172
159
|
case "plan":
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
160
|
+
return event.taskIds.length === 0
|
|
161
|
+
? s.dim("plan (no tasks)")
|
|
162
|
+
: `${s.dim("plan ")} ${planBody(event.taskIds, s)}`;
|
|
176
163
|
case "task_started":
|
|
177
164
|
return `${s.blue("▶")} ${s.bold(event.taskId)} ${s.dim(`starting (attempt ${event.attempt}/${event.maxAttempts})`)}`;
|
|
178
165
|
case "task_session":
|
|
179
|
-
return s.dim(` ↳
|
|
166
|
+
return s.dim(` ↳ ${sessionText(event.sessionId)}`);
|
|
180
167
|
case "task_check":
|
|
181
|
-
return s.dim(` ↳
|
|
168
|
+
return s.dim(` ↳ ${checkText(event.strategy, event.ok)}`);
|
|
182
169
|
case "task_done":
|
|
183
|
-
return `${s.green("✔")} ${
|
|
184
|
-
case "
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
case "task_blocked":
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
170
|
+
return `${s.green("✔")} ${doneMessage(event, s)}`;
|
|
171
|
+
case "task_committed":
|
|
172
|
+
return s.dim(" ↳ ") + committedDetail(event, s);
|
|
173
|
+
case "task_commit_failed":
|
|
174
|
+
return `${s.yellow("⚠")} ${s.bold(event.taskId)} ${s.yellow("commit failed")} ${s.dim(event.detail)}`;
|
|
175
|
+
case "task_failed":
|
|
176
|
+
return `${s.red("✖")} ${failureMessage(event, s)}`;
|
|
177
|
+
case "task_blocked":
|
|
178
|
+
return `${s.yellow("⊘")} ${s.bold(event.taskId)} ${s.yellow("blocked")}${blockedSuffix(event, s)} ${s.dim(event.reason)}`;
|
|
179
|
+
case "run_paused":
|
|
180
|
+
return `${s.yellow("⏸")} ${pausedMessage(event, s)}`;
|
|
194
181
|
case "summary": {
|
|
195
|
-
const
|
|
196
|
-
|
|
197
|
-
s.green(`${sm.done} done`),
|
|
198
|
-
count(sm.failed, "failed", s.red, s),
|
|
199
|
-
count(sm.blocked, "blocked", s.yellow, s),
|
|
200
|
-
count(sm.pending, "pending", s.yellow, s),
|
|
201
|
-
];
|
|
202
|
-
const ok = sm.failed === 0 && sm.blocked === 0;
|
|
203
|
-
const head = ok ? s.green("✔ done") : s.red("✖ finished with issues");
|
|
204
|
-
return `${head} ${parts.join(s.dim(" · "))} ${s.dim(`(${fmtDuration(event.durationMs)})`)}`;
|
|
182
|
+
const head = summaryHead(event.summary, s, true);
|
|
183
|
+
return `${head} ${summaryParts(event.summary, s).join(s.dim(" · "))} ${s.dim(`(${fmtDuration(event.durationMs)})`)}`;
|
|
205
184
|
}
|
|
206
185
|
case "dry_run":
|
|
207
186
|
// Plain, uncolored, pipeable plan — one task id per line.
|
|
@@ -210,6 +189,70 @@ function formatHuman(event, s) {
|
|
|
210
189
|
return s.dim(`• ${event.message}`);
|
|
211
190
|
}
|
|
212
191
|
}
|
|
192
|
+
// Shared content builders. Both renderers — the plain line writer and the clack
|
|
193
|
+
// pretty renderer — produce the same message bodies; only the framing (gutters,
|
|
194
|
+
// spinners, boxes) differs. Keeping the text in one place means a field added to
|
|
195
|
+
// an event is formatted once, not twice.
|
|
196
|
+
function runDetailLines(i, s) {
|
|
197
|
+
return [
|
|
198
|
+
`${s.dim("tasks ")} ${i.tasksPath} ${s.dim(`(${i.total} ${plural(i.total, "task")})`)}`,
|
|
199
|
+
`${s.dim("cwd ")} ${i.cwd}`,
|
|
200
|
+
`${s.dim("limits ")} ${i.maxAttempts} ${plural(i.maxAttempts, "attempt")} ${s.dim("·")} ${fmtDuration(i.timeoutMs)} timeout`,
|
|
201
|
+
];
|
|
202
|
+
}
|
|
203
|
+
function planBody(taskIds, s) {
|
|
204
|
+
return taskIds.length === 0 ? s.dim("(no tasks)") : taskIds.map((id) => s.cyan(id)).join(s.dim(" → "));
|
|
205
|
+
}
|
|
206
|
+
function sessionText(sessionId) {
|
|
207
|
+
return `session ${sessionId}`;
|
|
208
|
+
}
|
|
209
|
+
function checkText(strategy, ok) {
|
|
210
|
+
return `check ${strategy} → ${ok ? "ok" : "fail"}`;
|
|
211
|
+
}
|
|
212
|
+
function doneMessage(event, s) {
|
|
213
|
+
return `${s.bold(event.taskId)} ${s.green("done")} ${s.dim(`in ${fmtDuration(event.durationMs)} · ${event.done}/${event.total} complete`)}`;
|
|
214
|
+
}
|
|
215
|
+
function committedDetail(event, s) {
|
|
216
|
+
if (event.status === "nothing")
|
|
217
|
+
return s.dim("no changes to commit");
|
|
218
|
+
const sha = event.commit ? `${s.cyan(event.commit)} ` : "";
|
|
219
|
+
return `${s.dim("committed")} ${sha}${s.dim(event.message)}`;
|
|
220
|
+
}
|
|
221
|
+
function failureTail(event, s) {
|
|
222
|
+
return event.willRetry
|
|
223
|
+
? s.yellow(`will retry (attempt ${event.attempt}/${event.maxAttempts})`)
|
|
224
|
+
: s.red(`gave up after ${event.attempt} ${plural(event.attempt, "attempt")}`);
|
|
225
|
+
}
|
|
226
|
+
function failureMessage(event, s) {
|
|
227
|
+
return `${s.bold(event.taskId)} ${s.red("failed")} ${s.dim(`in ${fmtDuration(event.durationMs)}`)}: ${event.error} ${failureTail(event, s)}`;
|
|
228
|
+
}
|
|
229
|
+
function blockedSuffix(event, s) {
|
|
230
|
+
return event.blockedBy ? s.dim(` (by ${event.blockedBy})`) : "";
|
|
231
|
+
}
|
|
232
|
+
function pausedMessage(event, s) {
|
|
233
|
+
const resume = event.resumeAfterMs !== undefined ? s.dim(` · retry after ${fmtDuration(event.resumeAfterMs)}`) : "";
|
|
234
|
+
return `${s.bold(event.taskId)} ${s.yellow("paused")} ${s.dim(event.reason)}${resume} ${s.dim("— re-run to resume")}`;
|
|
235
|
+
}
|
|
236
|
+
function summaryParts(sm, s) {
|
|
237
|
+
return [
|
|
238
|
+
s.green(`${sm.done} done`),
|
|
239
|
+
count(sm.failed, "failed", s.red, s),
|
|
240
|
+
count(sm.blocked, "blocked", s.yellow, s),
|
|
241
|
+
count(sm.pending, "pending", s.yellow, s),
|
|
242
|
+
];
|
|
243
|
+
}
|
|
244
|
+
function summaryOk(sm) {
|
|
245
|
+
return sm.failed === 0 && sm.blocked === 0;
|
|
246
|
+
}
|
|
247
|
+
// `glyph` selects whether the leading symbol is included (plain renderer) or
|
|
248
|
+
// omitted (the clack outro draws its own framing).
|
|
249
|
+
function summaryHead(sm, s, glyph) {
|
|
250
|
+
if (sm.paused)
|
|
251
|
+
return glyph ? s.yellow("⏸ paused") : s.yellow("paused");
|
|
252
|
+
if (summaryOk(sm))
|
|
253
|
+
return glyph ? s.green("✔ done") : s.green("done");
|
|
254
|
+
return glyph ? s.red("✖ finished with issues") : s.red("finished with issues");
|
|
255
|
+
}
|
|
213
256
|
function count(n, label, color, s) {
|
|
214
257
|
const text = `${n} ${label}`;
|
|
215
258
|
return n > 0 ? color(text) : s.dim(text);
|
package/dist/orchestrator.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { GitCommitter } from "./git.js";
|
|
1
2
|
import { type SuccessFs } from "./success.js";
|
|
2
3
|
import { type TaskFs } from "./tasks.js";
|
|
3
4
|
import type { Logger } from "./log.js";
|
|
@@ -30,10 +31,11 @@ export interface OrchestratorOptions {
|
|
|
30
31
|
clock?: Clock;
|
|
31
32
|
log?: Logger;
|
|
32
33
|
cwd?: string;
|
|
33
|
-
resultsDir?: string;
|
|
34
34
|
maxAttempts?: number;
|
|
35
35
|
timeoutMs?: number;
|
|
36
36
|
version?: string;
|
|
37
|
+
/** Commits the working tree after each successful task. Omit to skip committing entirely. */
|
|
38
|
+
git?: GitCommitter;
|
|
37
39
|
}
|
|
38
40
|
export interface Summary {
|
|
39
41
|
total: number;
|
|
@@ -42,6 +44,9 @@ export interface Summary {
|
|
|
42
44
|
done: number;
|
|
43
45
|
failed: number;
|
|
44
46
|
blocked: number;
|
|
47
|
+
/** Set only when the run halted on a quota/rate-limit wall; otherwise absent. */
|
|
48
|
+
paused?: boolean;
|
|
49
|
+
/** Why the run paused — present iff `paused`. */
|
|
50
|
+
pauseReason?: string;
|
|
45
51
|
}
|
|
46
52
|
export declare function runOrchestrator(options: OrchestratorOptions): Promise<Summary>;
|
|
47
|
-
export declare function defaultResultsDir(cwd: string): string;
|
package/dist/orchestrator.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import fs from "node:fs/promises";
|
|
2
|
-
import path from "node:path";
|
|
3
2
|
import { dependencyState, topologicalOrder } from "./deps.js";
|
|
3
|
+
import { classifyQuotaError, RunPausedError } from "./pause.js";
|
|
4
4
|
import { checkSuccess } from "./success.js";
|
|
5
5
|
import { loadTasks, saveTasks } from "./tasks.js";
|
|
6
6
|
export const DEFAULT_TIMEOUT_MS = 10 * 60 * 1000;
|
|
@@ -27,7 +27,6 @@ export async function runOrchestrator(options) {
|
|
|
27
27
|
maxAttempts: options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS,
|
|
28
28
|
timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
|
29
29
|
cwd,
|
|
30
|
-
resultsDir: options.resultsDir ?? defaultResultsDir(cwd),
|
|
31
30
|
},
|
|
32
31
|
});
|
|
33
32
|
// The plan is a courtesy preview; a dependency cycle is handled gracefully
|
|
@@ -40,35 +39,58 @@ export async function runOrchestrator(options) {
|
|
|
40
39
|
// Cycle or missing dependency — the loop reports it as a deadlock/block.
|
|
41
40
|
}
|
|
42
41
|
}
|
|
43
|
-
|
|
42
|
+
let paused;
|
|
43
|
+
loop: while (true) {
|
|
44
44
|
let progressed = false;
|
|
45
45
|
for (const task of taskFile.tasks) {
|
|
46
46
|
if (!isRunnable(task, options.maxAttempts))
|
|
47
47
|
continue;
|
|
48
48
|
const state = dependencyState(task, taskFile.tasks);
|
|
49
|
-
if (state === "waiting")
|
|
49
|
+
if (state.state === "waiting")
|
|
50
50
|
continue;
|
|
51
|
-
if (state === "blocked") {
|
|
51
|
+
if (state.state === "blocked") {
|
|
52
52
|
task.status = "blocked";
|
|
53
53
|
task.error = "Dependency failed, blocked, or missing.";
|
|
54
|
-
task.finishedAt = nowIso(
|
|
54
|
+
task.finishedAt = nowIso(clock);
|
|
55
55
|
options.log?.event({
|
|
56
56
|
type: "task_blocked",
|
|
57
57
|
taskId: task.id,
|
|
58
58
|
reason: task.error,
|
|
59
|
-
blockedBy:
|
|
59
|
+
blockedBy: state.blockedBy,
|
|
60
60
|
});
|
|
61
61
|
await saveTasks(options.tasksPath, taskFile, queueFs);
|
|
62
62
|
progressed = true;
|
|
63
63
|
continue;
|
|
64
64
|
}
|
|
65
|
-
|
|
65
|
+
try {
|
|
66
|
+
await runTask(task, taskFile.tasks, options, queueFs);
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
// A quota/rate-limit wall halts the whole run: the task has already been
|
|
70
|
+
// reset to pending (attempt rolled back) inside runTask, so we just stop
|
|
71
|
+
// the sweep — every other ready task would hit the same wall this round.
|
|
72
|
+
if (error instanceof RunPausedError) {
|
|
73
|
+
paused = error;
|
|
74
|
+
break loop;
|
|
75
|
+
}
|
|
76
|
+
throw error;
|
|
77
|
+
}
|
|
66
78
|
progressed = true;
|
|
67
79
|
}
|
|
68
80
|
if (!progressed)
|
|
69
81
|
break;
|
|
70
82
|
}
|
|
71
83
|
const summary = summarize(taskFile.tasks);
|
|
84
|
+
if (paused) {
|
|
85
|
+
summary.paused = true;
|
|
86
|
+
summary.pauseReason = paused.message;
|
|
87
|
+
options.log?.event({
|
|
88
|
+
type: "run_paused",
|
|
89
|
+
taskId: paused.taskId,
|
|
90
|
+
reason: paused.message,
|
|
91
|
+
resumeAfterMs: paused.resumeAfterMs,
|
|
92
|
+
});
|
|
93
|
+
}
|
|
72
94
|
options.log?.event({ type: "summary", summary, durationMs: clock.now().getTime() - runStart });
|
|
73
95
|
return summary;
|
|
74
96
|
}
|
|
@@ -80,7 +102,7 @@ async function runTask(task, tasks, options, queueFs) {
|
|
|
80
102
|
task.status = "running";
|
|
81
103
|
task.attempts += 1;
|
|
82
104
|
const startMs = clock.now().getTime();
|
|
83
|
-
task.startedAt = clock
|
|
105
|
+
task.startedAt = nowIso(clock);
|
|
84
106
|
task.finishedAt = undefined;
|
|
85
107
|
task.error = undefined;
|
|
86
108
|
options.log?.event({
|
|
@@ -104,7 +126,7 @@ async function runTask(task, tasks, options, queueFs) {
|
|
|
104
126
|
if (!success)
|
|
105
127
|
throw new Error("Success check failed.");
|
|
106
128
|
task.status = "done";
|
|
107
|
-
task.finishedAt = clock
|
|
129
|
+
task.finishedAt = nowIso(clock);
|
|
108
130
|
options.log?.event({
|
|
109
131
|
type: "task_done",
|
|
110
132
|
taskId: task.id,
|
|
@@ -114,9 +136,23 @@ async function runTask(task, tasks, options, queueFs) {
|
|
|
114
136
|
});
|
|
115
137
|
}
|
|
116
138
|
catch (error) {
|
|
139
|
+
// Out of tokens is not a task defect. Roll back the attempt we charged at
|
|
140
|
+
// the top, leave the task pending (exactly as crash recovery would), persist
|
|
141
|
+
// that clean state, then unwind the run — index.ts maps the pause to a
|
|
142
|
+
// distinct exit code so re-invoking resumes once tokens return.
|
|
143
|
+
const quota = classifyQuotaError(error);
|
|
144
|
+
if (quota) {
|
|
145
|
+
task.attempts -= 1;
|
|
146
|
+
task.status = "pending";
|
|
147
|
+
task.startedAt = undefined;
|
|
148
|
+
task.finishedAt = undefined;
|
|
149
|
+
task.error = undefined;
|
|
150
|
+
await saveTasks(options.tasksPath, { tasks }, queueFs);
|
|
151
|
+
throw new RunPausedError(task.id, quota.reason, quota.resumeAfterMs);
|
|
152
|
+
}
|
|
117
153
|
task.error = error instanceof Error ? error.message : String(error);
|
|
118
154
|
task.status = task.attempts >= maxAttempts ? "failed" : "pending";
|
|
119
|
-
task.finishedAt = clock
|
|
155
|
+
task.finishedAt = nowIso(clock);
|
|
120
156
|
options.log?.event({
|
|
121
157
|
type: "task_failed",
|
|
122
158
|
taskId: task.id,
|
|
@@ -128,22 +164,42 @@ async function runTask(task, tasks, options, queueFs) {
|
|
|
128
164
|
});
|
|
129
165
|
}
|
|
130
166
|
await saveTasks(options.tasksPath, { tasks }, queueFs);
|
|
167
|
+
// Commit the work the task produced. This runs after the save above so the
|
|
168
|
+
// commit captures the task file in its final "done" state. A failed commit
|
|
169
|
+
// is reported but never flips the task back to failed — the task itself
|
|
170
|
+
// already passed its success check. Skipped when no committer is wired in.
|
|
171
|
+
if (task.status === "done" && options.git) {
|
|
172
|
+
await commitTask(task, options.git, options.log, cwd);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
async function commitTask(task, committer, log, cwd) {
|
|
176
|
+
const message = `feat: ${task.id}`;
|
|
177
|
+
try {
|
|
178
|
+
const result = await committer.commit({ cwd, message });
|
|
179
|
+
if (result.status === "error") {
|
|
180
|
+
log?.event({ type: "task_commit_failed", taskId: task.id, message, detail: result.detail ?? "" });
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
log?.event({
|
|
184
|
+
type: "task_committed",
|
|
185
|
+
taskId: task.id,
|
|
186
|
+
message,
|
|
187
|
+
status: result.status,
|
|
188
|
+
commit: result.commit,
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
catch (error) {
|
|
192
|
+
log?.event({
|
|
193
|
+
type: "task_commit_failed",
|
|
194
|
+
taskId: task.id,
|
|
195
|
+
message,
|
|
196
|
+
detail: error instanceof Error ? error.message : String(error),
|
|
197
|
+
});
|
|
198
|
+
}
|
|
131
199
|
}
|
|
132
200
|
function countDone(tasks) {
|
|
133
201
|
return tasks.reduce((total, task) => (task.status === "done" ? total + 1 : total), 0);
|
|
134
202
|
}
|
|
135
|
-
// Names the first dependency that is failed, blocked, or missing, for telemetry.
|
|
136
|
-
function blockingDependency(task, tasks) {
|
|
137
|
-
const byId = new Map(tasks.map((candidate) => [candidate.id, candidate]));
|
|
138
|
-
for (const depId of task.dependsOn) {
|
|
139
|
-
const dep = byId.get(depId);
|
|
140
|
-
if (!dep)
|
|
141
|
-
return `${depId} (missing)`;
|
|
142
|
-
if (dep.status === "failed" || dep.status === "blocked")
|
|
143
|
-
return `${depId} (${dep.status})`;
|
|
144
|
-
}
|
|
145
|
-
return undefined;
|
|
146
|
-
}
|
|
147
203
|
function isRunnable(task, globalMaxAttempts = DEFAULT_MAX_ATTEMPTS) {
|
|
148
204
|
if (task.status === "pending")
|
|
149
205
|
return true;
|
|
@@ -185,6 +241,3 @@ function summarize(tasks) {
|
|
|
185
241
|
function nowIso(clock = defaultClock) {
|
|
186
242
|
return clock.now().toISOString();
|
|
187
243
|
}
|
|
188
|
-
export function defaultResultsDir(cwd) {
|
|
189
|
-
return path.join(cwd, ".queue", "results");
|
|
190
|
-
}
|
package/dist/pause.d.ts
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pause/resume support for the orchestrator.
|
|
3
|
+
*
|
|
4
|
+
* Running out of provider tokens is a property of the *run*, not the *task*:
|
|
5
|
+
* nothing is wrong with the task that hit the wall, and every other ready task
|
|
6
|
+
* would hit the same wall this sweep. So instead of letting a quota error burn
|
|
7
|
+
* the task's attempt budget and cascade into `blocked` dependents, we classify
|
|
8
|
+
* it, leave the task resumable, and unwind the whole run with `RunPausedError`.
|
|
9
|
+
* The persisted state (running -> pending, atomic save after every transition)
|
|
10
|
+
* is already the resume substrate — re-invoking the CLI just picks up where it
|
|
11
|
+
* left off once tokens are back.
|
|
12
|
+
*/
|
|
13
|
+
/**
|
|
14
|
+
* Thrown out of a task to halt the whole run cleanly. The task it names has
|
|
15
|
+
* been reset to `pending` (its attempt rolled back), so no failure is recorded.
|
|
16
|
+
*/
|
|
17
|
+
export declare class RunPausedError extends Error {
|
|
18
|
+
readonly taskId: string;
|
|
19
|
+
/** Wall-clock ms the provider asked us to wait, when it told us (Retry-After). */
|
|
20
|
+
readonly resumeAfterMs?: number;
|
|
21
|
+
constructor(taskId: string, reason: string, resumeAfterMs?: number);
|
|
22
|
+
}
|
|
23
|
+
export interface QuotaSignal {
|
|
24
|
+
/** Human-readable reason, surfaced in logs and on the paused exit. */
|
|
25
|
+
reason: string;
|
|
26
|
+
/** Suggested wait before resuming, derived from a Retry-After header. */
|
|
27
|
+
resumeAfterMs?: number;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Inspects an unknown thrown value for a quota / rate-limit / overload signal.
|
|
31
|
+
* Returns a `QuotaSignal` when the run should pause, or `null` when the error
|
|
32
|
+
* is a genuine task failure that should follow the normal failure path.
|
|
33
|
+
*/
|
|
34
|
+
export declare function classifyQuotaError(error: unknown): QuotaSignal | null;
|
package/dist/pause.js
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pause/resume support for the orchestrator.
|
|
3
|
+
*
|
|
4
|
+
* Running out of provider tokens is a property of the *run*, not the *task*:
|
|
5
|
+
* nothing is wrong with the task that hit the wall, and every other ready task
|
|
6
|
+
* would hit the same wall this sweep. So instead of letting a quota error burn
|
|
7
|
+
* the task's attempt budget and cascade into `blocked` dependents, we classify
|
|
8
|
+
* it, leave the task resumable, and unwind the whole run with `RunPausedError`.
|
|
9
|
+
* The persisted state (running -> pending, atomic save after every transition)
|
|
10
|
+
* is already the resume substrate — re-invoking the CLI just picks up where it
|
|
11
|
+
* left off once tokens are back.
|
|
12
|
+
*/
|
|
13
|
+
/**
|
|
14
|
+
* Thrown out of a task to halt the whole run cleanly. The task it names has
|
|
15
|
+
* been reset to `pending` (its attempt rolled back), so no failure is recorded.
|
|
16
|
+
*/
|
|
17
|
+
export class RunPausedError extends Error {
|
|
18
|
+
taskId;
|
|
19
|
+
/** Wall-clock ms the provider asked us to wait, when it told us (Retry-After). */
|
|
20
|
+
resumeAfterMs;
|
|
21
|
+
constructor(taskId, reason, resumeAfterMs) {
|
|
22
|
+
super(reason);
|
|
23
|
+
this.name = "RunPausedError";
|
|
24
|
+
this.taskId = taskId;
|
|
25
|
+
this.resumeAfterMs = resumeAfterMs;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
// HTTP statuses that mean "retryable later, not a task defect":
|
|
29
|
+
// 429 too many requests (rate limit) 402 payment required (out of credit)
|
|
30
|
+
// 529 overloaded (Anthropic) 503 service unavailable
|
|
31
|
+
const QUOTA_STATUSES = new Set([402, 429, 503, 529]);
|
|
32
|
+
// Message fragments that signal a quota / rate / availability wall even when no
|
|
33
|
+
// numeric status is exposed. Kept deliberately broad — a false positive only
|
|
34
|
+
// pauses (recoverable) where a false negative would burn the retry budget.
|
|
35
|
+
const QUOTA_PATTERNS = [
|
|
36
|
+
/\brate.?limit/i,
|
|
37
|
+
/too many requests/i,
|
|
38
|
+
/\bquota\b/i,
|
|
39
|
+
/insufficient\s+(?:credit|quota|balance|funds?)/i,
|
|
40
|
+
/out of (?:credit|tokens?)/i,
|
|
41
|
+
/payment required/i,
|
|
42
|
+
/overloaded/i,
|
|
43
|
+
/\b(?:429|402|529)\b/,
|
|
44
|
+
];
|
|
45
|
+
/**
|
|
46
|
+
* Inspects an unknown thrown value for a quota / rate-limit / overload signal.
|
|
47
|
+
* Returns a `QuotaSignal` when the run should pause, or `null` when the error
|
|
48
|
+
* is a genuine task failure that should follow the normal failure path.
|
|
49
|
+
*/
|
|
50
|
+
export function classifyQuotaError(error) {
|
|
51
|
+
const status = extractStatus(error);
|
|
52
|
+
if (status !== undefined && QUOTA_STATUSES.has(status)) {
|
|
53
|
+
return { reason: reasonForStatus(status), resumeAfterMs: extractRetryAfterMs(error) };
|
|
54
|
+
}
|
|
55
|
+
const message = extractMessage(error);
|
|
56
|
+
if (message && QUOTA_PATTERNS.some((pattern) => pattern.test(message))) {
|
|
57
|
+
return { reason: message, resumeAfterMs: extractRetryAfterMs(error) };
|
|
58
|
+
}
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
function reasonForStatus(status) {
|
|
62
|
+
switch (status) {
|
|
63
|
+
case 402:
|
|
64
|
+
return "Provider returned 402 (payment required / out of credit).";
|
|
65
|
+
case 429:
|
|
66
|
+
return "Provider returned 429 (rate limit / quota exceeded).";
|
|
67
|
+
case 503:
|
|
68
|
+
return "Provider returned 503 (service unavailable).";
|
|
69
|
+
case 529:
|
|
70
|
+
return "Provider returned 529 (overloaded).";
|
|
71
|
+
default:
|
|
72
|
+
return `Provider returned ${status}.`;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
// Provider/SDK errors expose the status under several names and nesting depths,
|
|
76
|
+
// so probe the common shapes rather than assuming one.
|
|
77
|
+
function extractStatus(error) {
|
|
78
|
+
if (!error || typeof error !== "object")
|
|
79
|
+
return undefined;
|
|
80
|
+
const candidates = [
|
|
81
|
+
error.status,
|
|
82
|
+
error.statusCode,
|
|
83
|
+
asObject(error.response)?.status,
|
|
84
|
+
asObject(error.error)?.status,
|
|
85
|
+
asObject(error.cause)?.status,
|
|
86
|
+
];
|
|
87
|
+
for (const candidate of candidates) {
|
|
88
|
+
const n = typeof candidate === "string" ? Number(candidate) : candidate;
|
|
89
|
+
if (typeof n === "number" && Number.isInteger(n))
|
|
90
|
+
return n;
|
|
91
|
+
}
|
|
92
|
+
return undefined;
|
|
93
|
+
}
|
|
94
|
+
function extractMessage(error) {
|
|
95
|
+
if (error instanceof Error)
|
|
96
|
+
return error.message;
|
|
97
|
+
if (typeof error === "string")
|
|
98
|
+
return error;
|
|
99
|
+
if (error && typeof error === "object") {
|
|
100
|
+
const message = error.message;
|
|
101
|
+
if (typeof message === "string")
|
|
102
|
+
return message;
|
|
103
|
+
}
|
|
104
|
+
return "";
|
|
105
|
+
}
|
|
106
|
+
// Retry-After is seconds (or, per RFC, an HTTP date — we only honour the numeric
|
|
107
|
+
// form). Look in the header bags the common SDK error shapes carry.
|
|
108
|
+
function extractRetryAfterMs(error) {
|
|
109
|
+
const headers = asObject(error?.headers) ??
|
|
110
|
+
asObject(asObject(error?.response)?.headers);
|
|
111
|
+
if (!headers)
|
|
112
|
+
return undefined;
|
|
113
|
+
const raw = headers["retry-after"] ?? headers["Retry-After"];
|
|
114
|
+
const seconds = typeof raw === "string" ? Number(raw) : typeof raw === "number" ? raw : NaN;
|
|
115
|
+
return Number.isFinite(seconds) && seconds >= 0 ? Math.round(seconds * 1000) : undefined;
|
|
116
|
+
}
|
|
117
|
+
function asObject(value) {
|
|
118
|
+
return value && typeof value === "object" ? value : undefined;
|
|
119
|
+
}
|
package/dist/schema.d.ts
CHANGED
|
@@ -34,7 +34,6 @@ export declare const taskSchema: z.ZodObject<{
|
|
|
34
34
|
maxAttempts: z.ZodOptional<z.ZodNumber>;
|
|
35
35
|
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
36
36
|
cwd: z.ZodOptional<z.ZodString>;
|
|
37
|
-
resultPath: z.ZodOptional<z.ZodString>;
|
|
38
37
|
success: z.ZodOptional<z.ZodDefault<z.ZodDiscriminatedUnion<"strategy", [z.ZodObject<{
|
|
39
38
|
strategy: z.ZodLiteral<"file_exists">;
|
|
40
39
|
path: z.ZodString;
|
|
@@ -74,7 +73,6 @@ export declare const taskSchema: z.ZodObject<{
|
|
|
74
73
|
maxAttempts?: number | undefined;
|
|
75
74
|
timeoutMs?: number | undefined;
|
|
76
75
|
cwd?: string | undefined;
|
|
77
|
-
resultPath?: string | undefined;
|
|
78
76
|
success?: {
|
|
79
77
|
strategy: "file_exists";
|
|
80
78
|
path: string;
|
|
@@ -96,7 +94,6 @@ export declare const taskSchema: z.ZodObject<{
|
|
|
96
94
|
maxAttempts?: number | undefined;
|
|
97
95
|
timeoutMs?: number | undefined;
|
|
98
96
|
cwd?: string | undefined;
|
|
99
|
-
resultPath?: string | undefined;
|
|
100
97
|
success?: {
|
|
101
98
|
strategy: "file_exists";
|
|
102
99
|
path: string;
|
|
@@ -121,7 +118,6 @@ export declare const taskFileSchema: z.ZodUnion<[z.ZodObject<{
|
|
|
121
118
|
maxAttempts: z.ZodOptional<z.ZodNumber>;
|
|
122
119
|
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
123
120
|
cwd: z.ZodOptional<z.ZodString>;
|
|
124
|
-
resultPath: z.ZodOptional<z.ZodString>;
|
|
125
121
|
success: z.ZodOptional<z.ZodDefault<z.ZodDiscriminatedUnion<"strategy", [z.ZodObject<{
|
|
126
122
|
strategy: z.ZodLiteral<"file_exists">;
|
|
127
123
|
path: z.ZodString;
|
|
@@ -161,7 +157,6 @@ export declare const taskFileSchema: z.ZodUnion<[z.ZodObject<{
|
|
|
161
157
|
maxAttempts?: number | undefined;
|
|
162
158
|
timeoutMs?: number | undefined;
|
|
163
159
|
cwd?: string | undefined;
|
|
164
|
-
resultPath?: string | undefined;
|
|
165
160
|
success?: {
|
|
166
161
|
strategy: "file_exists";
|
|
167
162
|
path: string;
|
|
@@ -183,7 +178,6 @@ export declare const taskFileSchema: z.ZodUnion<[z.ZodObject<{
|
|
|
183
178
|
maxAttempts?: number | undefined;
|
|
184
179
|
timeoutMs?: number | undefined;
|
|
185
180
|
cwd?: string | undefined;
|
|
186
|
-
resultPath?: string | undefined;
|
|
187
181
|
success?: {
|
|
188
182
|
strategy: "file_exists";
|
|
189
183
|
path: string;
|
|
@@ -209,7 +203,6 @@ export declare const taskFileSchema: z.ZodUnion<[z.ZodObject<{
|
|
|
209
203
|
maxAttempts?: number | undefined;
|
|
210
204
|
timeoutMs?: number | undefined;
|
|
211
205
|
cwd?: string | undefined;
|
|
212
|
-
resultPath?: string | undefined;
|
|
213
206
|
success?: {
|
|
214
207
|
strategy: "file_exists";
|
|
215
208
|
path: string;
|
|
@@ -233,7 +226,6 @@ export declare const taskFileSchema: z.ZodUnion<[z.ZodObject<{
|
|
|
233
226
|
maxAttempts?: number | undefined;
|
|
234
227
|
timeoutMs?: number | undefined;
|
|
235
228
|
cwd?: string | undefined;
|
|
236
|
-
resultPath?: string | undefined;
|
|
237
229
|
success?: {
|
|
238
230
|
strategy: "file_exists";
|
|
239
231
|
path: string;
|
|
@@ -257,7 +249,6 @@ export declare const taskFileSchema: z.ZodUnion<[z.ZodObject<{
|
|
|
257
249
|
maxAttempts: z.ZodOptional<z.ZodNumber>;
|
|
258
250
|
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
259
251
|
cwd: z.ZodOptional<z.ZodString>;
|
|
260
|
-
resultPath: z.ZodOptional<z.ZodString>;
|
|
261
252
|
success: z.ZodOptional<z.ZodDefault<z.ZodDiscriminatedUnion<"strategy", [z.ZodObject<{
|
|
262
253
|
strategy: z.ZodLiteral<"file_exists">;
|
|
263
254
|
path: z.ZodString;
|
|
@@ -297,7 +288,6 @@ export declare const taskFileSchema: z.ZodUnion<[z.ZodObject<{
|
|
|
297
288
|
maxAttempts?: number | undefined;
|
|
298
289
|
timeoutMs?: number | undefined;
|
|
299
290
|
cwd?: string | undefined;
|
|
300
|
-
resultPath?: string | undefined;
|
|
301
291
|
success?: {
|
|
302
292
|
strategy: "file_exists";
|
|
303
293
|
path: string;
|
|
@@ -319,7 +309,6 @@ export declare const taskFileSchema: z.ZodUnion<[z.ZodObject<{
|
|
|
319
309
|
maxAttempts?: number | undefined;
|
|
320
310
|
timeoutMs?: number | undefined;
|
|
321
311
|
cwd?: string | undefined;
|
|
322
|
-
resultPath?: string | undefined;
|
|
323
312
|
success?: {
|
|
324
313
|
strategy: "file_exists";
|
|
325
314
|
path: string;
|
|
@@ -344,7 +333,6 @@ export declare const taskFileSchema: z.ZodUnion<[z.ZodObject<{
|
|
|
344
333
|
maxAttempts?: number | undefined;
|
|
345
334
|
timeoutMs?: number | undefined;
|
|
346
335
|
cwd?: string | undefined;
|
|
347
|
-
resultPath?: string | undefined;
|
|
348
336
|
success?: {
|
|
349
337
|
strategy: "file_exists";
|
|
350
338
|
path: string;
|
|
@@ -367,7 +355,6 @@ export declare const taskFileSchema: z.ZodUnion<[z.ZodObject<{
|
|
|
367
355
|
maxAttempts?: number | undefined;
|
|
368
356
|
timeoutMs?: number | undefined;
|
|
369
357
|
cwd?: string | undefined;
|
|
370
|
-
resultPath?: string | undefined;
|
|
371
358
|
success?: {
|
|
372
359
|
strategy: "file_exists";
|
|
373
360
|
path: string;
|
package/dist/schema.js
CHANGED
|
@@ -24,7 +24,6 @@ export const taskSchema = z.object({
|
|
|
24
24
|
maxAttempts: z.number().int().min(1).optional(),
|
|
25
25
|
timeoutMs: z.number().int().positive().optional(),
|
|
26
26
|
cwd: z.string().min(1).optional(),
|
|
27
|
-
resultPath: z.string().min(1).optional(),
|
|
28
27
|
success: successCheckSchema.optional(),
|
|
29
28
|
recoveryNotes: z.array(z.string()).default([]),
|
|
30
29
|
error: z.string().optional(),
|
package/dist/server.js
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
// Integration-test harness for standing up a real `opencode serve` at a known
|
|
2
|
+
// port. Production does not use this: createOpenCodeClient (client.ts) lets the
|
|
3
|
+
// SDK's createOpencode spawn its own ephemeral server. These helpers exist so the
|
|
4
|
+
// integration test can connect a client to an explicit baseUrl.
|
|
1
5
|
import { spawn } from "node:child_process";
|
|
2
6
|
import net from "node:net";
|
|
3
7
|
export async function startServer(options) {
|
package/package.json
CHANGED