squally-reporter 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +6 -1
- package/dist/cli.js.map +1 -1
- package/dist/http-config.d.ts.map +1 -1
- package/dist/http-config.js +3 -2
- package/dist/http-config.js.map +1 -1
- package/dist/index.d.ts +77 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +329 -104
- package/dist/index.js.map +1 -1
- package/dist/squally-response.d.ts +11 -0
- package/dist/squally-response.d.ts.map +1 -1
- package/dist/squally-response.js +38 -0
- package/dist/squally-response.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -143,7 +143,7 @@ const FLUSH_INTERVAL_MS = 2000;
|
|
|
143
143
|
// CAPPED, and the cap is about crash exposure rather than rate. Everything
|
|
144
144
|
// buffered is lost if the runner is killed (CI cancellation, OOM), so the
|
|
145
145
|
// interval is also the size of the window at risk. 30 s is the same figure
|
|
146
|
-
//
|
|
146
|
+
// TRANSIENT_BACKOFF_CAP_MS uses for "the longest we ever wait"; it is reached
|
|
147
147
|
// at 75 shards, and the aggregate stays inside the rule up to 150 shards
|
|
148
148
|
// (S x 60/30 <= 300) - far beyond the 4-to-16 the app's shard documentation
|
|
149
149
|
// describes. Past that, the retry path handles a 429, and any batch lost to it
|
|
@@ -165,17 +165,45 @@ function flushIntervalFor(shardTotal) {
|
|
|
165
165
|
const shards = Math.max(1, shardTotal ?? 1);
|
|
166
166
|
return Math.min(FLUSH_INTERVAL_CAP_MS, Math.max(FLUSH_INTERVAL_MS, SHARD_INTERVAL_MS * shards));
|
|
167
167
|
}
|
|
168
|
-
// --- Retry policy (spec section 3.2)
|
|
168
|
+
// --- Retry policy (spec section 3.2; one budget since 0.17.0) ---------------
|
|
169
169
|
//
|
|
170
|
-
//
|
|
171
|
-
//
|
|
172
|
-
//
|
|
173
|
-
//
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
170
|
+
// TRANSIENT, and retried: a network error (a timeout included), a 5xx, a 429,
|
|
171
|
+
// and a 403 that Vercel's firewall marks `x-vercel-mitigated: challenge` or
|
|
172
|
+
// `deny` (edgeMitigation). Until 0.16.0 a 429 got five tries and a 5xx or a
|
|
173
|
+
// network error one retry - and a 403 challenge none, so a challenge on begin
|
|
174
|
+
// cost the whole run (measured against app.squally.dev, 30.10.2026). The
|
|
175
|
+
// firewall's windows are 60 s, which neither old budget outlasted.
|
|
176
|
+
//
|
|
177
|
+
// ONE BUDGET PER REQUEST, in time rather than tries: waits of 1 s doubling to
|
|
178
|
+
// 30 s, each jittered (half fixed, half random, so shards that were refused
|
|
179
|
+
// together do not come back together), for as long as the next wait still
|
|
180
|
+
// ends within TRANSIENT_REQUEST_BUDGET_MS of the first try. Retry-After, when
|
|
181
|
+
// sent, replaces the computed wait, capped the same way.
|
|
182
|
+
//
|
|
183
|
+
// FINAL, and never retried: Squally's own JSON error (every 4xx it sends), a
|
|
184
|
+
// 403 without the header, a redirect, a 2xx that is not JSON. The caller
|
|
185
|
+
// decides what each means for its route.
|
|
186
|
+
const TRANSIENT_BACKOFF_BASE_MS = 1000;
|
|
187
|
+
const TRANSIENT_BACKOFF_CAP_MS = 30000;
|
|
188
|
+
const TRANSIENT_REQUEST_BUDGET_MS = 90000;
|
|
189
|
+
// At onEnd, after a transient error in this run: how long the drain and
|
|
190
|
+
// /complete may still take, from the start of onEnd. Only then - a run that
|
|
191
|
+
// met none keeps DRAIN_BUDGET_MS, so a healthy run never waits longer.
|
|
192
|
+
const END_BUDGET_AFTER_TRANSIENT_MS = 60000;
|
|
193
|
+
/**
|
|
194
|
+
* FOR THIS PACKAGE'S OWN TESTS: SQUALLY_INTERNAL_TIME_SCALE, between 0 and 1,
|
|
195
|
+
* shortens the transient waits and budgets above (not DRAIN_BUDGET_MS) so the
|
|
196
|
+
* unit and exit-code tests run them in seconds. It can only shorten, never
|
|
197
|
+
* lengthen; anything else reads as 1. Deliberately not in the README.
|
|
198
|
+
*/
|
|
199
|
+
function transientTimeScale() {
|
|
200
|
+
const value = Number(process.env.SQUALLY_INTERNAL_TIME_SCALE);
|
|
201
|
+
return Number.isFinite(value) && value > 0 && value < 1 ? value : 1;
|
|
202
|
+
}
|
|
203
|
+
/** Half the wait fixed, half random - the backoff stays between ms/2 and ms. */
|
|
204
|
+
function withJitter(ms, random = Math.random) {
|
|
205
|
+
return ms / 2 + random() * (ms / 2);
|
|
206
|
+
}
|
|
179
207
|
// (decision, spec section 3.2 closes 8.7) Drain budget at onEnd: 5 seconds,
|
|
180
208
|
// ON TOP OF EXIT_DRAIN_MS below. At the flush cadence above, at most one or
|
|
181
209
|
// two batches are ever in flight when onEnd fires; 5 s comfortably covers a
|
|
@@ -2049,7 +2077,52 @@ function parseRetryAfterMs(headerValue, nowMs) {
|
|
|
2049
2077
|
}
|
|
2050
2078
|
// retryNumber is 1-indexed (the first retry passes 1, not 0).
|
|
2051
2079
|
function computeBackoffMs(retryNumber) {
|
|
2052
|
-
return Math.min(
|
|
2080
|
+
return Math.min(TRANSIENT_BACKOFF_CAP_MS, TRANSIENT_BACKOFF_BASE_MS * 2 ** (retryNumber - 1));
|
|
2081
|
+
}
|
|
2082
|
+
const NOT_AFFECTED = "Your test run and its exit code are not affected.";
|
|
2083
|
+
const INCOMPLETE_LATER = "It will show as incomplete once the project's run timeout has passed.";
|
|
2084
|
+
function resultCount(n) {
|
|
2085
|
+
return `${n} result${n === 1 ? "" : "s"}`;
|
|
2086
|
+
}
|
|
2087
|
+
/**
|
|
2088
|
+
* The end-of-run line (0.17.0), or null when there is nothing to say. Pure,
|
|
2089
|
+
* for the tests; printDeliverySummary feeds it.
|
|
2090
|
+
*
|
|
2091
|
+
* `cause` is the last transient failure, already a phrase ("Vercel's firewall
|
|
2092
|
+
* ... challenged this runner (HTTP 403, ...)"). Without one, the loss came
|
|
2093
|
+
* from a refusal that printed its own line (a quota, a cancel), and the
|
|
2094
|
+
* summary points there instead of guessing.
|
|
2095
|
+
*/
|
|
2096
|
+
function deliverySummary(input) {
|
|
2097
|
+
const prefix = "[squally-reporter]";
|
|
2098
|
+
if (!input.runStarted) {
|
|
2099
|
+
const why = input.cause !== null ? `${input.cause}, and the retries ran out` : "no answer in time";
|
|
2100
|
+
const none = input.produced === 0 ? "Nothing was sent" : `None of its ${resultCount(input.produced)} were sent`;
|
|
2101
|
+
return `${prefix} Squally did not start this run - ${why}. ${none}. ${NOT_AFFECTED}`;
|
|
2102
|
+
}
|
|
2103
|
+
const missing = Math.max(0, input.produced - input.delivered);
|
|
2104
|
+
const why = input.cause !== null
|
|
2105
|
+
? `${input.cause}, and the retries ran out`
|
|
2106
|
+
: input.lossExplained
|
|
2107
|
+
? "see the warning above"
|
|
2108
|
+
: "they were still being sent when the run ended";
|
|
2109
|
+
if (missing > 0 && input.completeFailure === null) {
|
|
2110
|
+
return (`${prefix} ${missing} of ${resultCount(input.produced)} did not reach Squally - ${why}. ` +
|
|
2111
|
+
`${NOT_AFFECTED} The run page in Squally shows how many are missing.`);
|
|
2112
|
+
}
|
|
2113
|
+
if (missing > 0) {
|
|
2114
|
+
// The same cause twice in one line reads as two problems.
|
|
2115
|
+
const completeWhy = input.completeFailure === input.cause ? "either" : `(${input.completeFailure})`;
|
|
2116
|
+
return (`${prefix} ${missing} of ${resultCount(input.produced)} did not reach Squally - ${why} - and the run ` +
|
|
2117
|
+
`could not be marked complete ${completeWhy}. ${INCOMPLETE_LATER} ${NOT_AFFECTED}`);
|
|
2118
|
+
}
|
|
2119
|
+
if (input.completeFailure !== null) {
|
|
2120
|
+
const arrived = input.produced === 0 ? "" : `All ${resultCount(input.produced)} reached Squally, but `;
|
|
2121
|
+
const subject = arrived ? "the run" : "The run";
|
|
2122
|
+
return (`${prefix} ${arrived}${subject} could not be marked complete on Squally (${input.completeFailure}). ` +
|
|
2123
|
+
`${INCOMPLETE_LATER} ${NOT_AFFECTED}`);
|
|
2124
|
+
}
|
|
2125
|
+
return null;
|
|
2053
2126
|
}
|
|
2054
2127
|
/** An error answer as an outcome: Squally's own keeps status, code and sentence. */
|
|
2055
2128
|
function errorOutcome(response, answer) {
|
|
@@ -2140,6 +2213,35 @@ class SquallyReporter {
|
|
|
2140
2213
|
* against its own row, so no group arithmetic is needed on either side.
|
|
2141
2214
|
*/
|
|
2142
2215
|
this.resultsProduced = 0;
|
|
2216
|
+
/**
|
|
2217
|
+
* Results in a batch Squally accepted (0.17.0) - only for the end-of-run
|
|
2218
|
+
* line: produced minus delivered is what the line says did not arrive, the
|
|
2219
|
+
* same difference the run page shows from resultsProduced.
|
|
2220
|
+
*/
|
|
2221
|
+
this.resultsDelivered = 0;
|
|
2222
|
+
// TRANSIENT FAILURES (0.17.0), see the retry policy at the top of the file.
|
|
2223
|
+
// `transientSeen` stays set for the run and gives onEnd its longer budget;
|
|
2224
|
+
// `transientActive` is cleared by the next answer that succeeds and, while
|
|
2225
|
+
// set, keeps the batches to one request at a time (triggerFlush) - a
|
|
2226
|
+
// challenged runner retrying every 2 s flush in parallel would only keep
|
|
2227
|
+
// the challenge going. `lastTransientCause` is the latest one, passed or not.
|
|
2228
|
+
this.transientSeen = false;
|
|
2229
|
+
this.transientActive = false;
|
|
2230
|
+
this.lastTransientCause = null;
|
|
2231
|
+
/**
|
|
2232
|
+
* The cause of the last request whose budget RAN OUT - what the summary
|
|
2233
|
+
* blames for lost results. Not lastTransientCause: a challenge that passed
|
|
2234
|
+
* did not lose anything, and a later quota stop must not be pinned on it.
|
|
2235
|
+
*/
|
|
2236
|
+
this.exhaustedCause = null;
|
|
2237
|
+
/** When onEnd started draining - the end budget counts from here. */
|
|
2238
|
+
this.endStartedAt = null;
|
|
2239
|
+
/** Begin ran out of its budget on a transient failure: the summary says so, nothing else did. */
|
|
2240
|
+
this.beginFailedTransient = false;
|
|
2241
|
+
/** Why /complete failed, for the summary - null while it has not (yet). */
|
|
2242
|
+
this.completeFailure = null;
|
|
2243
|
+
/** A batch refusal printed its own line - the summary points to it. */
|
|
2244
|
+
this.lossExplained = false;
|
|
2143
2245
|
// True once nothing more will ever be sent for this run - a permanent
|
|
2144
2246
|
// give-up (402/404/409 on append, or onBegin never getting a runId at
|
|
2145
2247
|
// all). onTestEnd stops collecting further results/artifacts once this is
|
|
@@ -2571,15 +2673,14 @@ class SquallyReporter {
|
|
|
2571
2673
|
return;
|
|
2572
2674
|
this.startRun();
|
|
2573
2675
|
}
|
|
2574
|
-
|
|
2575
|
-
const runId = await
|
|
2576
|
-
if (runId === TIMED_OUT) {
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
// nothing to complete without a runId.
|
|
2676
|
+
this.endStartedAt = Date.now();
|
|
2677
|
+
const runId = await this.untilEndDeadline(this.runIdPromise);
|
|
2678
|
+
if (runId === TIMED_OUT || runId === null) {
|
|
2679
|
+
// null after a permanent refusal: beginRun() has already said why, and
|
|
2680
|
+
// there is nothing to complete without a runId. A transient one, or a
|
|
2681
|
+
// begin still retrying, is the summary's to say (0.17.0).
|
|
2682
|
+
if (runId === TIMED_OUT || this.beginFailedTransient)
|
|
2683
|
+
this.printDeliverySummary(false);
|
|
2583
2684
|
return;
|
|
2584
2685
|
}
|
|
2585
2686
|
this.runId = runId;
|
|
@@ -2587,9 +2688,70 @@ class SquallyReporter {
|
|
|
2587
2688
|
this.triggerFlush();
|
|
2588
2689
|
// Result intentionally unused: on expiry we proceed to /complete anyway,
|
|
2589
2690
|
// per the decision above - a hung append must not block it.
|
|
2590
|
-
await
|
|
2691
|
+
const drained = await this.untilEndDeadline(Promise.allSettled([...this.inFlightFlushes]));
|
|
2692
|
+
// A flush skipped while another was in flight (triggerFlush, 0.17.0) left
|
|
2693
|
+
// its results to that one, which may have checked the buffer just before
|
|
2694
|
+
// they arrived - one more pass picks them up.
|
|
2695
|
+
if (drained !== TIMED_OUT && this.buffer.length > 0 && !this.sendingDisabled) {
|
|
2696
|
+
this.triggerFlush();
|
|
2697
|
+
await this.untilEndDeadline(Promise.allSettled([...this.inFlightFlushes]));
|
|
2698
|
+
}
|
|
2591
2699
|
const completion = completionFor(result.status);
|
|
2592
2700
|
await this.completeRun(runId, completion.status, completion.interrupted);
|
|
2701
|
+
this.printDeliverySummary(true);
|
|
2702
|
+
}
|
|
2703
|
+
/**
|
|
2704
|
+
* The end of the drain (0.17.0): DRAIN_BUDGET_MS from the start of onEnd,
|
|
2705
|
+
* or END_BUDGET_AFTER_TRANSIENT_MS once this run has met a transient
|
|
2706
|
+
* failure. Infinity before onEnd - while the suite runs, only each
|
|
2707
|
+
* request's own budget applies.
|
|
2708
|
+
*/
|
|
2709
|
+
endDeadline() {
|
|
2710
|
+
if (this.endStartedAt === null)
|
|
2711
|
+
return Number.POSITIVE_INFINITY;
|
|
2712
|
+
const budget = this.transientSeen
|
|
2713
|
+
? Math.max(DRAIN_BUDGET_MS, END_BUDGET_AFTER_TRANSIENT_MS * transientTimeScale())
|
|
2714
|
+
: DRAIN_BUDGET_MS;
|
|
2715
|
+
return this.endStartedAt + budget;
|
|
2716
|
+
}
|
|
2717
|
+
/**
|
|
2718
|
+
* withDeadline against endDeadline(), which can move: a transient failure
|
|
2719
|
+
* during the drain lengthens it, and the wait follows.
|
|
2720
|
+
*/
|
|
2721
|
+
async untilEndDeadline(promise) {
|
|
2722
|
+
for (;;) {
|
|
2723
|
+
const outcome = await withDeadline(promise, this.endDeadline());
|
|
2724
|
+
if (outcome !== TIMED_OUT || Date.now() >= this.endDeadline())
|
|
2725
|
+
return outcome;
|
|
2726
|
+
}
|
|
2727
|
+
}
|
|
2728
|
+
/**
|
|
2729
|
+
* ONE LINE AT THE END (0.17.0) about what did not reach Squally, instead of
|
|
2730
|
+
* a line per dropped batch: how many results, why, and - when /complete
|
|
2731
|
+
* failed too - that the run will show as incomplete once the project's run
|
|
2732
|
+
* timeout has passed (squally-app's hourly sweep, counted from the run's
|
|
2733
|
+
* last activity). Silent when everything arrived.
|
|
2734
|
+
*
|
|
2735
|
+
* Never throws: it runs inside onEnd, and a line is never worth an exit code.
|
|
2736
|
+
*/
|
|
2737
|
+
printDeliverySummary(runStarted) {
|
|
2738
|
+
try {
|
|
2739
|
+
const line = deliverySummary({
|
|
2740
|
+
runStarted,
|
|
2741
|
+
produced: this.resultsProduced,
|
|
2742
|
+
delivered: this.resultsDelivered,
|
|
2743
|
+
// A begin still retrying when onEnd gave up has not run out of its own
|
|
2744
|
+
// budget - its latest cause is the one to name.
|
|
2745
|
+
cause: runStarted ? this.exhaustedCause : (this.exhaustedCause ?? this.lastTransientCause),
|
|
2746
|
+
completeFailure: this.completeFailure,
|
|
2747
|
+
lossExplained: this.lossExplained,
|
|
2748
|
+
});
|
|
2749
|
+
if (line !== null)
|
|
2750
|
+
console.warn(line);
|
|
2751
|
+
}
|
|
2752
|
+
catch {
|
|
2753
|
+
// See above.
|
|
2754
|
+
}
|
|
2593
2755
|
}
|
|
2594
2756
|
armFlushTimer() {
|
|
2595
2757
|
if (!this.enabled || this.sendingDisabled)
|
|
@@ -2623,6 +2785,12 @@ class SquallyReporter {
|
|
|
2623
2785
|
this.stopFlushTimer();
|
|
2624
2786
|
}
|
|
2625
2787
|
triggerFlush() {
|
|
2788
|
+
// ONE AT A TIME WHILE SQUALLY IS REFUSING (0.17.0). The flush in flight
|
|
2789
|
+
// loops until the buffer is empty, so these results go with it; without
|
|
2790
|
+
// this, every 2 s timer tick would start another request retrying into
|
|
2791
|
+
// the same firewall. Normal runs are untouched - parallel as before.
|
|
2792
|
+
if (this.transientActive && this.inFlightFlushes.size > 0)
|
|
2793
|
+
return;
|
|
2626
2794
|
const promise = this.flush().catch((err) => {
|
|
2627
2795
|
console.warn("[squally-reporter] Unexpected error while sending results:", err);
|
|
2628
2796
|
});
|
|
@@ -2654,17 +2822,26 @@ class SquallyReporter {
|
|
|
2654
2822
|
// 402 (quota) - warn once, stop sending for this run.
|
|
2655
2823
|
// 404 (unknown runId) - warn, stop sending for this run.
|
|
2656
2824
|
// 409 (already complete) - warn, stop sending for this run.
|
|
2657
|
-
// 5xx
|
|
2658
|
-
//
|
|
2659
|
-
//
|
|
2660
|
-
//
|
|
2661
|
-
//
|
|
2662
|
-
//
|
|
2663
|
-
//
|
|
2825
|
+
// transient - 5xx, 429, network error, Vercel's 403
|
|
2826
|
+
// challenge: retried inside requestJson()
|
|
2827
|
+
// until its budget runs out; then drop this
|
|
2828
|
+
// batch and keep going. A transient server
|
|
2829
|
+
// hiccup should not permanently disable
|
|
2830
|
+
// reporting for the rest of the run. No line
|
|
2831
|
+
// per batch since 0.17.0 - the end-of-run
|
|
2832
|
+
// summary counts them all once.
|
|
2664
2833
|
async sendBatch(runId, batch) {
|
|
2665
2834
|
const outcome = await this.requestJson("POST", `${this.apiUrl}/api/ingest/v2/runs/${runId}/results`, { results: batch });
|
|
2666
|
-
if (outcome.kind === "ok")
|
|
2835
|
+
if (outcome.kind === "ok") {
|
|
2836
|
+
this.resultsDelivered += batch.length;
|
|
2667
2837
|
return;
|
|
2838
|
+
}
|
|
2839
|
+
if (outcome.kind !== "parse_error" && outcome.transient) {
|
|
2840
|
+
debugLog(`dropped a batch of ${batch.length} result(s) after its retries: ${this.outcomeDetail(outcome)}`);
|
|
2841
|
+
return;
|
|
2842
|
+
}
|
|
2843
|
+
// Every branch below prints its own line; the summary then points to it.
|
|
2844
|
+
this.lossExplained = true;
|
|
2668
2845
|
if (outcome.kind === "http_error") {
|
|
2669
2846
|
switch (outcome.status) {
|
|
2670
2847
|
case 400:
|
|
@@ -2821,6 +2998,15 @@ class SquallyReporter {
|
|
|
2821
2998
|
console.warn(line);
|
|
2822
2999
|
return outcome.json.runId;
|
|
2823
3000
|
}
|
|
3001
|
+
// TRANSIENT TO THE END (0.17.0): retried for the whole request budget
|
|
3002
|
+
// while results stayed buffered. Not a refusal, so not the refusal's
|
|
3003
|
+
// line - "A login or proxy in front of it?" would be the wrong lead; the
|
|
3004
|
+
// end-of-run summary names the cause and the count once.
|
|
3005
|
+
if (outcome.kind !== "parse_error" && outcome.transient) {
|
|
3006
|
+
this.beginFailedTransient = true;
|
|
3007
|
+
this.giveUp();
|
|
3008
|
+
return null;
|
|
3009
|
+
}
|
|
2824
3010
|
// (spec section 7, "New reporter against old app") A 404 here means the
|
|
2825
3011
|
// TARGET SERVER does not have the v2 routes at all - an old squally-app.
|
|
2826
3012
|
// Cannot happen in production (the app deploys before any reporter that
|
|
@@ -2885,8 +3071,13 @@ class SquallyReporter {
|
|
|
2885
3071
|
...(interrupted ? { interrupted: true } : {}),
|
|
2886
3072
|
};
|
|
2887
3073
|
const outcome = await this.requestJson("POST", `${this.apiUrl}/api/ingest/v2/runs/${runId}/complete`, payload);
|
|
3074
|
+
// Said in the end-of-run summary (0.17.0), with what that means for the
|
|
3075
|
+
// run in Squally - not here, a line before it.
|
|
2888
3076
|
if (outcome.kind !== "ok") {
|
|
2889
|
-
|
|
3077
|
+
this.completeFailure =
|
|
3078
|
+
outcome.kind !== "parse_error" && outcome.transient && this.exhaustedCause !== null
|
|
3079
|
+
? this.exhaustedCause
|
|
3080
|
+
: this.outcomeDetail(outcome);
|
|
2890
3081
|
}
|
|
2891
3082
|
}
|
|
2892
3083
|
// (spec section 6) Called once, fired alongside beginRun() rather than
|
|
@@ -2939,12 +3130,10 @@ class SquallyReporter {
|
|
|
2939
3130
|
}
|
|
2940
3131
|
// Shared low-level request/retry machinery for begin/append/complete.
|
|
2941
3132
|
//
|
|
2942
|
-
//
|
|
2943
|
-
//
|
|
2944
|
-
//
|
|
2945
|
-
//
|
|
2946
|
-
// total tries) after a short fixed delay. Any other non-2xx status is
|
|
2947
|
-
// returned immediately, uninterpreted - the CALLER (beginRun/sendBatch/
|
|
3133
|
+
// A transient failure is retried within one time budget (the retry policy at
|
|
3134
|
+
// the top of the file, 0.17.0): TRANSIENT_REQUEST_BUDGET_MS from the first
|
|
3135
|
+
// try, and from onEnd on never past endDeadline(). Any other non-2xx status
|
|
3136
|
+
// is returned immediately, uninterpreted - the CALLER (beginRun/sendBatch/
|
|
2948
3137
|
// completeRun) decides what a given status code means for ITS route,
|
|
2949
3138
|
// since the same code means different things on different routes (404 on
|
|
2950
3139
|
// /runs means "no v2 support"; 404 on /results means "unknown runId").
|
|
@@ -2954,75 +3143,107 @@ class SquallyReporter {
|
|
|
2954
3143
|
// attempt tuple, complete keyed on shardIndex), which is what makes
|
|
2955
3144
|
// retrying safely possible in the first place.
|
|
2956
3145
|
async requestJson(method, url, body) {
|
|
2957
|
-
|
|
2958
|
-
|
|
2959
|
-
|
|
2960
|
-
for (;;) {
|
|
2961
|
-
|
|
2962
|
-
|
|
2963
|
-
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
"Content-Type": "application/json",
|
|
2967
|
-
Authorization: `Bearer ${this.apiKey}`,
|
|
2968
|
-
"User-Agent": http_config_1.USER_AGENT,
|
|
2969
|
-
},
|
|
2970
|
-
body: body === undefined ? undefined : JSON.stringify(body),
|
|
2971
|
-
dispatcher: this.dispatcher,
|
|
2972
|
-
signal: AbortSignal.timeout(http_config_1.API_REQUEST_TIMEOUT_MS),
|
|
2973
|
-
redirect: squally_response_1.SQUALLY_REDIRECT,
|
|
2974
|
-
});
|
|
2975
|
-
}
|
|
2976
|
-
catch (err) {
|
|
2977
|
-
networkAttempts++;
|
|
2978
|
-
if (networkAttempts >= NETWORK_ERROR_MAX_ATTEMPTS) {
|
|
2979
|
-
return { kind: "network_error", detail: describeRequestError(err, http_config_1.API_REQUEST_TIMEOUT_MS) };
|
|
2980
|
-
}
|
|
2981
|
-
await delay(http_config_1.SERVER_ERROR_RETRY_DELAY_MS);
|
|
2982
|
-
continue;
|
|
2983
|
-
}
|
|
2984
|
-
if (response.status === 429) {
|
|
2985
|
-
rateLimitAttempts++;
|
|
2986
|
-
if (rateLimitAttempts >= RATE_LIMIT_MAX_ATTEMPTS) {
|
|
2987
|
-
return errorOutcome(response, await (0, squally_response_1.readErrorAnswer)(response, url));
|
|
2988
|
-
}
|
|
2989
|
-
const retryAfterMs = parseRetryAfterMs(response.headers.get("retry-after"), Date.now());
|
|
2990
|
-
const waitMs = Math.min(RATE_LIMIT_BACKOFF_CAP_MS, retryAfterMs ?? computeBackoffMs(rateLimitAttempts));
|
|
2991
|
-
await delay(waitMs);
|
|
2992
|
-
continue;
|
|
2993
|
-
}
|
|
2994
|
-
if (response.status >= 500) {
|
|
2995
|
-
serverErrorAttempts++;
|
|
2996
|
-
if (serverErrorAttempts >= SERVER_ERROR_MAX_ATTEMPTS) {
|
|
2997
|
-
// A 5xx stays an http_error whoever sent it - every caller treats it
|
|
2998
|
-
// as transient - but its detail quotes only Squally's own sentence.
|
|
2999
|
-
const answer = await (0, squally_response_1.readErrorAnswer)(response, url);
|
|
3000
|
-
return answer.squally
|
|
3001
|
-
? errorOutcome(response, answer)
|
|
3002
|
-
: { kind: "http_error", status: response.status, detail: answer.detail };
|
|
3003
|
-
}
|
|
3004
|
-
await delay(http_config_1.SERVER_ERROR_RETRY_DELAY_MS);
|
|
3005
|
-
continue;
|
|
3006
|
-
}
|
|
3007
|
-
if (!response.ok) {
|
|
3008
|
-
// Read once: squally-app's `code` and `error` when it sent them
|
|
3009
|
-
// (begin's 409, 0.11.0); a redirect or a foreign body otherwise
|
|
3010
|
-
// (0.14.1). Never retried, like every other 4xx.
|
|
3011
|
-
return errorOutcome(response, await (0, squally_response_1.readErrorAnswer)(response, url));
|
|
3012
|
-
}
|
|
3013
|
-
// A 2xx that is not JSON is not Squally's (0.14.1): the measured case is a
|
|
3014
|
-
// login page at the end of a redirect, which fetch no longer follows -
|
|
3015
|
-
// the check stays for a proxy that answers 200 itself.
|
|
3016
|
-
if (!(0, squally_response_1.isJsonContentType)(response.headers.get("content-type"))) {
|
|
3017
|
-
return { kind: "not_squally", status: response.status, redirect: false, detail: (0, squally_response_1.notSquallyReason)(response, url) };
|
|
3018
|
-
}
|
|
3019
|
-
try {
|
|
3020
|
-
const json = (await response.json());
|
|
3021
|
-
return { kind: "ok", status: response.status, json };
|
|
3146
|
+
const scale = transientTimeScale();
|
|
3147
|
+
const budgetEnd = Date.now() + TRANSIENT_REQUEST_BUDGET_MS * scale;
|
|
3148
|
+
const cap = TRANSIENT_BACKOFF_CAP_MS * scale;
|
|
3149
|
+
for (let retry = 1;; retry++) {
|
|
3150
|
+
const attempt = await this.attemptJson(method, url, body);
|
|
3151
|
+
if (attempt.transient === null) {
|
|
3152
|
+
if (attempt.outcome.kind === "ok")
|
|
3153
|
+
this.transientActive = false;
|
|
3154
|
+
return attempt.outcome;
|
|
3022
3155
|
}
|
|
3023
|
-
|
|
3024
|
-
|
|
3156
|
+
this.transientSeen = true;
|
|
3157
|
+
this.transientActive = true;
|
|
3158
|
+
this.lastTransientCause = attempt.transient.cause;
|
|
3159
|
+
const waitMs = attempt.transient.retryAfterMs !== null
|
|
3160
|
+
? Math.min(cap, attempt.transient.retryAfterMs)
|
|
3161
|
+
: withJitter(computeBackoffMs(retry) * scale);
|
|
3162
|
+
// The wait must END within the budget, so the request after it is still
|
|
3163
|
+
// inside - a wait that would cross it is not started at all.
|
|
3164
|
+
if (Date.now() + waitMs > Math.min(budgetEnd, this.endDeadline())) {
|
|
3165
|
+
this.exhaustedCause = attempt.transient.cause;
|
|
3166
|
+
return { ...attempt.outcome, transient: true };
|
|
3025
3167
|
}
|
|
3168
|
+
await delay(waitMs);
|
|
3169
|
+
}
|
|
3170
|
+
}
|
|
3171
|
+
/** One try of requestJson: the outcome, and whether its failure is transient. */
|
|
3172
|
+
async attemptJson(method, url, body) {
|
|
3173
|
+
let response;
|
|
3174
|
+
try {
|
|
3175
|
+
response = await (0, undici_1.fetch)(url, {
|
|
3176
|
+
method,
|
|
3177
|
+
headers: {
|
|
3178
|
+
"Content-Type": "application/json",
|
|
3179
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
3180
|
+
"User-Agent": http_config_1.USER_AGENT,
|
|
3181
|
+
},
|
|
3182
|
+
body: body === undefined ? undefined : JSON.stringify(body),
|
|
3183
|
+
dispatcher: this.dispatcher,
|
|
3184
|
+
signal: AbortSignal.timeout(http_config_1.API_REQUEST_TIMEOUT_MS),
|
|
3185
|
+
redirect: squally_response_1.SQUALLY_REDIRECT,
|
|
3186
|
+
});
|
|
3187
|
+
}
|
|
3188
|
+
catch (err) {
|
|
3189
|
+
const detail = describeRequestError(err, http_config_1.API_REQUEST_TIMEOUT_MS);
|
|
3190
|
+
return {
|
|
3191
|
+
outcome: { kind: "network_error", detail },
|
|
3192
|
+
// "- giving up" is the single try's word; the summary says the retries ran out.
|
|
3193
|
+
transient: { cause: `Squally could not be reached (${detail.replace(/ - giving up$/, "")})`, retryAfterMs: null },
|
|
3194
|
+
};
|
|
3195
|
+
}
|
|
3196
|
+
if (response.status === 429) {
|
|
3197
|
+
const retryAfterMs = parseRetryAfterMs(response.headers.get("retry-after"), Date.now());
|
|
3198
|
+
return {
|
|
3199
|
+
outcome: errorOutcome(response, await (0, squally_response_1.readErrorAnswer)(response, url)),
|
|
3200
|
+
transient: { cause: "Squally's rate limit refused the requests (HTTP 429)", retryAfterMs },
|
|
3201
|
+
};
|
|
3202
|
+
}
|
|
3203
|
+
// Before the 4xx below: this 403 is the edge's, not a refusal of the key.
|
|
3204
|
+
const mitigated = response.status === 403 ? (0, squally_response_1.edgeMitigation)(response.headers) : null;
|
|
3205
|
+
if (mitigated !== null) {
|
|
3206
|
+
return {
|
|
3207
|
+
outcome: errorOutcome(response, await (0, squally_response_1.readErrorAnswer)(response, url)),
|
|
3208
|
+
transient: {
|
|
3209
|
+
cause: `Vercel's firewall in front of Squally ${mitigated === "deny" ? "blocked" : "challenged"} this runner (HTTP 403, x-vercel-mitigated: ${mitigated})`,
|
|
3210
|
+
retryAfterMs: null,
|
|
3211
|
+
},
|
|
3212
|
+
};
|
|
3213
|
+
}
|
|
3214
|
+
if (response.status >= 500) {
|
|
3215
|
+
// A 5xx stays an http_error whoever sent it - every caller treats it
|
|
3216
|
+
// as transient - but its detail quotes only Squally's own sentence.
|
|
3217
|
+
const answer = await (0, squally_response_1.readErrorAnswer)(response, url);
|
|
3218
|
+
return {
|
|
3219
|
+
outcome: answer.squally
|
|
3220
|
+
? errorOutcome(response, answer)
|
|
3221
|
+
: { kind: "http_error", status: response.status, detail: answer.detail },
|
|
3222
|
+
transient: { cause: `Squally answered ${answer.squally ? answer.detail : `HTTP ${response.status}`}`, retryAfterMs: null },
|
|
3223
|
+
};
|
|
3224
|
+
}
|
|
3225
|
+
if (!response.ok) {
|
|
3226
|
+
// Read once: squally-app's `code` and `error` when it sent them
|
|
3227
|
+
// (begin's 409, 0.11.0); a redirect or a foreign body otherwise
|
|
3228
|
+
// (0.14.1). Never retried, like every other 4xx - a 403 without the
|
|
3229
|
+
// firewall's header included.
|
|
3230
|
+
return { outcome: errorOutcome(response, await (0, squally_response_1.readErrorAnswer)(response, url)), transient: null };
|
|
3231
|
+
}
|
|
3232
|
+
// A 2xx that is not JSON is not Squally's (0.14.1): the measured case is a
|
|
3233
|
+
// login page at the end of a redirect, which fetch no longer follows -
|
|
3234
|
+
// the check stays for a proxy that answers 200 itself.
|
|
3235
|
+
if (!(0, squally_response_1.isJsonContentType)(response.headers.get("content-type"))) {
|
|
3236
|
+
return {
|
|
3237
|
+
outcome: { kind: "not_squally", status: response.status, redirect: false, detail: (0, squally_response_1.notSquallyReason)(response, url) },
|
|
3238
|
+
transient: null,
|
|
3239
|
+
};
|
|
3240
|
+
}
|
|
3241
|
+
try {
|
|
3242
|
+
const json = (await response.json());
|
|
3243
|
+
return { outcome: { kind: "ok", status: response.status, json }, transient: null };
|
|
3244
|
+
}
|
|
3245
|
+
catch (err) {
|
|
3246
|
+
return { outcome: { kind: "parse_error", detail: String(err) }, transient: null };
|
|
3026
3247
|
}
|
|
3027
3248
|
}
|
|
3028
3249
|
// Top-level safety net - presignAndUpload already handles presign/
|
|
@@ -3205,6 +3426,10 @@ exports.__internal = {
|
|
|
3205
3426
|
mapErrors,
|
|
3206
3427
|
parseRetryAfterMs,
|
|
3207
3428
|
computeBackoffMs,
|
|
3429
|
+
// 0.17.0
|
|
3430
|
+
withJitter,
|
|
3431
|
+
transientTimeScale,
|
|
3432
|
+
deliverySummary,
|
|
3208
3433
|
flushIntervalFor,
|
|
3209
3434
|
attemptQualifiesForArtifacts,
|
|
3210
3435
|
artifactKey,
|