reqon-dsl 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -3
- package/dist/ast/nodes.d.ts +8 -0
- package/dist/auth/circuit-breaker.d.ts +11 -0
- package/dist/auth/circuit-breaker.js +83 -12
- package/dist/auth/credentials.d.ts +6 -1
- package/dist/auth/credentials.js +12 -4
- package/dist/auth/oauth2-provider.js +13 -3
- package/dist/auth/rate-limiter.d.ts +8 -1
- package/dist/auth/rate-limiter.js +30 -10
- package/dist/auth/token-store.js +8 -1
- package/dist/cli.d.ts +11 -1
- package/dist/cli.js +65 -6
- package/dist/config/constants.d.ts +15 -4
- package/dist/config/constants.js +15 -4
- package/dist/control/server.d.ts +17 -0
- package/dist/control/server.js +82 -5
- package/dist/control/types.d.ts +6 -0
- package/dist/debug/cli-debugger.js +8 -3
- package/dist/execution/store.js +2 -2
- package/dist/execution-log/events.d.ts +125 -0
- package/dist/execution-log/events.js +17 -0
- package/dist/execution-log/fold.d.ts +38 -0
- package/dist/execution-log/fold.js +54 -0
- package/dist/execution-log/index.d.ts +18 -0
- package/dist/execution-log/index.js +6 -0
- package/dist/execution-log/postgres-store.d.ts +36 -0
- package/dist/execution-log/postgres-store.js +108 -0
- package/dist/execution-log/resume.d.ts +11 -0
- package/dist/execution-log/resume.js +5 -0
- package/dist/execution-log/sqlite-store.d.ts +16 -0
- package/dist/execution-log/sqlite-store.js +101 -0
- package/dist/execution-log/store.d.ts +72 -0
- package/dist/execution-log/store.js +182 -0
- package/dist/index.d.ts +4 -3
- package/dist/index.js +4 -3
- package/dist/interpreter/context.d.ts +15 -0
- package/dist/interpreter/context.js +3 -0
- package/dist/interpreter/evaluator.js +38 -8
- package/dist/interpreter/executor.d.ts +63 -1
- package/dist/interpreter/executor.js +406 -30
- package/dist/interpreter/fetch-handler.d.ts +39 -1
- package/dist/interpreter/fetch-handler.js +84 -15
- package/dist/interpreter/http.d.ts +31 -2
- package/dist/interpreter/http.js +187 -26
- package/dist/interpreter/index.d.ts +3 -3
- package/dist/interpreter/index.js +3 -3
- package/dist/interpreter/pagination.d.ts +1 -1
- package/dist/interpreter/pagination.js +7 -1
- package/dist/interpreter/step-handlers/for-handler.d.ts +3 -0
- package/dist/interpreter/step-handlers/for-handler.js +18 -3
- package/dist/interpreter/step-handlers/match-handler.js +5 -2
- package/dist/interpreter/step-handlers/store-handler.d.ts +7 -1
- package/dist/interpreter/step-handlers/store-handler.js +25 -16
- package/dist/interpreter/step-handlers/validate-handler.js +4 -1
- package/dist/interpreter/step-handlers/webhook-handler.d.ts +1 -0
- package/dist/interpreter/step-handlers/webhook-handler.js +13 -3
- package/dist/interpreter/store-manager.d.ts +1 -1
- package/dist/interpreter/store-manager.js +5 -1
- package/dist/loader/index.js +5 -8
- package/dist/mcp/sandbox.d.ts +41 -0
- package/dist/mcp/sandbox.js +76 -0
- package/dist/mcp/server.js +62 -9
- package/dist/oas/loader.d.ts +13 -1
- package/dist/oas/loader.js +25 -3
- package/dist/oas/mock-generator.js +13 -4
- package/dist/oas/validator.js +45 -5
- package/dist/observability/events.d.ts +6 -2
- package/dist/observability/events.js +0 -5
- package/dist/observability/logger.js +17 -10
- package/dist/observability/otel.d.ts +8 -0
- package/dist/observability/otel.js +45 -10
- package/dist/parser/action-parser.js +2 -2
- package/dist/parser/base.d.ts +7 -0
- package/dist/parser/base.js +11 -0
- package/dist/parser/expressions.d.ts +1 -0
- package/dist/parser/expressions.js +17 -4
- package/dist/parser/fetch-parser.js +13 -2
- package/dist/pause/index.d.ts +1 -0
- package/dist/pause/index.js +1 -0
- package/dist/pause/log-store.d.ts +33 -0
- package/dist/pause/log-store.js +98 -0
- package/dist/pause/manager.d.ts +12 -0
- package/dist/pause/manager.js +77 -28
- package/dist/pause/store.js +5 -3
- package/dist/scheduler/cron-parser.d.ts +10 -3
- package/dist/scheduler/cron-parser.js +227 -48
- package/dist/scheduler/scheduler.js +56 -22
- package/dist/stores/factory.d.ts +6 -0
- package/dist/stores/factory.js +11 -1
- package/dist/stores/file.js +9 -17
- package/dist/stores/memory.js +3 -12
- package/dist/stores/postgrest.d.ts +28 -0
- package/dist/stores/postgrest.js +84 -37
- package/dist/sync/index.d.ts +3 -2
- package/dist/sync/index.js +2 -1
- package/dist/sync/log-store.d.ts +30 -0
- package/dist/sync/log-store.js +45 -0
- package/dist/sync/store.js +1 -1
- package/dist/trace/index.d.ts +2 -0
- package/dist/trace/index.js +1 -0
- package/dist/trace/log-view.d.ts +57 -0
- package/dist/trace/log-view.js +76 -0
- package/dist/trace/recorder.d.ts +5 -1
- package/dist/trace/recorder.js +19 -6
- package/dist/trace/store.d.ts +6 -0
- package/dist/trace/store.js +47 -22
- package/dist/utils/deep-merge.d.ts +10 -0
- package/dist/utils/deep-merge.js +23 -0
- package/dist/utils/file.d.ts +13 -4
- package/dist/utils/file.js +70 -12
- package/dist/utils/index.d.ts +1 -1
- package/dist/utils/index.js +1 -1
- package/dist/utils/long-timeout.d.ts +19 -0
- package/dist/utils/long-timeout.js +33 -0
- package/dist/utils/path.d.ts +22 -1
- package/dist/utils/path.js +46 -1
- package/dist/utils/redact.d.ts +22 -0
- package/dist/utils/redact.js +42 -0
- package/dist/webhook/server.d.ts +9 -0
- package/dist/webhook/server.js +115 -30
- package/dist/webhook/types.d.ts +9 -1
- package/package.json +22 -4
package/README.md
CHANGED
|
@@ -73,9 +73,9 @@ reqon sync-invoices.vague --dry-run
|
|
|
73
73
|
### Programmatic
|
|
74
74
|
|
|
75
75
|
```typescript
|
|
76
|
-
import {
|
|
76
|
+
import { execute } from 'reqon';
|
|
77
77
|
|
|
78
|
-
const
|
|
78
|
+
const source = `
|
|
79
79
|
mission Test {
|
|
80
80
|
source API { auth: bearer, base: "https://api.example.com" }
|
|
81
81
|
store items: memory("items")
|
|
@@ -85,7 +85,7 @@ const program = parse(`
|
|
|
85
85
|
}
|
|
86
86
|
run Fetch
|
|
87
87
|
}
|
|
88
|
-
|
|
88
|
+
`;
|
|
89
89
|
|
|
90
90
|
const result = await execute(source, {
|
|
91
91
|
auth: { API: { type: 'bearer', token: 'your-token' } }
|
|
@@ -213,6 +213,26 @@ mission DurablePipeline {
|
|
|
213
213
|
}
|
|
214
214
|
```
|
|
215
215
|
|
|
216
|
+
#### Durability guarantees
|
|
217
|
+
|
|
218
|
+
Run a mission as a **durable execution** (`executionLog:`) and an append-only
|
|
219
|
+
event log lets an interrupted run — crash, deploy, `kill -9` — resume exactly
|
|
220
|
+
where it left off:
|
|
221
|
+
|
|
222
|
+
- **Delivery**: at-least-once (a step is never silently dropped).
|
|
223
|
+
- **Effects**: exactly-once where the API honours idempotency keys (mutating
|
|
224
|
+
fetches carry a stable `Idempotency-Key`), otherwise at-least-once + keyed
|
|
225
|
+
store dedup. Exactly-once on replay via recorded effect identity.
|
|
226
|
+
- **Resume across restart**: replay + fold; effects already applied are skipped.
|
|
227
|
+
- **Backends**: in-memory (tests), file (dev), `SqliteExecutionLog`
|
|
228
|
+
(transactional, fsync-backed) for single-process self-hosting, and
|
|
229
|
+
`PostgresExecutionLog` for multi-node.
|
|
230
|
+
|
|
231
|
+
These guarantees are proven by a crash-injection suite that kills the run at
|
|
232
|
+
every event boundary and asserts no lost record and no duplicated effect
|
|
233
|
+
(`npm run test:crash`). See **[DURABILITY.md](./DURABILITY.md)** for the full
|
|
234
|
+
statement and the guarantee → test map.
|
|
235
|
+
|
|
216
236
|
## OpenAPI Integration
|
|
217
237
|
|
|
218
238
|
Reqon can consume OpenAPI specs directly, eliminating the need for handwritten SDK code:
|
package/dist/ast/nodes.d.ts
CHANGED
|
@@ -197,6 +197,12 @@ export interface FetchStep {
|
|
|
197
197
|
until?: Expression;
|
|
198
198
|
retry?: RetryConfig;
|
|
199
199
|
since?: SinceConfig;
|
|
200
|
+
/**
|
|
201
|
+
* Resumable backfill: persist per-page progress to the execution log so a
|
|
202
|
+
* large paginated fetch survives a restart/deploy and continues from the last
|
|
203
|
+
* page. Requires durable mode (an executionLog). Only meaningful with paginate.
|
|
204
|
+
*/
|
|
205
|
+
backfill?: boolean;
|
|
200
206
|
}
|
|
201
207
|
export interface SinceConfig {
|
|
202
208
|
/** How to resolve the "since" timestamp */
|
|
@@ -229,6 +235,8 @@ export interface RetryConfig {
|
|
|
229
235
|
backoff: 'exponential' | 'linear' | 'constant';
|
|
230
236
|
initialDelay: number;
|
|
231
237
|
maxDelay?: number;
|
|
238
|
+
/** Per-attempt request timeout in ms; aborts a request that hangs */
|
|
239
|
+
timeout?: number;
|
|
232
240
|
}
|
|
233
241
|
export interface ForStep {
|
|
234
242
|
type: 'ForStep';
|
|
@@ -99,6 +99,7 @@ export declare class CircuitBreaker {
|
|
|
99
99
|
* Get current status for a source/endpoint
|
|
100
100
|
*/
|
|
101
101
|
getStatus(source: string, endpoint?: string): CircuitBreakerStatus;
|
|
102
|
+
private statusFromEntry;
|
|
102
103
|
/**
|
|
103
104
|
* Force reset a circuit to closed state
|
|
104
105
|
*/
|
|
@@ -107,8 +108,18 @@ export declare class CircuitBreaker {
|
|
|
107
108
|
* Get all circuit statuses
|
|
108
109
|
*/
|
|
109
110
|
getAllStatuses(): Map<string, CircuitBreakerStatus>;
|
|
111
|
+
/**
|
|
112
|
+
* Evict stale closed circuits to bound memory. Open and half-open circuits
|
|
113
|
+
* are always retained (they carry active backoff/recovery state).
|
|
114
|
+
*
|
|
115
|
+
* @param maxAgeMs Remove closed entries with no activity for this long.
|
|
116
|
+
* @returns Number of entries evicted.
|
|
117
|
+
*/
|
|
118
|
+
evictStale(maxAgeMs?: number): number;
|
|
110
119
|
private getKey;
|
|
111
120
|
private createEntry;
|
|
121
|
+
/** Read-only lookup. Never inserts — used by status/probe-check paths. */
|
|
122
|
+
private getEntry;
|
|
112
123
|
private getOrCreateEntry;
|
|
113
124
|
private pruneOldFailures;
|
|
114
125
|
private transitionTo;
|
|
@@ -67,7 +67,12 @@ export class CircuitBreaker {
|
|
|
67
67
|
* Check if a request can proceed (throws if circuit is open)
|
|
68
68
|
*/
|
|
69
69
|
canProceed(source, endpoint) {
|
|
70
|
-
|
|
70
|
+
// Read-only: an unknown key is implicitly closed and must not be inserted
|
|
71
|
+
// (otherwise every probed endpoint leaks an entry forever).
|
|
72
|
+
const entry = this.getEntry(source, endpoint);
|
|
73
|
+
if (!entry) {
|
|
74
|
+
return true;
|
|
75
|
+
}
|
|
71
76
|
const now = Date.now();
|
|
72
77
|
if (entry.state === 'closed') {
|
|
73
78
|
return true;
|
|
@@ -75,13 +80,20 @@ export class CircuitBreaker {
|
|
|
75
80
|
if (entry.state === 'open') {
|
|
76
81
|
const timeSinceOpen = now - (entry.openedAt ?? now);
|
|
77
82
|
if (timeSinceOpen >= entry.config.resetTimeout) {
|
|
78
|
-
// Transition to half-open
|
|
83
|
+
// Transition to half-open and let this single caller be the probe.
|
|
79
84
|
this.transitionTo(entry, 'half_open', source, endpoint);
|
|
85
|
+
entry.probeInFlight = true;
|
|
80
86
|
return true;
|
|
81
87
|
}
|
|
82
88
|
return false;
|
|
83
89
|
}
|
|
84
|
-
// Half-open: allow
|
|
90
|
+
// Half-open: allow exactly one probe at a time. Every other request is
|
|
91
|
+
// denied until the in-flight probe resolves (success closes the circuit or
|
|
92
|
+
// advances the count; failure re-opens it), preventing a recovery stampede.
|
|
93
|
+
if (entry.probeInFlight) {
|
|
94
|
+
return false;
|
|
95
|
+
}
|
|
96
|
+
entry.probeInFlight = true;
|
|
85
97
|
return true;
|
|
86
98
|
}
|
|
87
99
|
/**
|
|
@@ -89,9 +101,11 @@ export class CircuitBreaker {
|
|
|
89
101
|
*/
|
|
90
102
|
ensureCanProceed(source, endpoint) {
|
|
91
103
|
if (!this.canProceed(source, endpoint)) {
|
|
92
|
-
|
|
104
|
+
// canProceed only returns false for an existing open/half-open entry.
|
|
105
|
+
const entry = this.getEntry(source, endpoint);
|
|
106
|
+
const config = entry?.config ?? this.defaultConfig;
|
|
93
107
|
const now = Date.now();
|
|
94
|
-
const nextAttemptIn =
|
|
108
|
+
const nextAttemptIn = config.resetTimeout - (now - (entry?.openedAt ?? now));
|
|
95
109
|
this.callbacks.onRejected?.({
|
|
96
110
|
source,
|
|
97
111
|
endpoint,
|
|
@@ -104,13 +118,23 @@ export class CircuitBreaker {
|
|
|
104
118
|
* Record a successful request
|
|
105
119
|
*/
|
|
106
120
|
recordSuccess(source, endpoint) {
|
|
107
|
-
|
|
121
|
+
// Read-only lookup: a success on an untracked (implicitly closed) endpoint
|
|
122
|
+
// has nothing to record, so don't allocate an entry for it.
|
|
123
|
+
const entry = this.getEntry(source, endpoint);
|
|
124
|
+
if (!entry) {
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
entry.lastActivity = Date.now();
|
|
108
128
|
if (entry.state === 'half_open') {
|
|
109
129
|
entry.successes++;
|
|
110
130
|
if (entry.successes >= entry.config.successThreshold) {
|
|
111
131
|
// Recovery successful, close the circuit
|
|
112
132
|
this.transitionTo(entry, 'closed', source, endpoint);
|
|
113
133
|
}
|
|
134
|
+
else {
|
|
135
|
+
// Probe succeeded but more are needed; release the slot for the next.
|
|
136
|
+
entry.probeInFlight = false;
|
|
137
|
+
}
|
|
114
138
|
}
|
|
115
139
|
else if (entry.state === 'closed') {
|
|
116
140
|
// Clear old failures from window
|
|
@@ -121,15 +145,19 @@ export class CircuitBreaker {
|
|
|
121
145
|
* Record a failed request
|
|
122
146
|
*/
|
|
123
147
|
recordFailure(source, endpoint, statusCode, isNetworkError = false) {
|
|
124
|
-
|
|
125
|
-
|
|
148
|
+
// Resolve config without forcing an entry — ignored failures (e.g. a 404)
|
|
149
|
+
// must not allocate a permanent circuit for an otherwise-untracked endpoint.
|
|
150
|
+
const existing = this.getEntry(source, endpoint);
|
|
151
|
+
const config = existing?.config ?? this.getEntry(source)?.config ?? this.defaultConfig;
|
|
126
152
|
// Check if this failure type should be counted
|
|
127
153
|
const isFailureStatus = statusCode !== undefined && config.failureStatusCodes.includes(statusCode);
|
|
128
154
|
const shouldCount = isFailureStatus || (isNetworkError && config.countNetworkErrors);
|
|
129
155
|
if (!shouldCount) {
|
|
130
156
|
return;
|
|
131
157
|
}
|
|
158
|
+
const entry = existing ?? this.getOrCreateEntry(source, endpoint);
|
|
132
159
|
const now = Date.now();
|
|
160
|
+
entry.lastActivity = now;
|
|
133
161
|
if (entry.state === 'half_open') {
|
|
134
162
|
// Any failure in half-open immediately re-opens circuit
|
|
135
163
|
this.transitionTo(entry, 'open', source, endpoint, 'Failure during recovery attempt');
|
|
@@ -151,8 +179,14 @@ export class CircuitBreaker {
|
|
|
151
179
|
* Get current status for a source/endpoint
|
|
152
180
|
*/
|
|
153
181
|
getStatus(source, endpoint) {
|
|
154
|
-
|
|
155
|
-
const
|
|
182
|
+
// Read-only: querying an unknown circuit must not create one.
|
|
183
|
+
const entry = this.getEntry(source, endpoint);
|
|
184
|
+
if (!entry) {
|
|
185
|
+
return { state: 'closed', failures: 0, successes: 0, isOpen: false };
|
|
186
|
+
}
|
|
187
|
+
return this.statusFromEntry(entry);
|
|
188
|
+
}
|
|
189
|
+
statusFromEntry(entry) {
|
|
156
190
|
let nextAttemptTime;
|
|
157
191
|
if (entry.state === 'open' && entry.openedAt) {
|
|
158
192
|
const nextAttemptMs = entry.openedAt + entry.config.resetTimeout;
|
|
@@ -181,6 +215,8 @@ export class CircuitBreaker {
|
|
|
181
215
|
entry.failureTimestamps = [];
|
|
182
216
|
entry.lastFailureTime = undefined;
|
|
183
217
|
entry.openedAt = undefined;
|
|
218
|
+
entry.probeInFlight = false;
|
|
219
|
+
entry.lastActivity = Date.now();
|
|
184
220
|
if (previousState !== 'closed') {
|
|
185
221
|
this.callbacks.onClose?.({
|
|
186
222
|
source,
|
|
@@ -198,12 +234,37 @@ export class CircuitBreaker {
|
|
|
198
234
|
*/
|
|
199
235
|
getAllStatuses() {
|
|
200
236
|
const result = new Map();
|
|
237
|
+
// Build the status from each entry directly. Splitting the key on ':' would
|
|
238
|
+
// corrupt URL-shaped keys (e.g. `https://x` → source `https`), and routing
|
|
239
|
+
// back through getStatus is needless. We never mutate `circuits` here.
|
|
201
240
|
for (const [key, entry] of this.circuits) {
|
|
202
|
-
|
|
203
|
-
result.set(key, this.getStatus(source, endpoint === '' ? undefined : endpoint));
|
|
241
|
+
result.set(key, this.statusFromEntry(entry));
|
|
204
242
|
}
|
|
205
243
|
return result;
|
|
206
244
|
}
|
|
245
|
+
/**
|
|
246
|
+
* Evict stale closed circuits to bound memory. Open and half-open circuits
|
|
247
|
+
* are always retained (they carry active backoff/recovery state).
|
|
248
|
+
*
|
|
249
|
+
* @param maxAgeMs Remove closed entries with no activity for this long.
|
|
250
|
+
* @returns Number of entries evicted.
|
|
251
|
+
*/
|
|
252
|
+
evictStale(maxAgeMs = this.defaultConfig.failureWindow) {
|
|
253
|
+
const cutoff = Date.now() - maxAgeMs;
|
|
254
|
+
let evicted = 0;
|
|
255
|
+
// Collect first, then delete — never mutate the map mid-iteration.
|
|
256
|
+
const toDelete = [];
|
|
257
|
+
for (const [key, entry] of this.circuits) {
|
|
258
|
+
if (entry.state === 'closed' && entry.lastActivity < cutoff) {
|
|
259
|
+
toDelete.push(key);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
for (const key of toDelete) {
|
|
263
|
+
this.circuits.delete(key);
|
|
264
|
+
evicted++;
|
|
265
|
+
}
|
|
266
|
+
return evicted;
|
|
267
|
+
}
|
|
207
268
|
getKey(source, endpoint) {
|
|
208
269
|
return endpoint ? `${source}:${endpoint}` : source;
|
|
209
270
|
}
|
|
@@ -213,9 +274,15 @@ export class CircuitBreaker {
|
|
|
213
274
|
failures: 0,
|
|
214
275
|
successes: 0,
|
|
215
276
|
failureTimestamps: [],
|
|
277
|
+
probeInFlight: false,
|
|
278
|
+
lastActivity: Date.now(),
|
|
216
279
|
config,
|
|
217
280
|
};
|
|
218
281
|
}
|
|
282
|
+
/** Read-only lookup. Never inserts — used by status/probe-check paths. */
|
|
283
|
+
getEntry(source, endpoint) {
|
|
284
|
+
return this.circuits.get(this.getKey(source, endpoint));
|
|
285
|
+
}
|
|
219
286
|
getOrCreateEntry(source, endpoint) {
|
|
220
287
|
const key = this.getKey(source, endpoint);
|
|
221
288
|
let entry = this.circuits.get(key);
|
|
@@ -237,6 +304,10 @@ export class CircuitBreaker {
|
|
|
237
304
|
transitionTo(entry, newState, source, endpoint, reason) {
|
|
238
305
|
const previousState = entry.state;
|
|
239
306
|
entry.state = newState;
|
|
307
|
+
entry.lastActivity = Date.now();
|
|
308
|
+
// Any state change clears the half-open probe slot; canProceed re-claims it
|
|
309
|
+
// when it admits the next probe.
|
|
310
|
+
entry.probeInFlight = false;
|
|
240
311
|
const event = {
|
|
241
312
|
source,
|
|
242
313
|
endpoint,
|
|
@@ -33,12 +33,17 @@ export interface LoadEnvResult {
|
|
|
33
33
|
*/
|
|
34
34
|
export declare function loadEnv(options?: CredentialsConfig): LoadEnvResult;
|
|
35
35
|
/**
|
|
36
|
-
* Resolve environment variable references in a string
|
|
36
|
+
* Resolve environment variable references in a string.
|
|
37
37
|
*
|
|
38
38
|
* Supports:
|
|
39
39
|
* - $VAR_NAME
|
|
40
40
|
* - ${VAR_NAME}
|
|
41
41
|
* - ${VAR_NAME:-default}
|
|
42
|
+
*
|
|
43
|
+
* A reference with no value and no default is treated as *required* and throws,
|
|
44
|
+
* rather than silently resolving to '' (which previously let requests go out
|
|
45
|
+
* with an empty Authorization header). Use the `${VAR:-}` form to opt a
|
|
46
|
+
* reference into being optional with an empty fallback.
|
|
42
47
|
*/
|
|
43
48
|
export declare function resolveEnvString(value: string): string;
|
|
44
49
|
/**
|
package/dist/auth/credentials.js
CHANGED
|
@@ -55,12 +55,17 @@ export function loadEnv(options = {}) {
|
|
|
55
55
|
};
|
|
56
56
|
}
|
|
57
57
|
/**
|
|
58
|
-
* Resolve environment variable references in a string
|
|
58
|
+
* Resolve environment variable references in a string.
|
|
59
59
|
*
|
|
60
60
|
* Supports:
|
|
61
61
|
* - $VAR_NAME
|
|
62
62
|
* - ${VAR_NAME}
|
|
63
63
|
* - ${VAR_NAME:-default}
|
|
64
|
+
*
|
|
65
|
+
* A reference with no value and no default is treated as *required* and throws,
|
|
66
|
+
* rather than silently resolving to '' (which previously let requests go out
|
|
67
|
+
* with an empty Authorization header). Use the `${VAR:-}` form to opt a
|
|
68
|
+
* reference into being optional with an empty fallback.
|
|
64
69
|
*/
|
|
65
70
|
export function resolveEnvString(value) {
|
|
66
71
|
// Pattern for ${VAR:-default} or ${VAR}
|
|
@@ -75,15 +80,18 @@ export function resolveEnvString(value) {
|
|
|
75
80
|
return envValue;
|
|
76
81
|
}
|
|
77
82
|
if (defaultValue !== undefined) {
|
|
83
|
+
// Includes the empty-default escape hatch `${VAR:-}`.
|
|
78
84
|
return defaultValue;
|
|
79
85
|
}
|
|
80
|
-
|
|
81
|
-
return '';
|
|
86
|
+
throw new Error(`unresolved required credential ${varName}`);
|
|
82
87
|
});
|
|
83
88
|
// Then resolve $VAR patterns (only if not already resolved)
|
|
84
89
|
result = result.replace(simplePattern, (_match, varName) => {
|
|
85
90
|
const envValue = process.env[varName];
|
|
86
|
-
|
|
91
|
+
if (envValue !== undefined) {
|
|
92
|
+
return envValue;
|
|
93
|
+
}
|
|
94
|
+
throw new Error(`unresolved required credential ${varName}`);
|
|
87
95
|
});
|
|
88
96
|
return result;
|
|
89
97
|
}
|
|
@@ -60,10 +60,16 @@ export class OAuth2AuthProvider {
|
|
|
60
60
|
body,
|
|
61
61
|
});
|
|
62
62
|
if (!response.ok) {
|
|
63
|
-
|
|
64
|
-
|
|
63
|
+
// Do not include the response body: token endpoints can echo the
|
|
64
|
+
// submitted client secret / refresh token in error responses.
|
|
65
|
+
throw new Error(`Token refresh failed: ${response.status} ${response.statusText}`);
|
|
65
66
|
}
|
|
66
67
|
const data = (await response.json());
|
|
68
|
+
// A 2xx with no access_token would otherwise propagate as `Bearer undefined`
|
|
69
|
+
// on every subsequent request. Fail loudly instead.
|
|
70
|
+
if (!data.access_token) {
|
|
71
|
+
throw new Error(`Token refresh for connection "${this.connectionId}" returned no access_token`);
|
|
72
|
+
}
|
|
67
73
|
const newTokens = {
|
|
68
74
|
accessToken: data.access_token,
|
|
69
75
|
refreshToken: data.refresh_token ?? tokens.refreshToken, // Keep old if not rotated
|
|
@@ -80,7 +86,11 @@ export class OAuth2AuthProvider {
|
|
|
80
86
|
}
|
|
81
87
|
shouldRefresh(tokens) {
|
|
82
88
|
if (!tokens.expiresAt) {
|
|
83
|
-
|
|
89
|
+
// No expiry metadata: we cannot know when the token lapses, so we treat it
|
|
90
|
+
// as still valid and let a downstream 401 trigger an explicit
|
|
91
|
+
// refreshToken() call. Proactively refreshing here would burn the refresh
|
|
92
|
+
// token (and break non-expiring tokens) on every single request.
|
|
93
|
+
return false;
|
|
84
94
|
}
|
|
85
95
|
const now = Date.now();
|
|
86
96
|
const expiresAt = tokens.expiresAt.getTime();
|
|
@@ -32,7 +32,14 @@ export declare class AdaptiveRateLimiter implements RateLimiter {
|
|
|
32
32
|
maxStaleAgeMs?: number;
|
|
33
33
|
});
|
|
34
34
|
/**
|
|
35
|
-
* Clean up stale entries from the state map to prevent memory leaks
|
|
35
|
+
* Clean up stale entries from the state map to prevent memory leaks.
|
|
36
|
+
*
|
|
37
|
+
* Eviction is driven by `lastRequestAt`, not `resetAt`. Most of the leak came
|
|
38
|
+
* from keys whose API sends no reset header: `resetAt` is never set, so the
|
|
39
|
+
* old reset-passed condition could never fire and those keys lived forever.
|
|
40
|
+
* Now any key idle longer than the stale threshold is evicted regardless of
|
|
41
|
+
* `resetAt`, except while an active `retryAfter` backoff is still pending
|
|
42
|
+
* (that state must survive so we keep honouring the 429).
|
|
36
43
|
*/
|
|
37
44
|
private cleanup;
|
|
38
45
|
/**
|
|
@@ -52,7 +52,14 @@ export class AdaptiveRateLimiter {
|
|
|
52
52
|
this.maxStaleAgeMs = options.maxStaleAgeMs ?? RATE_LIMIT_DEFAULTS.MAX_STALE_AGE_MS;
|
|
53
53
|
}
|
|
54
54
|
/**
|
|
55
|
-
* Clean up stale entries from the state map to prevent memory leaks
|
|
55
|
+
* Clean up stale entries from the state map to prevent memory leaks.
|
|
56
|
+
*
|
|
57
|
+
* Eviction is driven by `lastRequestAt`, not `resetAt`. Most of the leak came
|
|
58
|
+
* from keys whose API sends no reset header: `resetAt` is never set, so the
|
|
59
|
+
* old reset-passed condition could never fire and those keys lived forever.
|
|
60
|
+
* Now any key idle longer than the stale threshold is evicted regardless of
|
|
61
|
+
* `resetAt`, except while an active `retryAfter` backoff is still pending
|
|
62
|
+
* (that state must survive so we keep honouring the 429).
|
|
56
63
|
*/
|
|
57
64
|
cleanup() {
|
|
58
65
|
const now = Date.now();
|
|
@@ -65,13 +72,14 @@ export class AdaptiveRateLimiter {
|
|
|
65
72
|
const nowDate = new Date(now);
|
|
66
73
|
const staleThreshold = new Date(now - this.maxStaleAgeMs);
|
|
67
74
|
for (const [key, state] of this.state) {
|
|
68
|
-
//
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
+
// A pending retry-after backoff must never be evicted out from under us.
|
|
76
|
+
const isRetryPending = state.retryAfter !== undefined && state.retryAfter >= nowDate;
|
|
77
|
+
if (isRetryPending)
|
|
78
|
+
continue;
|
|
79
|
+
// Stale if the last request is older than the threshold, or if no request
|
|
80
|
+
// was ever recorded for this key.
|
|
81
|
+
const isStale = !state.lastRequestAt || state.lastRequestAt < staleThreshold;
|
|
82
|
+
if (isStale) {
|
|
75
83
|
this.state.delete(key);
|
|
76
84
|
}
|
|
77
85
|
}
|
|
@@ -251,7 +259,16 @@ export class AdaptiveRateLimiter {
|
|
|
251
259
|
}
|
|
252
260
|
state.lastRequestAt = now;
|
|
253
261
|
this.state.set(key, state);
|
|
254
|
-
//
|
|
262
|
+
// Trigger a stale-entry sweep every N responses. The sweep itself is rate-
|
|
263
|
+
// limited by CLEANUP_INTERVAL_MS / MAX_ENTRIES_BEFORE_CLEANUP inside
|
|
264
|
+
// cleanup(), so this counter only decides how often we *consider* sweeping.
|
|
265
|
+
//
|
|
266
|
+
// TODO: this limiter is purely reactive — it learns limits from response
|
|
267
|
+
// headers after the fact. Add proactive in-flight token accounting (track
|
|
268
|
+
// tokens consumed by requests that have been issued but not yet returned a
|
|
269
|
+
// response) so concurrent callers can't collectively blow past the limit
|
|
270
|
+
// between header updates. That is a larger redesign; the eviction leak fix
|
|
271
|
+
// above is independent of it.
|
|
255
272
|
this.responsesSinceCleanup++;
|
|
256
273
|
if (this.responsesSinceCleanup >= RATE_LIMIT_DEFAULTS.CLEANUP_CHECK_INTERVAL) {
|
|
257
274
|
this.responsesSinceCleanup = 0;
|
|
@@ -266,7 +283,10 @@ export class AdaptiveRateLimiter {
|
|
|
266
283
|
return { isLimited: false };
|
|
267
284
|
}
|
|
268
285
|
const isLimited = (state.retryAfter && state.retryAfter > now) ||
|
|
269
|
-
(state.remaining !== undefined &&
|
|
286
|
+
(state.remaining !== undefined &&
|
|
287
|
+
state.remaining <= 0 &&
|
|
288
|
+
state.resetAt &&
|
|
289
|
+
state.resetAt > now);
|
|
270
290
|
let resetInSeconds;
|
|
271
291
|
if (isLimited && state.resetAt) {
|
|
272
292
|
resetInSeconds = Math.ceil((state.resetAt.getTime() - now.getTime()) / 1000);
|
package/dist/auth/token-store.js
CHANGED
|
@@ -103,7 +103,14 @@ export class FileTokenStore {
|
|
|
103
103
|
for (const [key, value] of this.cache) {
|
|
104
104
|
data[key] = value;
|
|
105
105
|
}
|
|
106
|
-
|
|
106
|
+
// Owner-only (0o600): the file holds OAuth tokens and must not be
|
|
107
|
+
// world-readable. `mode` only applies on creation, so chmod an existing
|
|
108
|
+
// file too (in case it was created before this fix or under a loose umask).
|
|
109
|
+
await fs.writeFile(this.filePath, JSON.stringify(data, null, 2), {
|
|
110
|
+
encoding: 'utf-8',
|
|
111
|
+
mode: 0o600,
|
|
112
|
+
});
|
|
113
|
+
await fs.chmod(this.filePath, 0o600).catch(() => { });
|
|
107
114
|
}
|
|
108
115
|
async get(connectionId) {
|
|
109
116
|
const map = await this.load();
|
package/dist/cli.d.ts
CHANGED
|
@@ -12,4 +12,14 @@
|
|
|
12
12
|
* - ./debug/cli-debugger.ts - interactive debugging
|
|
13
13
|
* ---
|
|
14
14
|
*/
|
|
15
|
-
|
|
15
|
+
/**
|
|
16
|
+
* Load and resolve an --auth credentials file, turning low-level fs/JSON
|
|
17
|
+
* failures into clean, user-facing messages instead of raw stack traces.
|
|
18
|
+
*/
|
|
19
|
+
export declare function loadAuthFile(rawPath: string): Promise<Record<string, unknown>>;
|
|
20
|
+
/**
|
|
21
|
+
* Print an error in a clean, user-facing form (no stack trace). ReqonErrors
|
|
22
|
+
* get their rich source-context formatting; everything else gets a one-line
|
|
23
|
+
* message.
|
|
24
|
+
*/
|
|
25
|
+
export declare function reportFatalError(error: unknown): void;
|
package/dist/cli.js
CHANGED
|
@@ -14,11 +14,57 @@
|
|
|
14
14
|
*/
|
|
15
15
|
import { readFile, writeFile, mkdir } from 'node:fs/promises';
|
|
16
16
|
import { resolve, dirname } from 'node:path';
|
|
17
|
+
import { pathToFileURL } from 'node:url';
|
|
17
18
|
import { fromPath, Scheduler, loadMission } from './index.js';
|
|
18
19
|
import { ReqonError } from './errors/index.js';
|
|
19
20
|
import { loadEnv, loadCredentials } from './auth/credentials.js';
|
|
20
21
|
import { WebhookServer } from './webhook/index.js';
|
|
21
22
|
import { ControlServer } from './control/index.js';
|
|
23
|
+
/**
|
|
24
|
+
* Load and resolve an --auth credentials file, turning low-level fs/JSON
|
|
25
|
+
* failures into clean, user-facing messages instead of raw stack traces.
|
|
26
|
+
*/
|
|
27
|
+
export async function loadAuthFile(rawPath) {
|
|
28
|
+
const authPath = resolve(rawPath);
|
|
29
|
+
let authContent;
|
|
30
|
+
try {
|
|
31
|
+
authContent = await readFile(authPath, 'utf-8');
|
|
32
|
+
}
|
|
33
|
+
catch (err) {
|
|
34
|
+
if (err.code === 'ENOENT') {
|
|
35
|
+
throw new Error(`auth file not found: ${authPath}`);
|
|
36
|
+
}
|
|
37
|
+
if (err.code === 'EISDIR') {
|
|
38
|
+
throw new Error(`auth path is a directory, not a file: ${authPath}`);
|
|
39
|
+
}
|
|
40
|
+
throw new Error(`could not read auth file ${authPath}: ${err.message}`);
|
|
41
|
+
}
|
|
42
|
+
let rawAuth;
|
|
43
|
+
try {
|
|
44
|
+
rawAuth = JSON.parse(authContent);
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
throw new Error(`auth file is not valid JSON: ${authPath}`);
|
|
48
|
+
}
|
|
49
|
+
// Resolve env var references in the auth config
|
|
50
|
+
return loadCredentials(rawAuth);
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Print an error in a clean, user-facing form (no stack trace). ReqonErrors
|
|
54
|
+
* get their rich source-context formatting; everything else gets a one-line
|
|
55
|
+
* message.
|
|
56
|
+
*/
|
|
57
|
+
export function reportFatalError(error) {
|
|
58
|
+
if (error instanceof ReqonError) {
|
|
59
|
+
console.error(error.format());
|
|
60
|
+
}
|
|
61
|
+
else if (error instanceof Error) {
|
|
62
|
+
console.error(`Error: ${error.message}`);
|
|
63
|
+
}
|
|
64
|
+
else {
|
|
65
|
+
console.error(`Error: ${String(error)}`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
22
68
|
async function main() {
|
|
23
69
|
const args = process.argv.slice(2);
|
|
24
70
|
if (args.length === 0 || args.includes('--help') || args.includes('-h')) {
|
|
@@ -31,6 +77,7 @@ Usage:
|
|
|
31
77
|
Options:
|
|
32
78
|
--dry-run Run without making actual HTTP requests
|
|
33
79
|
--verbose Enable verbose logging
|
|
80
|
+
--dev Development mode: let sql/nosql stores fall back to local JSON files
|
|
34
81
|
--auth <file> JSON file with auth credentials (supports env var interpolation)
|
|
35
82
|
--env <file> Path to .env file (default: .env in current directory)
|
|
36
83
|
--output <path> Export stores to JSON (file or directory)
|
|
@@ -77,6 +124,7 @@ Control Server:
|
|
|
77
124
|
const filePath = args[0];
|
|
78
125
|
const dryRun = args.includes('--dry-run');
|
|
79
126
|
const verbose = args.includes('--verbose');
|
|
127
|
+
const devMode = args.includes('--dev');
|
|
80
128
|
const daemon = args.includes('--daemon');
|
|
81
129
|
const once = args.includes('--once');
|
|
82
130
|
const webhookEnabled = args.includes('--webhook');
|
|
@@ -119,11 +167,7 @@ Control Server:
|
|
|
119
167
|
let auth;
|
|
120
168
|
const authIndex = args.indexOf('--auth');
|
|
121
169
|
if (authIndex !== -1 && args[authIndex + 1]) {
|
|
122
|
-
|
|
123
|
-
const authContent = await readFile(authPath, 'utf-8');
|
|
124
|
-
const rawAuth = JSON.parse(authContent);
|
|
125
|
-
// Resolve env var references in the auth config
|
|
126
|
-
auth = loadCredentials(rawAuth);
|
|
170
|
+
auth = await loadAuthFile(args[authIndex + 1]);
|
|
127
171
|
}
|
|
128
172
|
let outputPath;
|
|
129
173
|
const outputIndex = args.indexOf('--output');
|
|
@@ -174,6 +218,7 @@ Control Server:
|
|
|
174
218
|
const result = await fromPath(filePath, {
|
|
175
219
|
dryRun,
|
|
176
220
|
verbose,
|
|
221
|
+
developmentMode: devMode,
|
|
177
222
|
auth: auth,
|
|
178
223
|
webhookServer,
|
|
179
224
|
debugController,
|
|
@@ -347,4 +392,18 @@ async function runDaemon(filePath, options) {
|
|
|
347
392
|
function formatTime(date) {
|
|
348
393
|
return date.toISOString().replace('T', ' ').substring(0, 19);
|
|
349
394
|
}
|
|
350
|
-
|
|
395
|
+
// Only auto-run when invoked directly as the CLI entry point, so the module can
|
|
396
|
+
// be imported in tests without executing main() against the test runner's argv.
|
|
397
|
+
const isMain = typeof process.argv[1] === 'string' && import.meta.url === pathToFileURL(process.argv[1]).href;
|
|
398
|
+
if (isMain) {
|
|
399
|
+
// A rejected promise anywhere (including outside main's try/catch) should
|
|
400
|
+
// surface as a clean message and a non-zero exit, never a raw stack trace.
|
|
401
|
+
process.on('unhandledRejection', (reason) => {
|
|
402
|
+
reportFatalError(reason);
|
|
403
|
+
process.exit(1);
|
|
404
|
+
});
|
|
405
|
+
main().catch((error) => {
|
|
406
|
+
reportFatalError(error);
|
|
407
|
+
process.exit(1);
|
|
408
|
+
});
|
|
409
|
+
}
|
|
@@ -17,6 +17,8 @@ export declare const HTTP_RETRY_DEFAULTS: {
|
|
|
17
17
|
readonly MAX_DELAY_MS: 30000;
|
|
18
18
|
/** Backoff strategy: 'exponential', 'linear', or 'constant' */
|
|
19
19
|
readonly BACKOFF: "exponential";
|
|
20
|
+
/** Per-attempt request timeout in milliseconds (aborts a hung request) */
|
|
21
|
+
readonly TIMEOUT_MS: 30000;
|
|
20
22
|
};
|
|
21
23
|
/**
|
|
22
24
|
* Default HTTP headers
|
|
@@ -69,12 +71,16 @@ export declare const CIRCUIT_BREAKER_DEFAULTS: {
|
|
|
69
71
|
export declare const WEBHOOK_DEFAULTS: {
|
|
70
72
|
/** Default port for webhook server */
|
|
71
73
|
readonly PORT: 3000;
|
|
72
|
-
/** Default host binding */
|
|
73
|
-
readonly HOST: "
|
|
74
|
+
/** Default host binding — loopback only; opt into 0.0.0.0 explicitly. */
|
|
75
|
+
readonly HOST: "127.0.0.1";
|
|
74
76
|
/** Default timeout for wait steps (5 minutes in ms) */
|
|
75
77
|
readonly DEFAULT_TIMEOUT_MS: 300000;
|
|
76
78
|
/** Cleanup interval for expired registrations (1 minute in ms) */
|
|
77
79
|
readonly CLEANUP_INTERVAL_MS: 60000;
|
|
80
|
+
/** Maximum accepted request body size in bytes (1 MiB) */
|
|
81
|
+
readonly MAX_BODY_BYTES: number;
|
|
82
|
+
/** Idle socket timeout in ms; drops slow-drip connections */
|
|
83
|
+
readonly SOCKET_TIMEOUT_MS: 30000;
|
|
78
84
|
};
|
|
79
85
|
/**
|
|
80
86
|
* Default store configuration
|
|
@@ -100,8 +106,13 @@ export declare const SCHEDULER_DEFAULTS: {
|
|
|
100
106
|
* Default execution configuration
|
|
101
107
|
*/
|
|
102
108
|
export declare const EXECUTION_DEFAULTS: {
|
|
103
|
-
/**
|
|
104
|
-
|
|
109
|
+
/**
|
|
110
|
+
* Whether development mode is enabled by default. When true, sql/nosql
|
|
111
|
+
* stores fall back to local JSON files; default false so a mission that
|
|
112
|
+
* declares `store sql`/`nosql` errors loudly instead of silently writing
|
|
113
|
+
* to disk.
|
|
114
|
+
*/
|
|
115
|
+
readonly DEVELOPMENT_MODE: false;
|
|
105
116
|
/** Whether to persist execution state by default */
|
|
106
117
|
readonly PERSIST_STATE: false;
|
|
107
118
|
};
|