reqon-dsl 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +23 -3
  2. package/dist/ast/nodes.d.ts +8 -0
  3. package/dist/auth/circuit-breaker.d.ts +11 -0
  4. package/dist/auth/circuit-breaker.js +83 -12
  5. package/dist/auth/credentials.d.ts +6 -1
  6. package/dist/auth/credentials.js +12 -4
  7. package/dist/auth/oauth2-provider.js +13 -3
  8. package/dist/auth/rate-limiter.d.ts +8 -1
  9. package/dist/auth/rate-limiter.js +30 -10
  10. package/dist/auth/token-store.js +8 -1
  11. package/dist/cli.d.ts +11 -1
  12. package/dist/cli.js +65 -6
  13. package/dist/config/constants.d.ts +15 -4
  14. package/dist/config/constants.js +15 -4
  15. package/dist/control/server.d.ts +17 -0
  16. package/dist/control/server.js +82 -5
  17. package/dist/control/types.d.ts +6 -0
  18. package/dist/debug/cli-debugger.js +8 -3
  19. package/dist/execution/store.js +2 -2
  20. package/dist/execution-log/events.d.ts +125 -0
  21. package/dist/execution-log/events.js +17 -0
  22. package/dist/execution-log/fold.d.ts +38 -0
  23. package/dist/execution-log/fold.js +54 -0
  24. package/dist/execution-log/index.d.ts +18 -0
  25. package/dist/execution-log/index.js +6 -0
  26. package/dist/execution-log/postgres-store.d.ts +36 -0
  27. package/dist/execution-log/postgres-store.js +108 -0
  28. package/dist/execution-log/resume.d.ts +11 -0
  29. package/dist/execution-log/resume.js +5 -0
  30. package/dist/execution-log/sqlite-store.d.ts +16 -0
  31. package/dist/execution-log/sqlite-store.js +101 -0
  32. package/dist/execution-log/store.d.ts +72 -0
  33. package/dist/execution-log/store.js +182 -0
  34. package/dist/index.d.ts +4 -3
  35. package/dist/index.js +4 -3
  36. package/dist/interpreter/context.d.ts +15 -0
  37. package/dist/interpreter/context.js +3 -0
  38. package/dist/interpreter/evaluator.js +38 -8
  39. package/dist/interpreter/executor.d.ts +63 -1
  40. package/dist/interpreter/executor.js +406 -30
  41. package/dist/interpreter/fetch-handler.d.ts +39 -1
  42. package/dist/interpreter/fetch-handler.js +84 -15
  43. package/dist/interpreter/http.d.ts +31 -2
  44. package/dist/interpreter/http.js +187 -26
  45. package/dist/interpreter/index.d.ts +3 -3
  46. package/dist/interpreter/index.js +3 -3
  47. package/dist/interpreter/pagination.d.ts +1 -1
  48. package/dist/interpreter/pagination.js +7 -1
  49. package/dist/interpreter/step-handlers/for-handler.d.ts +3 -0
  50. package/dist/interpreter/step-handlers/for-handler.js +18 -3
  51. package/dist/interpreter/step-handlers/match-handler.js +5 -2
  52. package/dist/interpreter/step-handlers/store-handler.d.ts +7 -1
  53. package/dist/interpreter/step-handlers/store-handler.js +25 -16
  54. package/dist/interpreter/step-handlers/validate-handler.js +4 -1
  55. package/dist/interpreter/step-handlers/webhook-handler.d.ts +1 -0
  56. package/dist/interpreter/step-handlers/webhook-handler.js +13 -3
  57. package/dist/interpreter/store-manager.d.ts +1 -1
  58. package/dist/interpreter/store-manager.js +5 -1
  59. package/dist/loader/index.js +5 -8
  60. package/dist/mcp/sandbox.d.ts +41 -0
  61. package/dist/mcp/sandbox.js +76 -0
  62. package/dist/mcp/server.js +62 -9
  63. package/dist/oas/loader.d.ts +13 -1
  64. package/dist/oas/loader.js +25 -3
  65. package/dist/oas/mock-generator.js +13 -4
  66. package/dist/oas/validator.js +45 -5
  67. package/dist/observability/events.d.ts +6 -2
  68. package/dist/observability/events.js +0 -5
  69. package/dist/observability/logger.js +17 -10
  70. package/dist/observability/otel.d.ts +8 -0
  71. package/dist/observability/otel.js +45 -10
  72. package/dist/parser/action-parser.js +2 -2
  73. package/dist/parser/base.d.ts +7 -0
  74. package/dist/parser/base.js +11 -0
  75. package/dist/parser/expressions.d.ts +1 -0
  76. package/dist/parser/expressions.js +17 -4
  77. package/dist/parser/fetch-parser.js +13 -2
  78. package/dist/pause/index.d.ts +1 -0
  79. package/dist/pause/index.js +1 -0
  80. package/dist/pause/log-store.d.ts +33 -0
  81. package/dist/pause/log-store.js +98 -0
  82. package/dist/pause/manager.d.ts +12 -0
  83. package/dist/pause/manager.js +77 -28
  84. package/dist/pause/store.js +5 -3
  85. package/dist/scheduler/cron-parser.d.ts +10 -3
  86. package/dist/scheduler/cron-parser.js +227 -48
  87. package/dist/scheduler/scheduler.js +56 -22
  88. package/dist/stores/factory.d.ts +6 -0
  89. package/dist/stores/factory.js +11 -1
  90. package/dist/stores/file.js +9 -17
  91. package/dist/stores/memory.js +3 -12
  92. package/dist/stores/postgrest.d.ts +28 -0
  93. package/dist/stores/postgrest.js +84 -37
  94. package/dist/sync/index.d.ts +3 -2
  95. package/dist/sync/index.js +2 -1
  96. package/dist/sync/log-store.d.ts +30 -0
  97. package/dist/sync/log-store.js +45 -0
  98. package/dist/sync/store.js +1 -1
  99. package/dist/trace/index.d.ts +2 -0
  100. package/dist/trace/index.js +1 -0
  101. package/dist/trace/log-view.d.ts +57 -0
  102. package/dist/trace/log-view.js +76 -0
  103. package/dist/trace/recorder.d.ts +5 -1
  104. package/dist/trace/recorder.js +19 -6
  105. package/dist/trace/store.d.ts +6 -0
  106. package/dist/trace/store.js +47 -22
  107. package/dist/utils/deep-merge.d.ts +10 -0
  108. package/dist/utils/deep-merge.js +23 -0
  109. package/dist/utils/file.d.ts +13 -4
  110. package/dist/utils/file.js +70 -12
  111. package/dist/utils/index.d.ts +1 -1
  112. package/dist/utils/index.js +1 -1
  113. package/dist/utils/long-timeout.d.ts +19 -0
  114. package/dist/utils/long-timeout.js +33 -0
  115. package/dist/utils/path.d.ts +22 -1
  116. package/dist/utils/path.js +46 -1
  117. package/dist/utils/redact.d.ts +22 -0
  118. package/dist/utils/redact.js +42 -0
  119. package/dist/webhook/server.d.ts +9 -0
  120. package/dist/webhook/server.js +115 -30
  121. package/dist/webhook/types.d.ts +9 -1
  122. package/package.json +22 -4
package/README.md CHANGED
@@ -73,9 +73,9 @@ reqon sync-invoices.vague --dry-run
73
73
  ### Programmatic
74
74
 
75
75
  ```typescript
76
- import { parse, execute } from 'reqon';
76
+ import { execute } from 'reqon';
77
77
 
78
- const program = parse(`
78
+ const source = `
79
79
  mission Test {
80
80
  source API { auth: bearer, base: "https://api.example.com" }
81
81
  store items: memory("items")
@@ -85,7 +85,7 @@ const program = parse(`
85
85
  }
86
86
  run Fetch
87
87
  }
88
- `);
88
+ `;
89
89
 
90
90
  const result = await execute(source, {
91
91
  auth: { API: { type: 'bearer', token: 'your-token' } }
@@ -213,6 +213,26 @@ mission DurablePipeline {
213
213
  }
214
214
  ```
215
215
 
216
+ #### Durability guarantees
217
+
218
+ Run a mission as a **durable execution** (`executionLog:`) and an append-only
219
+ event log lets an interrupted run — crash, deploy, `kill -9` — resume exactly
220
+ where it left off:
221
+
222
+ - **Delivery**: at-least-once (a step is never silently dropped).
223
+ - **Effects**: exactly-once where the API honours idempotency keys (mutating
224
+ fetches carry a stable `Idempotency-Key`), otherwise at-least-once + keyed
225
+ store dedup. Exactly-once on replay via recorded effect identity.
226
+ - **Resume across restart**: replay + fold; effects already applied are skipped.
227
+ - **Backends**: in-memory (tests), file (dev), `SqliteExecutionLog`
228
+ (transactional, fsync-backed) for single-process self-hosting, and
229
+ `PostgresExecutionLog` for multi-node.
230
+
231
+ These guarantees are proven by a crash-injection suite that kills the run at
232
+ every event boundary and asserts no lost record and no duplicated effect
233
+ (`npm run test:crash`). See **[DURABILITY.md](./DURABILITY.md)** for the full
234
+ statement and the guarantee → test map.
235
+
216
236
  ## OpenAPI Integration
217
237
 
218
238
  Reqon can consume OpenAPI specs directly, eliminating the need for handwritten SDK code:
@@ -197,6 +197,12 @@ export interface FetchStep {
197
197
  until?: Expression;
198
198
  retry?: RetryConfig;
199
199
  since?: SinceConfig;
200
+ /**
201
+ * Resumable backfill: persist per-page progress to the execution log so a
202
+ * large paginated fetch survives a restart/deploy and continues from the last
203
+ * page. Requires durable mode (an executionLog). Only meaningful with paginate.
204
+ */
205
+ backfill?: boolean;
200
206
  }
201
207
  export interface SinceConfig {
202
208
  /** How to resolve the "since" timestamp */
@@ -229,6 +235,8 @@ export interface RetryConfig {
229
235
  backoff: 'exponential' | 'linear' | 'constant';
230
236
  initialDelay: number;
231
237
  maxDelay?: number;
238
+ /** Per-attempt request timeout in ms; aborts a request that hangs */
239
+ timeout?: number;
232
240
  }
233
241
  export interface ForStep {
234
242
  type: 'ForStep';
@@ -99,6 +99,7 @@ export declare class CircuitBreaker {
99
99
  * Get current status for a source/endpoint
100
100
  */
101
101
  getStatus(source: string, endpoint?: string): CircuitBreakerStatus;
102
+ private statusFromEntry;
102
103
  /**
103
104
  * Force reset a circuit to closed state
104
105
  */
@@ -107,8 +108,18 @@ export declare class CircuitBreaker {
107
108
  * Get all circuit statuses
108
109
  */
109
110
  getAllStatuses(): Map<string, CircuitBreakerStatus>;
111
+ /**
112
+ * Evict stale closed circuits to bound memory. Open and half-open circuits
113
+ * are always retained (they carry active backoff/recovery state).
114
+ *
115
+ * @param maxAgeMs Remove closed entries with no activity for this long.
116
+ * @returns Number of entries evicted.
117
+ */
118
+ evictStale(maxAgeMs?: number): number;
110
119
  private getKey;
111
120
  private createEntry;
121
+ /** Read-only lookup. Never inserts — used by status/probe-check paths. */
122
+ private getEntry;
112
123
  private getOrCreateEntry;
113
124
  private pruneOldFailures;
114
125
  private transitionTo;
@@ -67,7 +67,12 @@ export class CircuitBreaker {
67
67
  * Check if a request can proceed (throws if circuit is open)
68
68
  */
69
69
  canProceed(source, endpoint) {
70
- const entry = this.getOrCreateEntry(source, endpoint);
70
+ // Read-only: an unknown key is implicitly closed and must not be inserted
71
+ // (otherwise every probed endpoint leaks an entry forever).
72
+ const entry = this.getEntry(source, endpoint);
73
+ if (!entry) {
74
+ return true;
75
+ }
71
76
  const now = Date.now();
72
77
  if (entry.state === 'closed') {
73
78
  return true;
@@ -75,13 +80,20 @@ export class CircuitBreaker {
75
80
  if (entry.state === 'open') {
76
81
  const timeSinceOpen = now - (entry.openedAt ?? now);
77
82
  if (timeSinceOpen >= entry.config.resetTimeout) {
78
- // Transition to half-open
83
+ // Transition to half-open and let this single caller be the probe.
79
84
  this.transitionTo(entry, 'half_open', source, endpoint);
85
+ entry.probeInFlight = true;
80
86
  return true;
81
87
  }
82
88
  return false;
83
89
  }
84
- // Half-open: allow the request through for testing
90
+ // Half-open: allow exactly one probe at a time. Every other request is
91
+ // denied until the in-flight probe resolves (success closes the circuit or
92
+ // advances the count; failure re-opens it), preventing a recovery stampede.
93
+ if (entry.probeInFlight) {
94
+ return false;
95
+ }
96
+ entry.probeInFlight = true;
85
97
  return true;
86
98
  }
87
99
  /**
@@ -89,9 +101,11 @@ export class CircuitBreaker {
89
101
  */
90
102
  ensureCanProceed(source, endpoint) {
91
103
  if (!this.canProceed(source, endpoint)) {
92
- const entry = this.getOrCreateEntry(source, endpoint);
104
+ // canProceed only returns false for an existing open/half-open entry.
105
+ const entry = this.getEntry(source, endpoint);
106
+ const config = entry?.config ?? this.defaultConfig;
93
107
  const now = Date.now();
94
- const nextAttemptIn = entry.config.resetTimeout - (now - (entry.openedAt ?? now));
108
+ const nextAttemptIn = config.resetTimeout - (now - (entry?.openedAt ?? now));
95
109
  this.callbacks.onRejected?.({
96
110
  source,
97
111
  endpoint,
@@ -104,13 +118,23 @@ export class CircuitBreaker {
104
118
  * Record a successful request
105
119
  */
106
120
  recordSuccess(source, endpoint) {
107
- const entry = this.getOrCreateEntry(source, endpoint);
121
+ // Read-only lookup: a success on an untracked (implicitly closed) endpoint
122
+ // has nothing to record, so don't allocate an entry for it.
123
+ const entry = this.getEntry(source, endpoint);
124
+ if (!entry) {
125
+ return;
126
+ }
127
+ entry.lastActivity = Date.now();
108
128
  if (entry.state === 'half_open') {
109
129
  entry.successes++;
110
130
  if (entry.successes >= entry.config.successThreshold) {
111
131
  // Recovery successful, close the circuit
112
132
  this.transitionTo(entry, 'closed', source, endpoint);
113
133
  }
134
+ else {
135
+ // Probe succeeded but more are needed; release the slot for the next.
136
+ entry.probeInFlight = false;
137
+ }
114
138
  }
115
139
  else if (entry.state === 'closed') {
116
140
  // Clear old failures from window
@@ -121,15 +145,19 @@ export class CircuitBreaker {
121
145
  * Record a failed request
122
146
  */
123
147
  recordFailure(source, endpoint, statusCode, isNetworkError = false) {
124
- const entry = this.getOrCreateEntry(source, endpoint);
125
- const config = entry.config;
148
+ // Resolve config without forcing an entry ignored failures (e.g. a 404)
149
+ // must not allocate a permanent circuit for an otherwise-untracked endpoint.
150
+ const existing = this.getEntry(source, endpoint);
151
+ const config = existing?.config ?? this.getEntry(source)?.config ?? this.defaultConfig;
126
152
  // Check if this failure type should be counted
127
153
  const isFailureStatus = statusCode !== undefined && config.failureStatusCodes.includes(statusCode);
128
154
  const shouldCount = isFailureStatus || (isNetworkError && config.countNetworkErrors);
129
155
  if (!shouldCount) {
130
156
  return;
131
157
  }
158
+ const entry = existing ?? this.getOrCreateEntry(source, endpoint);
132
159
  const now = Date.now();
160
+ entry.lastActivity = now;
133
161
  if (entry.state === 'half_open') {
134
162
  // Any failure in half-open immediately re-opens circuit
135
163
  this.transitionTo(entry, 'open', source, endpoint, 'Failure during recovery attempt');
@@ -151,8 +179,14 @@ export class CircuitBreaker {
151
179
  * Get current status for a source/endpoint
152
180
  */
153
181
  getStatus(source, endpoint) {
154
- const entry = this.getOrCreateEntry(source, endpoint);
155
- const now = Date.now();
182
+ // Read-only: querying an unknown circuit must not create one.
183
+ const entry = this.getEntry(source, endpoint);
184
+ if (!entry) {
185
+ return { state: 'closed', failures: 0, successes: 0, isOpen: false };
186
+ }
187
+ return this.statusFromEntry(entry);
188
+ }
189
+ statusFromEntry(entry) {
156
190
  let nextAttemptTime;
157
191
  if (entry.state === 'open' && entry.openedAt) {
158
192
  const nextAttemptMs = entry.openedAt + entry.config.resetTimeout;
@@ -181,6 +215,8 @@ export class CircuitBreaker {
181
215
  entry.failureTimestamps = [];
182
216
  entry.lastFailureTime = undefined;
183
217
  entry.openedAt = undefined;
218
+ entry.probeInFlight = false;
219
+ entry.lastActivity = Date.now();
184
220
  if (previousState !== 'closed') {
185
221
  this.callbacks.onClose?.({
186
222
  source,
@@ -198,12 +234,37 @@ export class CircuitBreaker {
198
234
  */
199
235
  getAllStatuses() {
200
236
  const result = new Map();
237
+ // Build the status from each entry directly. Splitting the key on ':' would
238
+ // corrupt URL-shaped keys (e.g. `https://x` → source `https`), and routing
239
+ // back through getStatus is needless. We never mutate `circuits` here.
201
240
  for (const [key, entry] of this.circuits) {
202
- const [source, endpoint] = key.split(':');
203
- result.set(key, this.getStatus(source, endpoint === '' ? undefined : endpoint));
241
+ result.set(key, this.statusFromEntry(entry));
204
242
  }
205
243
  return result;
206
244
  }
245
+ /**
246
+ * Evict stale closed circuits to bound memory. Open and half-open circuits
247
+ * are always retained (they carry active backoff/recovery state).
248
+ *
249
+ * @param maxAgeMs Remove closed entries with no activity for this long.
250
+ * @returns Number of entries evicted.
251
+ */
252
+ evictStale(maxAgeMs = this.defaultConfig.failureWindow) {
253
+ const cutoff = Date.now() - maxAgeMs;
254
+ let evicted = 0;
255
+ // Collect first, then delete — never mutate the map mid-iteration.
256
+ const toDelete = [];
257
+ for (const [key, entry] of this.circuits) {
258
+ if (entry.state === 'closed' && entry.lastActivity < cutoff) {
259
+ toDelete.push(key);
260
+ }
261
+ }
262
+ for (const key of toDelete) {
263
+ this.circuits.delete(key);
264
+ evicted++;
265
+ }
266
+ return evicted;
267
+ }
207
268
  getKey(source, endpoint) {
208
269
  return endpoint ? `${source}:${endpoint}` : source;
209
270
  }
@@ -213,9 +274,15 @@ export class CircuitBreaker {
213
274
  failures: 0,
214
275
  successes: 0,
215
276
  failureTimestamps: [],
277
+ probeInFlight: false,
278
+ lastActivity: Date.now(),
216
279
  config,
217
280
  };
218
281
  }
282
+ /** Read-only lookup. Never inserts — used by status/probe-check paths. */
283
+ getEntry(source, endpoint) {
284
+ return this.circuits.get(this.getKey(source, endpoint));
285
+ }
219
286
  getOrCreateEntry(source, endpoint) {
220
287
  const key = this.getKey(source, endpoint);
221
288
  let entry = this.circuits.get(key);
@@ -237,6 +304,10 @@ export class CircuitBreaker {
237
304
  transitionTo(entry, newState, source, endpoint, reason) {
238
305
  const previousState = entry.state;
239
306
  entry.state = newState;
307
+ entry.lastActivity = Date.now();
308
+ // Any state change clears the half-open probe slot; canProceed re-claims it
309
+ // when it admits the next probe.
310
+ entry.probeInFlight = false;
240
311
  const event = {
241
312
  source,
242
313
  endpoint,
@@ -33,12 +33,17 @@ export interface LoadEnvResult {
33
33
  */
34
34
  export declare function loadEnv(options?: CredentialsConfig): LoadEnvResult;
35
35
  /**
36
- * Resolve environment variable references in a string
36
+ * Resolve environment variable references in a string.
37
37
  *
38
38
  * Supports:
39
39
  * - $VAR_NAME
40
40
  * - ${VAR_NAME}
41
41
  * - ${VAR_NAME:-default}
42
+ *
43
+ * A reference with no value and no default is treated as *required* and throws,
44
+ * rather than silently resolving to '' (which previously let requests go out
45
+ * with an empty Authorization header). Use the `${VAR:-}` form to opt a
46
+ * reference into being optional with an empty fallback.
42
47
  */
43
48
  export declare function resolveEnvString(value: string): string;
44
49
  /**
@@ -55,12 +55,17 @@ export function loadEnv(options = {}) {
55
55
  };
56
56
  }
57
57
  /**
58
- * Resolve environment variable references in a string
58
+ * Resolve environment variable references in a string.
59
59
  *
60
60
  * Supports:
61
61
  * - $VAR_NAME
62
62
  * - ${VAR_NAME}
63
63
  * - ${VAR_NAME:-default}
64
+ *
65
+ * A reference with no value and no default is treated as *required* and throws,
66
+ * rather than silently resolving to '' (which previously let requests go out
67
+ * with an empty Authorization header). Use the `${VAR:-}` form to opt a
68
+ * reference into being optional with an empty fallback.
64
69
  */
65
70
  export function resolveEnvString(value) {
66
71
  // Pattern for ${VAR:-default} or ${VAR}
@@ -75,15 +80,18 @@ export function resolveEnvString(value) {
75
80
  return envValue;
76
81
  }
77
82
  if (defaultValue !== undefined) {
83
+ // Includes the empty-default escape hatch `${VAR:-}`.
78
84
  return defaultValue;
79
85
  }
80
- // Return empty string if no value and no default
81
- return '';
86
+ throw new Error(`unresolved required credential ${varName}`);
82
87
  });
83
88
  // Then resolve $VAR patterns (only if not already resolved)
84
89
  result = result.replace(simplePattern, (_match, varName) => {
85
90
  const envValue = process.env[varName];
86
- return envValue !== undefined ? envValue : '';
91
+ if (envValue !== undefined) {
92
+ return envValue;
93
+ }
94
+ throw new Error(`unresolved required credential ${varName}`);
87
95
  });
88
96
  return result;
89
97
  }
@@ -60,10 +60,16 @@ export class OAuth2AuthProvider {
60
60
  body,
61
61
  });
62
62
  if (!response.ok) {
63
- const error = await response.text();
64
- throw new Error(`Token refresh failed: ${response.status} ${error}`);
63
+ // Do not include the response body: token endpoints can echo the
64
+ // submitted client secret / refresh token in error responses.
65
+ throw new Error(`Token refresh failed: ${response.status} ${response.statusText}`);
65
66
  }
66
67
  const data = (await response.json());
68
+ // A 2xx with no access_token would otherwise propagate as `Bearer undefined`
69
+ // on every subsequent request. Fail loudly instead.
70
+ if (!data.access_token) {
71
+ throw new Error(`Token refresh for connection "${this.connectionId}" returned no access_token`);
72
+ }
67
73
  const newTokens = {
68
74
  accessToken: data.access_token,
69
75
  refreshToken: data.refresh_token ?? tokens.refreshToken, // Keep old if not rotated
@@ -80,7 +86,11 @@ export class OAuth2AuthProvider {
80
86
  }
81
87
  shouldRefresh(tokens) {
82
88
  if (!tokens.expiresAt) {
83
- return false; // No expiry info, assume valid
89
+ // No expiry metadata: we cannot know when the token lapses, so we treat it
90
+ // as still valid and let a downstream 401 trigger an explicit
91
+ // refreshToken() call. Proactively refreshing here would burn the refresh
92
+ // token (and break non-expiring tokens) on every single request.
93
+ return false;
84
94
  }
85
95
  const now = Date.now();
86
96
  const expiresAt = tokens.expiresAt.getTime();
@@ -32,7 +32,14 @@ export declare class AdaptiveRateLimiter implements RateLimiter {
32
32
  maxStaleAgeMs?: number;
33
33
  });
34
34
  /**
35
- * Clean up stale entries from the state map to prevent memory leaks
35
+ * Clean up stale entries from the state map to prevent memory leaks.
36
+ *
37
+ * Eviction is driven by `lastRequestAt`, not `resetAt`. Most of the leak came
38
+ * from keys whose API sends no reset header: `resetAt` is never set, so the
39
+ * old reset-passed condition could never fire and those keys lived forever.
40
+ * Now any key idle longer than the stale threshold is evicted regardless of
41
+ * `resetAt`, except while an active `retryAfter` backoff is still pending
42
+ * (that state must survive so we keep honouring the 429).
36
43
  */
37
44
  private cleanup;
38
45
  /**
@@ -52,7 +52,14 @@ export class AdaptiveRateLimiter {
52
52
  this.maxStaleAgeMs = options.maxStaleAgeMs ?? RATE_LIMIT_DEFAULTS.MAX_STALE_AGE_MS;
53
53
  }
54
54
  /**
55
- * Clean up stale entries from the state map to prevent memory leaks
55
+ * Clean up stale entries from the state map to prevent memory leaks.
56
+ *
57
+ * Eviction is driven by `lastRequestAt`, not `resetAt`. Most of the leak came
58
+ * from keys whose API sends no reset header: `resetAt` is never set, so the
59
+ * old reset-passed condition could never fire and those keys lived forever.
60
+ * Now any key idle longer than the stale threshold is evicted regardless of
61
+ * `resetAt`, except while an active `retryAfter` backoff is still pending
62
+ * (that state must survive so we keep honouring the 429).
56
63
  */
57
64
  cleanup() {
58
65
  const now = Date.now();
@@ -65,13 +72,14 @@ export class AdaptiveRateLimiter {
65
72
  const nowDate = new Date(now);
66
73
  const staleThreshold = new Date(now - this.maxStaleAgeMs);
67
74
  for (const [key, state] of this.state) {
68
- // Remove if:
69
- // 1. Reset time has passed AND no recent requests
70
- // 2. Last request is older than max stale age
71
- const isResetPassed = state.resetAt && state.resetAt < nowDate;
72
- const isRetryPassed = !state.retryAfter || state.retryAfter < nowDate;
73
- const isStale = state.lastRequestAt && state.lastRequestAt < staleThreshold;
74
- if ((isResetPassed && isRetryPassed && isStale) || (!state.lastRequestAt && isResetPassed)) {
75
+ // A pending retry-after backoff must never be evicted out from under us.
76
+ const isRetryPending = state.retryAfter !== undefined && state.retryAfter >= nowDate;
77
+ if (isRetryPending)
78
+ continue;
79
+ // Stale if the last request is older than the threshold, or if no request
80
+ // was ever recorded for this key.
81
+ const isStale = !state.lastRequestAt || state.lastRequestAt < staleThreshold;
82
+ if (isStale) {
75
83
  this.state.delete(key);
76
84
  }
77
85
  }
@@ -251,7 +259,16 @@ export class AdaptiveRateLimiter {
251
259
  }
252
260
  state.lastRequestAt = now;
253
261
  this.state.set(key, state);
254
- // Periodically clean up stale entries (every N responses)
262
+ // Trigger a stale-entry sweep every N responses. The sweep itself is rate-
263
+ // limited by CLEANUP_INTERVAL_MS / MAX_ENTRIES_BEFORE_CLEANUP inside
264
+ // cleanup(), so this counter only decides how often we *consider* sweeping.
265
+ //
266
+ // TODO: this limiter is purely reactive — it learns limits from response
267
+ // headers after the fact. Add proactive in-flight token accounting (track
268
+ // tokens consumed by requests that have been issued but not yet returned a
269
+ // response) so concurrent callers can't collectively blow past the limit
270
+ // between header updates. That is a larger redesign; the eviction leak fix
271
+ // above is independent of it.
255
272
  this.responsesSinceCleanup++;
256
273
  if (this.responsesSinceCleanup >= RATE_LIMIT_DEFAULTS.CLEANUP_CHECK_INTERVAL) {
257
274
  this.responsesSinceCleanup = 0;
@@ -266,7 +283,10 @@ export class AdaptiveRateLimiter {
266
283
  return { isLimited: false };
267
284
  }
268
285
  const isLimited = (state.retryAfter && state.retryAfter > now) ||
269
- (state.remaining !== undefined && state.remaining <= 0 && state.resetAt && state.resetAt > now);
286
+ (state.remaining !== undefined &&
287
+ state.remaining <= 0 &&
288
+ state.resetAt &&
289
+ state.resetAt > now);
270
290
  let resetInSeconds;
271
291
  if (isLimited && state.resetAt) {
272
292
  resetInSeconds = Math.ceil((state.resetAt.getTime() - now.getTime()) / 1000);
@@ -103,7 +103,14 @@ export class FileTokenStore {
103
103
  for (const [key, value] of this.cache) {
104
104
  data[key] = value;
105
105
  }
106
- await fs.writeFile(this.filePath, JSON.stringify(data, null, 2), 'utf-8');
106
+ // Owner-only (0o600): the file holds OAuth tokens and must not be
107
+ // world-readable. `mode` only applies on creation, so chmod an existing
108
+ // file too (in case it was created before this fix or under a loose umask).
109
+ await fs.writeFile(this.filePath, JSON.stringify(data, null, 2), {
110
+ encoding: 'utf-8',
111
+ mode: 0o600,
112
+ });
113
+ await fs.chmod(this.filePath, 0o600).catch(() => { });
107
114
  }
108
115
  async get(connectionId) {
109
116
  const map = await this.load();
package/dist/cli.d.ts CHANGED
@@ -12,4 +12,14 @@
12
12
  * - ./debug/cli-debugger.ts - interactive debugging
13
13
  * ---
14
14
  */
15
- export {};
15
+ /**
16
+ * Load and resolve an --auth credentials file, turning low-level fs/JSON
17
+ * failures into clean, user-facing messages instead of raw stack traces.
18
+ */
19
+ export declare function loadAuthFile(rawPath: string): Promise<Record<string, unknown>>;
20
+ /**
21
+ * Print an error in a clean, user-facing form (no stack trace). ReqonErrors
22
+ * get their rich source-context formatting; everything else gets a one-line
23
+ * message.
24
+ */
25
+ export declare function reportFatalError(error: unknown): void;
package/dist/cli.js CHANGED
@@ -14,11 +14,57 @@
14
14
  */
15
15
  import { readFile, writeFile, mkdir } from 'node:fs/promises';
16
16
  import { resolve, dirname } from 'node:path';
17
+ import { pathToFileURL } from 'node:url';
17
18
  import { fromPath, Scheduler, loadMission } from './index.js';
18
19
  import { ReqonError } from './errors/index.js';
19
20
  import { loadEnv, loadCredentials } from './auth/credentials.js';
20
21
  import { WebhookServer } from './webhook/index.js';
21
22
  import { ControlServer } from './control/index.js';
23
+ /**
24
+ * Load and resolve an --auth credentials file, turning low-level fs/JSON
25
+ * failures into clean, user-facing messages instead of raw stack traces.
26
+ */
27
+ export async function loadAuthFile(rawPath) {
28
+ const authPath = resolve(rawPath);
29
+ let authContent;
30
+ try {
31
+ authContent = await readFile(authPath, 'utf-8');
32
+ }
33
+ catch (err) {
34
+ if (err.code === 'ENOENT') {
35
+ throw new Error(`auth file not found: ${authPath}`);
36
+ }
37
+ if (err.code === 'EISDIR') {
38
+ throw new Error(`auth path is a directory, not a file: ${authPath}`);
39
+ }
40
+ throw new Error(`could not read auth file ${authPath}: ${err.message}`);
41
+ }
42
+ let rawAuth;
43
+ try {
44
+ rawAuth = JSON.parse(authContent);
45
+ }
46
+ catch {
47
+ throw new Error(`auth file is not valid JSON: ${authPath}`);
48
+ }
49
+ // Resolve env var references in the auth config
50
+ return loadCredentials(rawAuth);
51
+ }
52
+ /**
53
+ * Print an error in a clean, user-facing form (no stack trace). ReqonErrors
54
+ * get their rich source-context formatting; everything else gets a one-line
55
+ * message.
56
+ */
57
+ export function reportFatalError(error) {
58
+ if (error instanceof ReqonError) {
59
+ console.error(error.format());
60
+ }
61
+ else if (error instanceof Error) {
62
+ console.error(`Error: ${error.message}`);
63
+ }
64
+ else {
65
+ console.error(`Error: ${String(error)}`);
66
+ }
67
+ }
22
68
  async function main() {
23
69
  const args = process.argv.slice(2);
24
70
  if (args.length === 0 || args.includes('--help') || args.includes('-h')) {
@@ -31,6 +77,7 @@ Usage:
31
77
  Options:
32
78
  --dry-run Run without making actual HTTP requests
33
79
  --verbose Enable verbose logging
80
+ --dev Development mode: let sql/nosql stores fall back to local JSON files
34
81
  --auth <file> JSON file with auth credentials (supports env var interpolation)
35
82
  --env <file> Path to .env file (default: .env in current directory)
36
83
  --output <path> Export stores to JSON (file or directory)
@@ -77,6 +124,7 @@ Control Server:
77
124
  const filePath = args[0];
78
125
  const dryRun = args.includes('--dry-run');
79
126
  const verbose = args.includes('--verbose');
127
+ const devMode = args.includes('--dev');
80
128
  const daemon = args.includes('--daemon');
81
129
  const once = args.includes('--once');
82
130
  const webhookEnabled = args.includes('--webhook');
@@ -119,11 +167,7 @@ Control Server:
119
167
  let auth;
120
168
  const authIndex = args.indexOf('--auth');
121
169
  if (authIndex !== -1 && args[authIndex + 1]) {
122
- const authPath = resolve(args[authIndex + 1]);
123
- const authContent = await readFile(authPath, 'utf-8');
124
- const rawAuth = JSON.parse(authContent);
125
- // Resolve env var references in the auth config
126
- auth = loadCredentials(rawAuth);
170
+ auth = await loadAuthFile(args[authIndex + 1]);
127
171
  }
128
172
  let outputPath;
129
173
  const outputIndex = args.indexOf('--output');
@@ -174,6 +218,7 @@ Control Server:
174
218
  const result = await fromPath(filePath, {
175
219
  dryRun,
176
220
  verbose,
221
+ developmentMode: devMode,
177
222
  auth: auth,
178
223
  webhookServer,
179
224
  debugController,
@@ -347,4 +392,18 @@ async function runDaemon(filePath, options) {
347
392
  function formatTime(date) {
348
393
  return date.toISOString().replace('T', ' ').substring(0, 19);
349
394
  }
350
- main();
395
+ // Only auto-run when invoked directly as the CLI entry point, so the module can
396
+ // be imported in tests without executing main() against the test runner's argv.
397
+ const isMain = typeof process.argv[1] === 'string' && import.meta.url === pathToFileURL(process.argv[1]).href;
398
+ if (isMain) {
399
+ // A rejected promise anywhere (including outside main's try/catch) should
400
+ // surface as a clean message and a non-zero exit, never a raw stack trace.
401
+ process.on('unhandledRejection', (reason) => {
402
+ reportFatalError(reason);
403
+ process.exit(1);
404
+ });
405
+ main().catch((error) => {
406
+ reportFatalError(error);
407
+ process.exit(1);
408
+ });
409
+ }
@@ -17,6 +17,8 @@ export declare const HTTP_RETRY_DEFAULTS: {
17
17
  readonly MAX_DELAY_MS: 30000;
18
18
  /** Backoff strategy: 'exponential', 'linear', or 'constant' */
19
19
  readonly BACKOFF: "exponential";
20
+ /** Per-attempt request timeout in milliseconds (aborts a hung request) */
21
+ readonly TIMEOUT_MS: 30000;
20
22
  };
21
23
  /**
22
24
  * Default HTTP headers
@@ -69,12 +71,16 @@ export declare const CIRCUIT_BREAKER_DEFAULTS: {
69
71
  export declare const WEBHOOK_DEFAULTS: {
70
72
  /** Default port for webhook server */
71
73
  readonly PORT: 3000;
72
- /** Default host binding */
73
- readonly HOST: "0.0.0.0";
74
+ /** Default host binding — loopback only; opt into 0.0.0.0 explicitly. */
75
+ readonly HOST: "127.0.0.1";
74
76
  /** Default timeout for wait steps (5 minutes in ms) */
75
77
  readonly DEFAULT_TIMEOUT_MS: 300000;
76
78
  /** Cleanup interval for expired registrations (1 minute in ms) */
77
79
  readonly CLEANUP_INTERVAL_MS: 60000;
80
+ /** Maximum accepted request body size in bytes (1 MiB) */
81
+ readonly MAX_BODY_BYTES: number;
82
+ /** Idle socket timeout in ms; drops slow-drip connections */
83
+ readonly SOCKET_TIMEOUT_MS: 30000;
78
84
  };
79
85
  /**
80
86
  * Default store configuration
@@ -100,8 +106,13 @@ export declare const SCHEDULER_DEFAULTS: {
100
106
  * Default execution configuration
101
107
  */
102
108
  export declare const EXECUTION_DEFAULTS: {
103
- /** Whether development mode is enabled by default (uses file stores) */
104
- readonly DEVELOPMENT_MODE: true;
109
+ /**
110
+ * Whether development mode is enabled by default. When true, sql/nosql
111
+ * stores fall back to local JSON files; default false so a mission that
112
+ * declares `store sql`/`nosql` errors loudly instead of silently writing
113
+ * to disk.
114
+ */
115
+ readonly DEVELOPMENT_MODE: false;
105
116
  /** Whether to persist execution state by default */
106
117
  readonly PERSIST_STATE: false;
107
118
  };