reqon-dsl 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -3
- package/dist/ast/nodes.d.ts +8 -0
- package/dist/auth/circuit-breaker.d.ts +11 -0
- package/dist/auth/circuit-breaker.js +83 -12
- package/dist/auth/credentials.d.ts +6 -1
- package/dist/auth/credentials.js +12 -4
- package/dist/auth/oauth2-provider.js +13 -3
- package/dist/auth/rate-limiter.d.ts +8 -1
- package/dist/auth/rate-limiter.js +30 -10
- package/dist/auth/token-store.js +8 -1
- package/dist/cli.d.ts +11 -1
- package/dist/cli.js +65 -6
- package/dist/config/constants.d.ts +15 -4
- package/dist/config/constants.js +15 -4
- package/dist/control/server.d.ts +17 -0
- package/dist/control/server.js +82 -5
- package/dist/control/types.d.ts +6 -0
- package/dist/debug/cli-debugger.js +8 -3
- package/dist/execution/store.js +2 -2
- package/dist/execution-log/events.d.ts +125 -0
- package/dist/execution-log/events.js +17 -0
- package/dist/execution-log/fold.d.ts +38 -0
- package/dist/execution-log/fold.js +54 -0
- package/dist/execution-log/index.d.ts +18 -0
- package/dist/execution-log/index.js +6 -0
- package/dist/execution-log/postgres-store.d.ts +36 -0
- package/dist/execution-log/postgres-store.js +108 -0
- package/dist/execution-log/resume.d.ts +11 -0
- package/dist/execution-log/resume.js +5 -0
- package/dist/execution-log/sqlite-store.d.ts +16 -0
- package/dist/execution-log/sqlite-store.js +101 -0
- package/dist/execution-log/store.d.ts +72 -0
- package/dist/execution-log/store.js +182 -0
- package/dist/index.d.ts +4 -3
- package/dist/index.js +4 -3
- package/dist/interpreter/context.d.ts +15 -0
- package/dist/interpreter/context.js +3 -0
- package/dist/interpreter/evaluator.js +38 -8
- package/dist/interpreter/executor.d.ts +63 -1
- package/dist/interpreter/executor.js +406 -30
- package/dist/interpreter/fetch-handler.d.ts +39 -1
- package/dist/interpreter/fetch-handler.js +84 -15
- package/dist/interpreter/http.d.ts +31 -2
- package/dist/interpreter/http.js +187 -26
- package/dist/interpreter/index.d.ts +3 -3
- package/dist/interpreter/index.js +3 -3
- package/dist/interpreter/pagination.d.ts +1 -1
- package/dist/interpreter/pagination.js +7 -1
- package/dist/interpreter/step-handlers/for-handler.d.ts +3 -0
- package/dist/interpreter/step-handlers/for-handler.js +18 -3
- package/dist/interpreter/step-handlers/match-handler.js +5 -2
- package/dist/interpreter/step-handlers/store-handler.d.ts +7 -1
- package/dist/interpreter/step-handlers/store-handler.js +25 -16
- package/dist/interpreter/step-handlers/validate-handler.js +4 -1
- package/dist/interpreter/step-handlers/webhook-handler.d.ts +1 -0
- package/dist/interpreter/step-handlers/webhook-handler.js +13 -3
- package/dist/interpreter/store-manager.d.ts +1 -1
- package/dist/interpreter/store-manager.js +5 -1
- package/dist/loader/index.js +5 -8
- package/dist/mcp/sandbox.d.ts +41 -0
- package/dist/mcp/sandbox.js +76 -0
- package/dist/mcp/server.js +62 -9
- package/dist/oas/loader.d.ts +13 -1
- package/dist/oas/loader.js +25 -3
- package/dist/oas/mock-generator.js +13 -4
- package/dist/oas/validator.js +45 -5
- package/dist/observability/events.d.ts +6 -2
- package/dist/observability/events.js +0 -5
- package/dist/observability/logger.js +17 -10
- package/dist/observability/otel.d.ts +8 -0
- package/dist/observability/otel.js +45 -10
- package/dist/parser/action-parser.js +2 -2
- package/dist/parser/base.d.ts +7 -0
- package/dist/parser/base.js +11 -0
- package/dist/parser/expressions.d.ts +1 -0
- package/dist/parser/expressions.js +17 -4
- package/dist/parser/fetch-parser.js +13 -2
- package/dist/pause/index.d.ts +1 -0
- package/dist/pause/index.js +1 -0
- package/dist/pause/log-store.d.ts +33 -0
- package/dist/pause/log-store.js +98 -0
- package/dist/pause/manager.d.ts +12 -0
- package/dist/pause/manager.js +77 -28
- package/dist/pause/store.js +5 -3
- package/dist/scheduler/cron-parser.d.ts +10 -3
- package/dist/scheduler/cron-parser.js +227 -48
- package/dist/scheduler/scheduler.js +56 -22
- package/dist/stores/factory.d.ts +6 -0
- package/dist/stores/factory.js +11 -1
- package/dist/stores/file.js +9 -17
- package/dist/stores/memory.js +3 -12
- package/dist/stores/postgrest.d.ts +28 -0
- package/dist/stores/postgrest.js +84 -37
- package/dist/sync/index.d.ts +3 -2
- package/dist/sync/index.js +2 -1
- package/dist/sync/log-store.d.ts +30 -0
- package/dist/sync/log-store.js +45 -0
- package/dist/sync/store.js +1 -1
- package/dist/trace/index.d.ts +2 -0
- package/dist/trace/index.js +1 -0
- package/dist/trace/log-view.d.ts +57 -0
- package/dist/trace/log-view.js +76 -0
- package/dist/trace/recorder.d.ts +5 -1
- package/dist/trace/recorder.js +19 -6
- package/dist/trace/store.d.ts +6 -0
- package/dist/trace/store.js +47 -22
- package/dist/utils/deep-merge.d.ts +10 -0
- package/dist/utils/deep-merge.js +23 -0
- package/dist/utils/file.d.ts +13 -4
- package/dist/utils/file.js +70 -12
- package/dist/utils/index.d.ts +1 -1
- package/dist/utils/index.js +1 -1
- package/dist/utils/long-timeout.d.ts +19 -0
- package/dist/utils/long-timeout.js +33 -0
- package/dist/utils/path.d.ts +22 -1
- package/dist/utils/path.js +46 -1
- package/dist/utils/redact.d.ts +22 -0
- package/dist/utils/redact.js +42 -0
- package/dist/webhook/server.d.ts +9 -0
- package/dist/webhook/server.js +115 -30
- package/dist/webhook/types.d.ts +9 -1
- package/package.json +22 -4
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { reduceCheckpoints } from './store.js';
|
|
2
|
+
const UNIQUE_VIOLATION = '23505';
|
|
3
|
+
const MAX_APPEND_ATTEMPTS = 8;
|
|
4
|
+
export class PostgresExecutionLog {
|
|
5
|
+
connectionString;
|
|
6
|
+
pool;
|
|
7
|
+
ready;
|
|
8
|
+
table;
|
|
9
|
+
constructor(connectionString, options = {}) {
|
|
10
|
+
this.connectionString = connectionString;
|
|
11
|
+
this.table = options.table ?? 'reqon_execution_events';
|
|
12
|
+
// The table name is interpolated into SQL (identifiers can't be bound), so
|
|
13
|
+
// constrain it to a safe identifier to keep that interpolation injection-free.
|
|
14
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(this.table)) {
|
|
15
|
+
throw new Error(`Invalid table name: ${this.table}`);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
async ensure() {
|
|
19
|
+
if (this.pool)
|
|
20
|
+
return this.pool;
|
|
21
|
+
if (!this.ready)
|
|
22
|
+
this.ready = this.init();
|
|
23
|
+
return this.ready;
|
|
24
|
+
}
|
|
25
|
+
async init() {
|
|
26
|
+
let Pool;
|
|
27
|
+
try {
|
|
28
|
+
// Non-literal specifier: keeps the driver an optional dependency with no
|
|
29
|
+
// compile-time type coupling (no @types/pg required).
|
|
30
|
+
const specifier = 'pg';
|
|
31
|
+
const mod = (await import(specifier));
|
|
32
|
+
const ctor = mod.Pool ?? mod.default?.Pool;
|
|
33
|
+
if (!ctor)
|
|
34
|
+
throw new Error('pg.Pool not found');
|
|
35
|
+
Pool = ctor;
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
throw new Error("PostgresExecutionLog requires the optional peer dependency 'pg'. " +
|
|
39
|
+
'Install it with: npm install pg');
|
|
40
|
+
}
|
|
41
|
+
const pool = new Pool({ connectionString: this.connectionString });
|
|
42
|
+
await pool.query(`CREATE TABLE IF NOT EXISTS ${this.table} (
|
|
43
|
+
execution_id text NOT NULL,
|
|
44
|
+
seq integer NOT NULL,
|
|
45
|
+
at text NOT NULL,
|
|
46
|
+
data jsonb NOT NULL,
|
|
47
|
+
PRIMARY KEY (execution_id, seq)
|
|
48
|
+
)`);
|
|
49
|
+
this.pool = pool;
|
|
50
|
+
return pool;
|
|
51
|
+
}
|
|
52
|
+
async append(event) {
|
|
53
|
+
const pool = await this.ensure();
|
|
54
|
+
const at = new Date().toISOString();
|
|
55
|
+
const data = JSON.stringify(event);
|
|
56
|
+
for (let attempt = 0; attempt < MAX_APPEND_ATTEMPTS; attempt++) {
|
|
57
|
+
try {
|
|
58
|
+
const res = await pool.query(`INSERT INTO ${this.table} (execution_id, seq, at, data)
|
|
59
|
+
VALUES (
|
|
60
|
+
$1,
|
|
61
|
+
(SELECT COALESCE(MAX(seq) + 1, 0) FROM ${this.table} WHERE execution_id = $1),
|
|
62
|
+
$2,
|
|
63
|
+
$3::jsonb
|
|
64
|
+
)
|
|
65
|
+
RETURNING seq`, [event.executionId, at, data]);
|
|
66
|
+
return { ...event, seq: res.rows[0].seq, at };
|
|
67
|
+
}
|
|
68
|
+
catch (err) {
|
|
69
|
+
// A concurrent appender claimed this seq first; retry against the new max.
|
|
70
|
+
if (err.code === UNIQUE_VIOLATION)
|
|
71
|
+
continue;
|
|
72
|
+
throw err;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
throw new Error(`append failed after ${MAX_APPEND_ATTEMPTS} attempts for execution ${event.executionId}`);
|
|
76
|
+
}
|
|
77
|
+
async read(executionId) {
|
|
78
|
+
const pool = await this.ensure();
|
|
79
|
+
const res = await pool.query(`SELECT seq, at, data FROM ${this.table} WHERE execution_id = $1 ORDER BY seq ASC`, [executionId]);
|
|
80
|
+
// jsonb comes back already parsed; overlay the assigned seq + recorded time.
|
|
81
|
+
return res.rows.map((r) => ({ ...r.data, seq: r.seq, at: r.at }));
|
|
82
|
+
}
|
|
83
|
+
async listCheckpoints(mission) {
|
|
84
|
+
const pool = await this.ensure();
|
|
85
|
+
const res = await pool.query(`SELECT seq, at, data FROM ${this.table} WHERE data->>'type' = 'checkpoint.advanced'`);
|
|
86
|
+
const events = res.rows.map((r) => ({ ...r.data, seq: r.seq, at: r.at }));
|
|
87
|
+
return reduceCheckpoints(events, mission);
|
|
88
|
+
}
|
|
89
|
+
async listPauses() {
|
|
90
|
+
const pool = await this.ensure();
|
|
91
|
+
const res = await pool.query(`SELECT seq, at, data FROM ${this.table}
|
|
92
|
+
WHERE data->>'type' IN ('pause.created', 'pause.resumed')`);
|
|
93
|
+
return res.rows.map((r) => ({ ...r.data, seq: r.seq, at: r.at }));
|
|
94
|
+
}
|
|
95
|
+
/** Drop all rows. Intended for test setup, not production use. */
|
|
96
|
+
async reset() {
|
|
97
|
+
const pool = await this.ensure();
|
|
98
|
+
await pool.query(`TRUNCATE ${this.table}`);
|
|
99
|
+
}
|
|
100
|
+
/** Release the connection pool. */
|
|
101
|
+
async close() {
|
|
102
|
+
if (this.pool) {
|
|
103
|
+
await this.pool.end();
|
|
104
|
+
this.pool = undefined;
|
|
105
|
+
this.ready = undefined;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resume helpers: load the current folded state of a run from its log.
|
|
3
|
+
*
|
|
4
|
+
* Resume is "read the log, fold it, continue" — never a second execution. A
|
|
5
|
+
* replaying caller checks {@link FoldedState.appliedEffects} before re-running
|
|
6
|
+
* a side effect, so an effect recorded before a crash is skipped on resume.
|
|
7
|
+
*/
|
|
8
|
+
import type { ExecutionLogStore } from './store.js';
|
|
9
|
+
import { type FoldedState } from './fold.js';
|
|
10
|
+
/** Read and fold an execution's log into its current state. */
|
|
11
|
+
export declare function loadState(store: ExecutionLogStore, executionId: string): Promise<FoldedState>;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { ExecutionEvent, StoredEvent } from './events.js';
|
|
2
|
+
import type { CheckpointRecord, ExecutionLogStore } from './store.js';
|
|
3
|
+
export declare class SqliteExecutionLog implements ExecutionLogStore {
|
|
4
|
+
private path;
|
|
5
|
+
private db?;
|
|
6
|
+
private insertStmt?;
|
|
7
|
+
private readStmt?;
|
|
8
|
+
private checkpointStmt?;
|
|
9
|
+
private pauseStmt?;
|
|
10
|
+
constructor(path?: string);
|
|
11
|
+
private ensureDb;
|
|
12
|
+
append(event: ExecutionEvent): Promise<StoredEvent>;
|
|
13
|
+
read(executionId: string): Promise<StoredEvent[]>;
|
|
14
|
+
listCheckpoints(mission?: string): Promise<CheckpointRecord[]>;
|
|
15
|
+
listPauses(): Promise<StoredEvent[]>;
|
|
16
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SQLite-backed execution log — durable, transactional state for single-process
|
|
3
|
+
* self-hosting (the "SQLite-of-durable-execution" lane).
|
|
4
|
+
*
|
|
5
|
+
* The `better-sqlite3` driver is an *optional* peer dependency: only installed
|
|
6
|
+
* by users who opt into the SQLite backend. It is imported lazily so the core
|
|
7
|
+
* package carries no native dependency, and a missing driver yields a clear,
|
|
8
|
+
* actionable error rather than an opaque module-resolution failure.
|
|
9
|
+
*
|
|
10
|
+
* Durability: WAL mode + `synchronous = NORMAL` gives fsync-backed commits that
|
|
11
|
+
* survive a process crash, and `seq` is assigned atomically inside the INSERT
|
|
12
|
+
* (guarded by a `(execution_id, seq)` primary key) so concurrent appenders
|
|
13
|
+
* cannot collide on a sequence number.
|
|
14
|
+
*/
|
|
15
|
+
import { mkdir } from 'node:fs/promises';
|
|
16
|
+
import { dirname } from 'node:path';
|
|
17
|
+
import { reduceCheckpoints } from './store.js';
|
|
18
|
+
export class SqliteExecutionLog {
|
|
19
|
+
path;
|
|
20
|
+
db;
|
|
21
|
+
insertStmt;
|
|
22
|
+
readStmt;
|
|
23
|
+
checkpointStmt;
|
|
24
|
+
pauseStmt;
|
|
25
|
+
constructor(path = '.reqon-data/execution-log.sqlite') {
|
|
26
|
+
this.path = path;
|
|
27
|
+
}
|
|
28
|
+
async ensureDb() {
|
|
29
|
+
if (this.db)
|
|
30
|
+
return this.db;
|
|
31
|
+
if (this.path !== ':memory:') {
|
|
32
|
+
await mkdir(dirname(this.path), { recursive: true });
|
|
33
|
+
}
|
|
34
|
+
let Database;
|
|
35
|
+
try {
|
|
36
|
+
// Non-literal specifier: keeps the driver an optional dependency with no
|
|
37
|
+
// compile-time type coupling (no @types/better-sqlite3 required).
|
|
38
|
+
const specifier = 'better-sqlite3';
|
|
39
|
+
const mod = (await import(specifier));
|
|
40
|
+
Database = mod.default;
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
throw new Error("SqliteExecutionLog requires the optional peer dependency 'better-sqlite3'. " +
|
|
44
|
+
'Install it with: npm install better-sqlite3');
|
|
45
|
+
}
|
|
46
|
+
const db = new Database(this.path);
|
|
47
|
+
db.pragma('journal_mode = WAL');
|
|
48
|
+
db.pragma('synchronous = NORMAL');
|
|
49
|
+
db.exec(`CREATE TABLE IF NOT EXISTS events (
|
|
50
|
+
execution_id TEXT NOT NULL,
|
|
51
|
+
seq INTEGER NOT NULL,
|
|
52
|
+
at TEXT NOT NULL,
|
|
53
|
+
data TEXT NOT NULL,
|
|
54
|
+
PRIMARY KEY (execution_id, seq)
|
|
55
|
+
)`);
|
|
56
|
+
// seq is computed inside the INSERT so it is assigned atomically, and
|
|
57
|
+
// RETURNING hands it back without a second round-trip.
|
|
58
|
+
this.insertStmt = db.prepare(`INSERT INTO events (execution_id, seq, at, data)
|
|
59
|
+
VALUES (
|
|
60
|
+
@id,
|
|
61
|
+
(SELECT COALESCE(MAX(seq) + 1, 0) FROM events WHERE execution_id = @id),
|
|
62
|
+
@at,
|
|
63
|
+
@data
|
|
64
|
+
)
|
|
65
|
+
RETURNING seq`);
|
|
66
|
+
this.readStmt = db.prepare(`SELECT seq, at, data FROM events WHERE execution_id = @id ORDER BY seq ASC`);
|
|
67
|
+
this.checkpointStmt = db.prepare(`SELECT execution_id, seq, at, data FROM events
|
|
68
|
+
WHERE json_extract(data, '$.type') = 'checkpoint.advanced'`);
|
|
69
|
+
this.pauseStmt = db.prepare(`SELECT execution_id, seq, at, data FROM events
|
|
70
|
+
WHERE json_extract(data, '$.type') IN ('pause.created', 'pause.resumed')`);
|
|
71
|
+
this.db = db;
|
|
72
|
+
return db;
|
|
73
|
+
}
|
|
74
|
+
async append(event) {
|
|
75
|
+
await this.ensureDb();
|
|
76
|
+
const at = new Date().toISOString();
|
|
77
|
+
const { executionId } = event;
|
|
78
|
+
const row = this.insertStmt.get({
|
|
79
|
+
id: executionId,
|
|
80
|
+
at,
|
|
81
|
+
data: JSON.stringify(event),
|
|
82
|
+
});
|
|
83
|
+
return { ...event, seq: row.seq, at };
|
|
84
|
+
}
|
|
85
|
+
async read(executionId) {
|
|
86
|
+
await this.ensureDb();
|
|
87
|
+
const rows = this.readStmt.all({ id: executionId });
|
|
88
|
+
return rows.map((r) => ({ ...JSON.parse(r.data), seq: r.seq, at: r.at }));
|
|
89
|
+
}
|
|
90
|
+
async listCheckpoints(mission) {
|
|
91
|
+
await this.ensureDb();
|
|
92
|
+
const rows = this.checkpointStmt.all({});
|
|
93
|
+
const events = rows.map((r) => ({ ...JSON.parse(r.data), seq: r.seq, at: r.at }));
|
|
94
|
+
return reduceCheckpoints(events, mission);
|
|
95
|
+
}
|
|
96
|
+
async listPauses() {
|
|
97
|
+
await this.ensureDb();
|
|
98
|
+
const rows = this.pauseStmt.all({});
|
|
99
|
+
return rows.map((r) => ({ ...JSON.parse(r.data), seq: r.seq, at: r.at }));
|
|
100
|
+
}
|
|
101
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import type { ExecutionEvent, StoredEvent } from './events.js';
|
|
2
|
+
/**
|
|
3
|
+
* The latest sync checkpoint for a key, materialised across executions — the
|
|
4
|
+
* read model behind log-backed incremental sync. Derived from the most recent
|
|
5
|
+
* `checkpoint.advanced` event for each key.
|
|
6
|
+
*/
|
|
7
|
+
export interface CheckpointRecord {
|
|
8
|
+
key: string;
|
|
9
|
+
syncedAt: string;
|
|
10
|
+
recordCount?: number;
|
|
11
|
+
cursor?: string;
|
|
12
|
+
mission?: string;
|
|
13
|
+
/** The execution that advanced this checkpoint. */
|
|
14
|
+
executionId: string;
|
|
15
|
+
}
|
|
16
|
+
export interface ExecutionLogStore {
|
|
17
|
+
/** Append one event; returns it with its assigned seq and recorded timestamp. */
|
|
18
|
+
append(event: ExecutionEvent): Promise<StoredEvent>;
|
|
19
|
+
/** Read all events for an execution, in append order. */
|
|
20
|
+
read(executionId: string): Promise<StoredEvent[]>;
|
|
21
|
+
/**
|
|
22
|
+
* Latest checkpoint per key across all executions in this store (optionally
|
|
23
|
+
* filtered by mission). The read model for log-backed incremental sync.
|
|
24
|
+
*/
|
|
25
|
+
listCheckpoints(mission?: string): Promise<CheckpointRecord[]>;
|
|
26
|
+
/**
|
|
27
|
+
* All pause lifecycle events (pause.created / pause.resumed) across every
|
|
28
|
+
* execution in this store, in no guaranteed order. The raw input the
|
|
29
|
+
* log-backed pause store folds into pause state.
|
|
30
|
+
*/
|
|
31
|
+
listPauses(): Promise<StoredEvent[]>;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Fold `checkpoint.advanced` events into the latest checkpoint per key. "Latest"
|
|
35
|
+
* is the furthest-advanced sync: ordered by `syncedAt` (so a checkpoint never
|
|
36
|
+
* moves backwards across runs — incremental sync stays monotonic), ties broken
|
|
37
|
+
* by the recording time `at` then `seq`. Shared by every store backend so the
|
|
38
|
+
* read-model semantics stay identical across them.
|
|
39
|
+
*/
|
|
40
|
+
export declare function reduceCheckpoints(events: StoredEvent[], mission?: string): CheckpointRecord[];
|
|
41
|
+
/** In-memory execution log — for tests and ephemeral runs. */
|
|
42
|
+
export declare class MemoryExecutionLog implements ExecutionLogStore {
|
|
43
|
+
private events;
|
|
44
|
+
append(event: ExecutionEvent): Promise<StoredEvent>;
|
|
45
|
+
read(executionId: string): Promise<StoredEvent[]>;
|
|
46
|
+
listCheckpoints(mission?: string): Promise<CheckpointRecord[]>;
|
|
47
|
+
listPauses(): Promise<StoredEvent[]>;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Durable execution log: one append-only JSON-lines file per execution.
|
|
51
|
+
*
|
|
52
|
+
* Appends are O_APPEND single-line writes, so a crash mid-write can at worst
|
|
53
|
+
* leave a torn final line, which {@link read} skips. The log therefore survives
|
|
54
|
+
* a process restart — a resumed run reads its prior events back. Dev-grade: no
|
|
55
|
+
* atomic seq assignment or locking. For a transactional single-process backend
|
|
56
|
+
* use {@link SqliteExecutionLog}; Postgres (multi-node) is still to come.
|
|
57
|
+
*/
|
|
58
|
+
export declare class FileExecutionLog implements ExecutionLogStore {
|
|
59
|
+
private dir;
|
|
60
|
+
/** Cached next-seq per execution, lazily initialised from disk. */
|
|
61
|
+
private counts;
|
|
62
|
+
constructor(dir?: string);
|
|
63
|
+
private pathFor;
|
|
64
|
+
append(event: ExecutionEvent): Promise<StoredEvent>;
|
|
65
|
+
read(executionId: string): Promise<StoredEvent[]>;
|
|
66
|
+
listCheckpoints(mission?: string): Promise<CheckpointRecord[]>;
|
|
67
|
+
listPauses(): Promise<StoredEvent[]>;
|
|
68
|
+
/** Read and parse every execution's events in this directory. */
|
|
69
|
+
private readAll;
|
|
70
|
+
private readRaw;
|
|
71
|
+
private parse;
|
|
72
|
+
}
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Append-only storage for execution events. The contract: events for a given
|
|
3
|
+
* execution are read back in append order with a contiguous `seq` starting at 0.
|
|
4
|
+
*/
|
|
5
|
+
import { appendFile, readFile, readdir, mkdir } from 'node:fs/promises';
|
|
6
|
+
import { join } from 'node:path';
|
|
7
|
+
/** True for the pause lifecycle events the pause read-model folds. */
|
|
8
|
+
function isPauseEvent(event) {
|
|
9
|
+
return event.type === 'pause.created' || event.type === 'pause.resumed';
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Fold `checkpoint.advanced` events into the latest checkpoint per key. "Latest"
|
|
13
|
+
* is the furthest-advanced sync: ordered by `syncedAt` (so a checkpoint never
|
|
14
|
+
* moves backwards across runs — incremental sync stays monotonic), ties broken
|
|
15
|
+
* by the recording time `at` then `seq`. Shared by every store backend so the
|
|
16
|
+
* read-model semantics stay identical across them.
|
|
17
|
+
*/
|
|
18
|
+
export function reduceCheckpoints(events, mission) {
|
|
19
|
+
const latest = new Map();
|
|
20
|
+
const isNewer = (a, prior) => {
|
|
21
|
+
if (a.syncedAt !== prior.syncedAt)
|
|
22
|
+
return a.syncedAt > prior.syncedAt;
|
|
23
|
+
if (a.at !== prior.at)
|
|
24
|
+
return a.at > prior.at;
|
|
25
|
+
return a.seq > prior.seq;
|
|
26
|
+
};
|
|
27
|
+
for (const event of events) {
|
|
28
|
+
if (event.type !== 'checkpoint.advanced')
|
|
29
|
+
continue;
|
|
30
|
+
if (mission !== undefined && event.mission !== mission)
|
|
31
|
+
continue;
|
|
32
|
+
const candidate = {
|
|
33
|
+
syncedAt: event.syncedAt,
|
|
34
|
+
at: event.at,
|
|
35
|
+
seq: event.seq,
|
|
36
|
+
record: {
|
|
37
|
+
key: event.key,
|
|
38
|
+
syncedAt: event.syncedAt,
|
|
39
|
+
recordCount: event.recordCount,
|
|
40
|
+
cursor: event.cursor,
|
|
41
|
+
mission: event.mission,
|
|
42
|
+
executionId: event.executionId,
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
const prior = latest.get(event.key);
|
|
46
|
+
if (!prior || isNewer(candidate, prior))
|
|
47
|
+
latest.set(event.key, candidate);
|
|
48
|
+
}
|
|
49
|
+
return Array.from(latest.values()).map((v) => v.record);
|
|
50
|
+
}
|
|
51
|
+
/** In-memory execution log — for tests and ephemeral runs. */
|
|
52
|
+
export class MemoryExecutionLog {
|
|
53
|
+
events = new Map();
|
|
54
|
+
async append(event) {
|
|
55
|
+
const existing = this.events.get(event.executionId);
|
|
56
|
+
const log = existing ?? [];
|
|
57
|
+
const stored = { ...event, seq: log.length, at: new Date().toISOString() };
|
|
58
|
+
log.push(stored);
|
|
59
|
+
if (!existing)
|
|
60
|
+
this.events.set(event.executionId, log);
|
|
61
|
+
return stored;
|
|
62
|
+
}
|
|
63
|
+
async read(executionId) {
|
|
64
|
+
return [...(this.events.get(executionId) ?? [])];
|
|
65
|
+
}
|
|
66
|
+
async listCheckpoints(mission) {
|
|
67
|
+
const all = [];
|
|
68
|
+
for (const log of this.events.values())
|
|
69
|
+
all.push(...log);
|
|
70
|
+
return reduceCheckpoints(all, mission);
|
|
71
|
+
}
|
|
72
|
+
async listPauses() {
|
|
73
|
+
const pauses = [];
|
|
74
|
+
for (const log of this.events.values())
|
|
75
|
+
pauses.push(...log.filter(isPauseEvent));
|
|
76
|
+
return pauses;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Durable execution log: one append-only JSON-lines file per execution.
|
|
81
|
+
*
|
|
82
|
+
* Appends are O_APPEND single-line writes, so a crash mid-write can at worst
|
|
83
|
+
* leave a torn final line, which {@link read} skips. The log therefore survives
|
|
84
|
+
* a process restart — a resumed run reads its prior events back. Dev-grade: no
|
|
85
|
+
* atomic seq assignment or locking. For a transactional single-process backend
|
|
86
|
+
* use {@link SqliteExecutionLog}; Postgres (multi-node) is still to come.
|
|
87
|
+
*/
|
|
88
|
+
export class FileExecutionLog {
|
|
89
|
+
dir;
|
|
90
|
+
/** Cached next-seq per execution, lazily initialised from disk. */
|
|
91
|
+
counts = new Map();
|
|
92
|
+
constructor(dir = '.reqon-data/execution-log') {
|
|
93
|
+
this.dir = dir;
|
|
94
|
+
}
|
|
95
|
+
pathFor(executionId) {
|
|
96
|
+
// executionId is runtime-generated (uuid); keep the filename simple/safe.
|
|
97
|
+
const safe = executionId.replace(/[^A-Za-z0-9_.-]/g, '_');
|
|
98
|
+
return join(this.dir, `${safe}.jsonl`);
|
|
99
|
+
}
|
|
100
|
+
async append(event) {
|
|
101
|
+
await mkdir(this.dir, { recursive: true });
|
|
102
|
+
let seq = this.counts.get(event.executionId);
|
|
103
|
+
let prefix = '';
|
|
104
|
+
if (seq === undefined) {
|
|
105
|
+
// First append this instance: derive next seq from any persisted events,
|
|
106
|
+
// and detect an unterminated tail left by a crash mid-append. Writing
|
|
107
|
+
// straight after it would concatenate this event onto the torn line and
|
|
108
|
+
// corrupt it; a leading newline terminates the torn line (which read()
|
|
109
|
+
// still skips) and keeps this event a clean, parseable record.
|
|
110
|
+
const raw = await this.readRaw(event.executionId);
|
|
111
|
+
seq = this.parse(raw).length;
|
|
112
|
+
if (raw.length > 0 && !raw.endsWith('\n'))
|
|
113
|
+
prefix = '\n';
|
|
114
|
+
}
|
|
115
|
+
const stored = { ...event, seq, at: new Date().toISOString() };
|
|
116
|
+
// The log can carry resume state (e.g. a pause checkpoint's captured
|
|
117
|
+
// variables), so create files owner-only (0o600) — never world-readable.
|
|
118
|
+
// mode applies on creation; appends to an existing file keep its mode.
|
|
119
|
+
await appendFile(this.pathFor(event.executionId), `${prefix}${JSON.stringify(stored)}\n`, {
|
|
120
|
+
encoding: 'utf-8',
|
|
121
|
+
mode: 0o600,
|
|
122
|
+
});
|
|
123
|
+
this.counts.set(event.executionId, seq + 1);
|
|
124
|
+
return stored;
|
|
125
|
+
}
|
|
126
|
+
async read(executionId) {
|
|
127
|
+
return this.parse(await this.readRaw(executionId));
|
|
128
|
+
}
|
|
129
|
+
async listCheckpoints(mission) {
|
|
130
|
+
return reduceCheckpoints(await this.readAll(), mission);
|
|
131
|
+
}
|
|
132
|
+
async listPauses() {
|
|
133
|
+
return (await this.readAll()).filter(isPauseEvent);
|
|
134
|
+
}
|
|
135
|
+
/** Read and parse every execution's events in this directory. */
|
|
136
|
+
async readAll() {
|
|
137
|
+
let files;
|
|
138
|
+
try {
|
|
139
|
+
files = await readdir(this.dir);
|
|
140
|
+
}
|
|
141
|
+
catch (err) {
|
|
142
|
+
if (err.code === 'ENOENT')
|
|
143
|
+
return [];
|
|
144
|
+
throw err;
|
|
145
|
+
}
|
|
146
|
+
const all = [];
|
|
147
|
+
for (const file of files) {
|
|
148
|
+
if (!file.endsWith('.jsonl'))
|
|
149
|
+
continue;
|
|
150
|
+
all.push(...this.parse(await readFile(join(this.dir, file), 'utf-8')));
|
|
151
|
+
}
|
|
152
|
+
return all;
|
|
153
|
+
}
|
|
154
|
+
async readRaw(executionId) {
|
|
155
|
+
try {
|
|
156
|
+
return await readFile(this.pathFor(executionId), 'utf-8');
|
|
157
|
+
}
|
|
158
|
+
catch (err) {
|
|
159
|
+
if (err.code === 'ENOENT')
|
|
160
|
+
return '';
|
|
161
|
+
throw err;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
parse(content) {
|
|
165
|
+
const events = [];
|
|
166
|
+
for (const line of content.split('\n')) {
|
|
167
|
+
if (line.trim() === '')
|
|
168
|
+
continue;
|
|
169
|
+
try {
|
|
170
|
+
events.push(JSON.parse(line));
|
|
171
|
+
}
|
|
172
|
+
catch {
|
|
173
|
+
// A torn line from a crash mid-append. Each event is an independent
|
|
174
|
+
// single-line record, so skip just this line rather than discarding
|
|
175
|
+
// everything after it (a healed torn line can sit mid-file, followed
|
|
176
|
+
// by valid events appended on resume).
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
return events;
|
|
181
|
+
}
|
|
182
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -19,8 +19,9 @@ export * from './ast/index.js';
|
|
|
19
19
|
export { MissionExecutor, HttpClient, BearerAuthProvider, OAuth2AuthProvider, createContext, evaluate, type ExecutionResult, type ExecutionError, type ExecutorConfig, type ExecutionContext, type ProgressCallbacks, type ExecutionStartEvent, type ExecutionCompleteEvent, type StageStartEvent, type StageCompleteEvent, } from './interpreter/index.js';
|
|
20
20
|
export { MemoryStore, FileStore, createStore, type StoreAdapter, type StoreFilter, type StoreConfig, } from './stores/index.js';
|
|
21
21
|
export { createExecutionState, findResumePoint, canResume, getProgress, getExecutionSummary, FileExecutionStore, MemoryExecutionStore, type ExecutionState, type ExecutionStore, type StageState, type LiveProgress, } from './execution/index.js';
|
|
22
|
+
export { MemoryExecutionLog, FileExecutionLog, SqliteExecutionLog, PostgresExecutionLog, foldLog, loadState, effectId, reduceCheckpoints, type ExecutionEvent, type StoredEvent, type ExecutionLogStore, type CheckpointRecord, type FoldedState, } from './execution-log/index.js';
|
|
22
23
|
export { Scheduler, parseCronExpression, getNextRunTime, intervalToMs, shouldRunNow, type ScheduledJob, type SchedulerState, type ScheduleEvent, type SchedulerCallbacks, type SchedulerConfig, type ScheduledMission, } from './scheduler/index.js';
|
|
23
|
-
export { generateCheckpointKey, formatSinceDate, parseSinceDate, EPOCH, FileSyncStore, MemorySyncStore, type SyncCheckpoint, type SyncStore, } from './sync/index.js';
|
|
24
|
+
export { generateCheckpointKey, formatSinceDate, parseSinceDate, EPOCH, FileSyncStore, MemorySyncStore, LogBackedSyncStore, type SyncCheckpoint, type SyncStore, } from './sync/index.js';
|
|
24
25
|
export { ReqonError, ParseError, LexerError, RuntimeError, ValidationError, formatErrors, getSourceLine, getSourceContext, type SourceLocation, type ErrorContext, } from './errors/index.js';
|
|
25
26
|
export { loadMission, isMissionFolder, getMissionName, type LoadResult, type LoadOptions, } from './loader/index.js';
|
|
26
27
|
export { loadEnv, loadCredentials, resolveCredentials, resolveEnvString, hasEnvReference, credentialsFromEnv, type CredentialsConfig, type LoadEnvResult, type AuthCredentials, type SourceCredentials, } from './auth/credentials.js';
|
|
@@ -29,8 +30,8 @@ export { ControlServer, type ControlServerConfig, type ControlServerCallbacks, t
|
|
|
29
30
|
export { ObservabilityEmitter, createEmitter, type ObservabilityEvent, type EventType, type EventHandler, type EventEmitter, type MissionStartPayload, type MissionCompletePayload, type StageStartPayload, type StageCompletePayload, type StepStartPayload, type StepCompletePayload, type FetchStartPayload, type FetchCompletePayload, type DataStorePayload, type LoopStartPayload, type LoopCompletePayload, type WebhookRegisterPayload, type WebhookCompletePayload, createStructuredLogger, ConsoleOutput, JsonLinesOutput, BufferOutput, type StructuredLogger, type LogEntry, type LogOutput, type Span, type CreateLoggerOptions, SpanBuilder, OTelEventAdapter, OTLPExporter, createOTelListener, type OTelSpan, type OTLPExporterConfig, } from './observability/index.js';
|
|
30
31
|
export { MCP_SERVER_VERSION } from './mcp/index.js';
|
|
31
32
|
export { BaseDebugController, CLIDebugger, type DebugController, type DebugMode, type DebugPauseReason, type DebugSnapshot, type DebugCommand, type DebugLocation, } from './debug/index.js';
|
|
32
|
-
export { FileTraceStore, MemoryTraceStore, TraceRecorder, TraceReplayer, createTraceRecorder, createTraceReplayer, createExecutionTrace, safeClone, truncateForTrace, type TraceStore, type TraceSnapshot, type StoreSnapshot as TraceStoreSnapshot, type LoopContext as TraceLoopContext, type ExecutionTrace, type TraceRecorderConfig, type ReplaySession, type ReplayStepResult, type TimelineEvent, type VariableChange, type SnapshotDiff, } from './trace/index.js';
|
|
33
|
-
export { FilePauseStore, MemoryPauseStore, PauseManager, createPauseManager, parseDuration, formatDuration, createPauseState, isPauseExpired, getRemainingTime, getPauseSummary, type PauseStore, type PauseState, type PauseCheckpoint, type PauseResumeTriggerState, type PauseManagerConfig, type CreatePauseOptions, type PauseStatus, } from './pause/index.js';
|
|
33
|
+
export { FileTraceStore, MemoryTraceStore, TraceRecorder, TraceReplayer, LogTraceView, traceTimelineFromLog, createTraceRecorder, createTraceReplayer, createExecutionTrace, safeClone, truncateForTrace, type LogTimelineEntry, type LogTraceSummary, type TraceStore, type TraceSnapshot, type StoreSnapshot as TraceStoreSnapshot, type LoopContext as TraceLoopContext, type ExecutionTrace, type TraceRecorderConfig, type ReplaySession, type ReplayStepResult, type TimelineEvent, type VariableChange, type SnapshotDiff, } from './trace/index.js';
|
|
34
|
+
export { FilePauseStore, MemoryPauseStore, LogBackedPauseStore, PauseManager, createPauseManager, parseDuration, formatDuration, createPauseState, isPauseExpired, getRemainingTime, getPauseSummary, type PauseStore, type PauseState, type PauseCheckpoint, type PauseResumeTriggerState, type PauseManagerConfig, type CreatePauseOptions, type PauseStatus, } from './pause/index.js';
|
|
34
35
|
import { type ExecutorConfig } from './interpreter/index.js';
|
|
35
36
|
import type { ReqonProgram } from './ast/index.js';
|
|
36
37
|
export declare function parse(source: string, filePath?: string): ReqonProgram;
|
package/dist/index.js
CHANGED
|
@@ -19,8 +19,9 @@ export * from './ast/index.js';
|
|
|
19
19
|
export { MissionExecutor, HttpClient, BearerAuthProvider, OAuth2AuthProvider, createContext, evaluate, } from './interpreter/index.js';
|
|
20
20
|
export { MemoryStore, FileStore, createStore, } from './stores/index.js';
|
|
21
21
|
export { createExecutionState, findResumePoint, canResume, getProgress, getExecutionSummary, FileExecutionStore, MemoryExecutionStore, } from './execution/index.js';
|
|
22
|
+
export { MemoryExecutionLog, FileExecutionLog, SqliteExecutionLog, PostgresExecutionLog, foldLog, loadState, effectId, reduceCheckpoints, } from './execution-log/index.js';
|
|
22
23
|
export { Scheduler, parseCronExpression, getNextRunTime, intervalToMs, shouldRunNow, } from './scheduler/index.js';
|
|
23
|
-
export { generateCheckpointKey, formatSinceDate, parseSinceDate, EPOCH, FileSyncStore, MemorySyncStore, } from './sync/index.js';
|
|
24
|
+
export { generateCheckpointKey, formatSinceDate, parseSinceDate, EPOCH, FileSyncStore, MemorySyncStore, LogBackedSyncStore, } from './sync/index.js';
|
|
24
25
|
export { ReqonError, ParseError, LexerError, RuntimeError, ValidationError, formatErrors, getSourceLine, getSourceContext, } from './errors/index.js';
|
|
25
26
|
export { loadMission, isMissionFolder, getMissionName, } from './loader/index.js';
|
|
26
27
|
export { loadEnv, loadCredentials, resolveCredentials, resolveEnvString, hasEnvReference, credentialsFromEnv, } from './auth/credentials.js';
|
|
@@ -39,9 +40,9 @@ export { MCP_SERVER_VERSION } from './mcp/index.js';
|
|
|
39
40
|
// Debug
|
|
40
41
|
export { BaseDebugController, CLIDebugger, } from './debug/index.js';
|
|
41
42
|
// Trace - time-travel debugging
|
|
42
|
-
export { FileTraceStore, MemoryTraceStore, TraceRecorder, TraceReplayer, createTraceRecorder, createTraceReplayer, createExecutionTrace, safeClone, truncateForTrace, } from './trace/index.js';
|
|
43
|
+
export { FileTraceStore, MemoryTraceStore, TraceRecorder, TraceReplayer, LogTraceView, traceTimelineFromLog, createTraceRecorder, createTraceReplayer, createExecutionTrace, safeClone, truncateForTrace, } from './trace/index.js';
|
|
43
44
|
// Pause - resource-free long pauses
|
|
44
|
-
export { FilePauseStore, MemoryPauseStore, PauseManager, createPauseManager, parseDuration, formatDuration, createPauseState, isPauseExpired, getRemainingTime, getPauseSummary, } from './pause/index.js';
|
|
45
|
+
export { FilePauseStore, MemoryPauseStore, LogBackedPauseStore, PauseManager, createPauseManager, parseDuration, formatDuration, createPauseState, isPauseExpired, getRemainingTime, getPauseSummary, } from './pause/index.js';
|
|
45
46
|
import { readFile } from 'node:fs/promises';
|
|
46
47
|
import { resolve } from 'node:path';
|
|
47
48
|
import { ReqonLexer } from './lexer/index.js';
|
|
@@ -14,6 +14,20 @@
|
|
|
14
14
|
import type { SchemaDefinition } from 'vague-lang';
|
|
15
15
|
import type { StoreAdapter } from '../stores/types.js';
|
|
16
16
|
import type { HttpClient } from './http.js';
|
|
17
|
+
/**
|
|
18
|
+
* Mutable per-action execution state. Parallel actions (`run [A, B]`) must not
|
|
19
|
+
* share this — each gets its own scope so their step counters and deferred sync
|
|
20
|
+
* checkpoints can't corrupt each other. Nested scopes (for/match) inherit the
|
|
21
|
+
* same reference so the whole action shares one counter/checkpoint list.
|
|
22
|
+
*/
|
|
23
|
+
export interface ActionScope {
|
|
24
|
+
/** Monotonic step index within the action (for trace/debug positions). */
|
|
25
|
+
stepIndex: number;
|
|
26
|
+
/** Current retry attempt of the action (0-based), for effect identity. */
|
|
27
|
+
attempt: number;
|
|
28
|
+
/** Sync checkpoints deferred until the action's data is durably stored. */
|
|
29
|
+
pendingCheckpoints: Array<() => Promise<void>>;
|
|
30
|
+
}
|
|
17
31
|
export interface ExecutionContext {
|
|
18
32
|
stores: Map<string, StoreAdapter>;
|
|
19
33
|
storeTypes: Map<string, string>;
|
|
@@ -22,6 +36,7 @@ export interface ExecutionContext {
|
|
|
22
36
|
variables: Map<string, unknown>;
|
|
23
37
|
response?: unknown;
|
|
24
38
|
parent?: ExecutionContext;
|
|
39
|
+
actionScope?: ActionScope;
|
|
25
40
|
}
|
|
26
41
|
export declare function createContext(): ExecutionContext;
|
|
27
42
|
export declare function childContext(parent: ExecutionContext): ExecutionContext;
|
|
@@ -28,6 +28,9 @@ export function childContext(parent) {
|
|
|
28
28
|
schemas: parent.schemas,
|
|
29
29
|
variables: new Map(),
|
|
30
30
|
parent,
|
|
31
|
+
// Nested scopes (for/match) share the action's scope so the step counter
|
|
32
|
+
// and checkpoint list stay coherent across the whole action.
|
|
33
|
+
actionScope: parent.actionScope,
|
|
31
34
|
};
|
|
32
35
|
}
|
|
33
36
|
export function getVariable(ctx, name) {
|