reqon-dsl 0.2.0 → 0.4.0

package/dist/scheduler/scheduler.js

@@ -1,7 +1,9 @@
-import { readFile, writeFile, mkdir } from 'node:fs/promises';
+import { readFile, mkdir } from 'node:fs/promises';
 import { join } from 'node:path';
+import { writeFileAtomic } from '../utils/file.js';
 import { MissionExecutor } from '../interpreter/executor.js';
 import { getNextRunTime, shouldRunNow } from './cron-parser.js';
+import { setLongTimeout } from '../utils/long-timeout.js';
 /**
  * Scheduler for running missions on a schedule
  */
@@ -35,9 +37,18 @@ export class Scheduler {
             throw new Error(`Mission '${mission.name}' has no schedule defined`);
         }
         this.missions.set(mission.name, { mission, filePath });
-        // Create or update job state
+        // Create or update job state. A schedule that can never produce a next run
+        // (e.g. an impossible cron like Feb 31) must not abort registration — the
+        // job is registered with an undefined nextRun and the check loop isolates
+        // it so it can't starve sibling jobs.
         const existingJob = this.state.jobs[mission.name];
-        const nextRun = getNextRunTime(mission.schedule);
+        let nextRun = null;
+        try {
+            nextRun = getNextRunTime(mission.schedule);
+        }
+        catch (error) {
+            this.log(`Mission '${mission.name}' has an unschedulable cron: ${error.message}`);
+        }
         this.state.jobs[mission.name] = {
             id: mission.name,
             missionName: mission.name,
@@ -97,7 +108,7 @@ export class Scheduler {
         }
         // Clear all pending retry timers to prevent memory leaks
         for (const timer of this.retryTimers) {
-            clearTimeout(timer);
+            timer.clear();
         }
         this.retryTimers.clear();
         // Save state
@@ -109,19 +120,25 @@ export class Scheduler {
      */
     async checkAndRun() {
         const now = new Date();
+        // Each job is isolated: a throw while evaluating one schedule (e.g. an
+        // impossible cron) must not break the loop and starve siblings. Due jobs
+        // are dispatched without awaiting so a slow mission can't head-of-line
+        // block the rest of the tick.
         for (const [name, job] of Object.entries(this.state.jobs)) {
-            if (!job.enabled)
-                continue;
-            const scheduledMission = this.missions.get(name);
-            if (!scheduledMission)
-                continue;
-            const schedule = scheduledMission.mission.schedule;
-            if (!schedule)
-                continue; // Mission must have schedule
-            // Check if job should run
-            if (shouldRunNow(schedule, job.lastRun, this.config.checkInterval)) {
+            try {
+                if (!job.enabled)
+                    continue;
+                const scheduledMission = this.missions.get(name);
+                if (!scheduledMission)
+                    continue;
+                const schedule = scheduledMission.mission.schedule;
+                if (!schedule)
+                    continue; // Mission must have schedule
+                // Check if job should run
+                if (!shouldRunNow(schedule, job.lastRun, this.config.checkInterval))
+                    continue;
                 // Check if already running
-                if (job.isRunning && (schedule.skipIfRunning !== false)) {
+                if (job.isRunning && schedule.skipIfRunning !== false) {
                     this.emitEvent({
                         type: 'skipped',
                         jobId: job.id,
@@ -131,8 +148,14 @@ export class Scheduler {
                     });
                     continue;
                 }
-                // Run the job
-                await this.runJob(job, scheduledMission);
+                // Dispatch without awaiting: the loop keeps checking other jobs while
+                // this mission runs. Failures are handled inside runJob.
+                void this.runJob(job, scheduledMission).catch((error) => {
+                    this.log(`Job '${name}' run error: ${error.message}`);
+                });
+            }
+            catch (error) {
+                this.log(`Error checking job '${name}': ${error.message}`);
             }
         }
     }
@@ -206,8 +229,16 @@ export class Scheduler {
         }
         finally {
             job.isRunning = false;
-            // Calculate next run time
-            job.nextRun = getNextRunTime(scheduledMission.mission.schedule, new Date()) ?? undefined;
+            // Calculate next run time; never let an unschedulable cron throw out of
+            // the finally block and mask the run's real outcome.
+            try {
+                job.nextRun =
+                    getNextRunTime(scheduledMission.mission.schedule, new Date()) ?? undefined;
+            }
+            catch (error) {
+                job.nextRun = undefined;
+                this.log(`Could not compute next run for '${job.missionName}': ${error.message}`);
+            }
             await this.saveState();
         }
     }
@@ -221,8 +252,9 @@ export class Scheduler {
         if (job.consecutiveFailures <= retryConfig.maxRetries) {
             this.log(`Job '${job.missionName}' failed, retrying in ${retryConfig.delaySeconds}s ` +
                 `(attempt ${job.consecutiveFailures}/${retryConfig.maxRetries})`);
-            // Schedule retry and track the timer to prevent memory leaks
-            const timer = setTimeout(() => {
+            // Schedule retry and track the timer to prevent memory leaks. Use a
+            // long-delay-safe timer so a large retry delay can't overflow setTimeout.
+            const timer = setLongTimeout(() => {
                 this.retryTimers.delete(timer);
                 const scheduledMission = this.missions.get(job.missionName);
                 if (scheduledMission && this.running) {
@@ -318,7 +350,9 @@ export class Scheduler {
         try {
             await mkdir(this.config.stateDir, { recursive: true });
             const statePath = join(this.config.stateDir, 'state.json');
-            await writeFile(statePath, JSON.stringify(this.state, null, 2));
+            // Atomic write (temp file + fsync + rename) so a crash mid-write can't
+            // truncate or corrupt state.json.
+            await writeFileAtomic(statePath, JSON.stringify(this.state, null, 2));
         }
         catch (error) {
             this.log(`Failed to save scheduler state: ${error.message}`);

package/dist/stores/factory.d.ts

@@ -16,6 +16,12 @@ export interface CreateStoreOptions {
     postgrest?: Omit<PostgRESTOptions, 'table'>;
     /** Logger instance */
     logger?: Logger;
+    /**
+     * Explicitly allow sql/nosql to fall back to a local file store. Without
+     * this, an unimplemented sql/nosql store hard-errors rather than silently
+     * writing JSON to disk (a data-loss trap).
+     */
+    allowFileFallback?: boolean;
 }
 /**
  * Create a store adapter based on type
@@ -33,4 +39,4 @@ export declare function createStore(options: CreateStoreOptions): StoreAdapter;
  * Map DSL store type to adapter type
  * In development mode, sql/nosql fall back to file stores
  */
-export declare function resolveStoreType(dslType: 'sql' | 'nosql' | 'memory', developmentMode?: boolean): StoreType;
+export declare function resolveStoreType(dslType: 'sql' | 'nosql' | 'memory' | 'file' | 'postgrest', developmentMode?: boolean): StoreType;

package/dist/stores/factory.js

@@ -40,6 +40,11 @@ export function createStore(options) {
                 });
             }
             // TODO: Implement raw SQL adapter (pg, mysql2, etc.)
+            if (!options.allowFileFallback) {
+                throw new Error(`SQL store is not implemented. Configure a PostgREST/Supabase backend, or ` +
+                    `opt into the local file fallback (enable development mode / allowFileFallback) ` +
+                    `for store '${options.name}'.`);
+            }
             logger.warn(`SQL store not yet implemented, falling back to file store for '${options.name}'`);
             return new FileStore(options.name, {
                 ...options.fileOptions,
@@ -47,6 +52,11 @@ export function createStore(options) {
             });
         case 'nosql':
             // TODO: Implement NoSQL adapter
+            if (!options.allowFileFallback) {
+                throw new Error(`NoSQL store is not implemented. Configure a real backend, or opt into the ` +
+                    `local file fallback (enable development mode / allowFileFallback) for ` +
+                    `store '${options.name}'.`);
+            }
             logger.warn(`NoSQL store not yet implemented, falling back to file store for '${options.name}'`);
             return new FileStore(options.name, {
                 ...options.fileOptions,
@@ -60,9 +70,10 @@ export function createStore(options) {
  * Map DSL store type to adapter type
  * In development mode, sql/nosql fall back to file stores
  */
-export function resolveStoreType(dslType, developmentMode = true) {
-    if (dslType === 'memory') {
-        return 'memory';
+export function resolveStoreType(dslType, developmentMode = false) {
+    // These types are used directly
+    if (dslType === 'memory' || dslType === 'file' || dslType === 'postgrest') {
+        return dslType;
     }
     if (developmentMode) {
         // Use file stores for local development

package/dist/stores/file.d.ts

@@ -12,6 +12,9 @@ export interface FileStoreOptions {
 /**
  * File-based JSON store for local development
  * Persists data to .reqon-data/{name}.json
+ *
+ * Use FileStore.create() for async initialization (recommended),
+ * or new FileStore() for lazy initialization (backward compatible).
  */
 export declare class FileStore implements StoreAdapter {
     private data;
@@ -19,9 +22,28 @@ export declare class FileStore implements StoreAdapter {
     private options;
     private dirty;
     private initialized;
+    private initError;
     private debounceTimer;
     private pendingWrite;
+    /**
+     * Create a FileStore with async initialization (recommended).
+     * The store is fully initialized when the promise resolves.
+     *
+     * @example
+     * const store = await FileStore.create('my-store', { baseDir: '.data' });
+     */
+    static create(name: string, options?: FileStoreOptions): Promise<FileStore>;
+    /**
+     * Constructor for lazy initialization (backward compatible).
+     * Each operation will wait for initialization to complete.
+     * Prefer FileStore.create() for explicit async initialization.
+     */
     constructor(name: string, options?: FileStoreOptions);
+    /**
+     * Ensure the store is initialized. Safe to call multiple times.
+     * Throws if initialization previously failed.
+     */
+    private ensureInitialized;
     private init;
     private load;
     private persist;
@@ -35,6 +57,10 @@ export declare class FileStore implements StoreAdapter {
         key: string;
         value: Record<string, unknown>;
     }>): Promise<void>;
+    bulkUpsert(records: Array<{
+        key: string;
+        value: Record<string, unknown>;
+    }>): Promise<void>;
     update(key: string, value: Partial<Record<string, unknown>>): Promise<void>;
     delete(key: string): Promise<void>;
     list(filter?: StoreFilter): Promise<Record<string, unknown>[]>;

package/dist/stores/file.js

@@ -1,8 +1,10 @@
 import { writeFile } from 'node:fs/promises';
-import { writeFileSync, existsSync } from 'node:fs';
+import { existsSync } from 'node:fs';
 import { join, dirname } from 'node:path';
 import { applyStoreFilter } from './types.js';
-import { ensureDirectory, readJsonFile, serialize, } from '../utils/file.js';
+import { ensureDirectory, readJsonFile, serialize, writeFileAtomic, writeFileAtomicSync, } from '../utils/file.js';
+import { safeJoin } from '../utils/path.js';
+import { deepMerge } from '../utils/deep-merge.js';
 const DEFAULT_OPTIONS = {
     baseDir: '.reqon-data',
     persist: 'immediate',
@@ -12,19 +14,58 @@ const DEFAULT_OPTIONS = {
 /**
  * File-based JSON store for local development
  * Persists data to .reqon-data/{name}.json
+ *
+ * Use FileStore.create() for async initialization (recommended),
+ * or new FileStore() for lazy initialization (backward compatible).
  */
 export class FileStore {
     data = new Map();
     filePath;
     options;
     dirty = false;
-    initialized;
+    initialized = null;
+    initError = null;
     debounceTimer = null;
     pendingWrite = null;
+    /**
+     * Create a FileStore with async initialization (recommended).
+     * The store is fully initialized when the promise resolves.
+     *
+     * @example
+     * const store = await FileStore.create('my-store', { baseDir: '.data' });
+     */
+    static async create(name, options = {}) {
+        const store = new FileStore(name, options);
+        await store.ensureInitialized();
+        return store;
+    }
+    /**
+     * Constructor for lazy initialization (backward compatible).
+     * Each operation will wait for initialization to complete.
+     * Prefer FileStore.create() for explicit async initialization.
+     */
     constructor(name, options = {}) {
         this.options = { ...DEFAULT_OPTIONS, ...options };
-        this.filePath = join(this.options.baseDir, `${name}.json`);
-        this.initialized = this.init();
+        this.filePath = safeJoin(this.options.baseDir, `${name}.json`);
+        // Lazy initialization - init() is called on first operation
+    }
+    /**
+     * Ensure the store is initialized. Safe to call multiple times.
+     * Throws if initialization previously failed.
+     */
+    async ensureInitialized() {
+        // If init previously failed, throw the cached error
+        if (this.initError) {
+            throw this.initError;
+        }
+        // If not yet started, start initialization
+        if (this.initialized === null) {
+            this.initialized = this.init().catch((error) => {
+                this.initError = error instanceof Error ? error : new Error(String(error));
+                throw this.initError;
+            });
+        }
+        await this.initialized;
     }
     async init() {
         const dir = dirname(this.filePath);
@@ -72,27 +113,27 @@ export class FileStore {
     async writeToDisk() {
         const obj = Object.fromEntries(this.data);
         const content = serialize(obj, this.options.pretty);
-        await writeFile(this.filePath, content, 'utf-8');
+        await writeFileAtomic(this.filePath, content);
         this.dirty = false;
     }
     /** Synchronous write for flush/close operations */
     writeToDiskSync() {
         const obj = Object.fromEntries(this.data);
         const content = serialize(obj, this.options.pretty);
-        writeFileSync(this.filePath, content, 'utf-8');
+        writeFileAtomicSync(this.filePath, content);
         this.dirty = false;
     }
     async get(key) {
-        await this.initialized;
+        await this.ensureInitialized();
         return this.data.get(key) ?? null;
     }
     async set(key, value) {
-        await this.initialized;
+        await this.ensureInitialized();
         this.data.set(key, { ...value });
         await this.persist();
     }
     async bulkSet(records) {
-        await this.initialized;
+        await this.ensureInitialized();
         // Set all records in memory first (no disk I/O per record)
         for (const { key, value } of records) {
             this.data.set(key, { ...value });
@@ -100,28 +141,33 @@ export class FileStore {
         // Single persist operation for all records
         await this.persist();
     }
+    async bulkUpsert(records) {
+        await this.ensureInitialized();
+        // Upsert all records in memory first (no disk I/O per record)
+        for (const { key, value } of records) {
+            const existing = this.data.get(key);
+            this.data.set(key, existing ? deepMerge(existing, value) : { ...value });
+        }
+        // Single persist operation for all records
+        await this.persist();
+    }
     async update(key, value) {
-        await this.initialized;
+        await this.ensureInitialized();
         const existing = this.data.get(key);
-        if (existing) {
-            this.data.set(key, { ...existing, ...value });
-        }
-        else {
-            this.data.set(key, value);
-        }
+        this.data.set(key, existing ? deepMerge(existing, value) : { ...value });
         await this.persist();
     }
     async delete(key) {
-        await this.initialized;
+        await this.ensureInitialized();
         this.data.delete(key);
         await this.persist();
     }
     async list(filter) {
-        await this.initialized;
+        await this.ensureInitialized();
         return applyStoreFilter(Array.from(this.data.values()), filter);
     }
     async count(filter) {
-        await this.initialized;
+        await this.ensureInitialized();
         // Apply only the where clause for counting (ignore limit/offset)
         const filtered = applyStoreFilter(Array.from(this.data.values()), {
             where: filter?.where,
@@ -129,7 +175,7 @@ export class FileStore {
         return filtered.length;
     }
     async clear() {
-        await this.initialized;
+        await this.ensureInitialized();
         this.data.clear();
         await this.persist();
     }

package/dist/stores/index.d.ts

@@ -1,6 +1,21 @@
+/**
+ * ---
+ * purpose: Store adapters - pluggable storage backends for pipeline data
+ * exports:
+ *   - StoreAdapter - interface for all stores
+ *   - MemoryStore, FileStore, PostgRESTStore - implementations
+ *   - createStore - factory function
+ * related:
+ *   - ./types.ts - StoreAdapter interface definition
+ *   - ./memory.ts - in-memory store (default)
+ *   - ./file.ts - JSON/CSV file store
+ *   - ./postgrest.ts - PostgREST API store
+ *   - ../interpreter/context.ts - stores registered in context
+ * ---
+ */
 export type { StoreAdapter, StoreFilter, StoreConfig } from './types.js';
 export { applyStoreFilter } from './types.js';
 export { MemoryStore } from './memory.js';
 export { FileStore, type FileStoreOptions } from './file.js';
 export { PostgRESTStore, PostgRESTError, type PostgRESTOptions } from './postgrest.js';
-export { createStore, resolveStoreType, type StoreType, type CreateStoreOptions } from './factory.js';
+export { createStore, resolveStoreType, type StoreType, type CreateStoreOptions, } from './factory.js';

package/dist/stores/index.js

@@ -1,5 +1,20 @@
+/**
+ * ---
+ * purpose: Store adapters - pluggable storage backends for pipeline data
+ * exports:
+ *   - StoreAdapter - interface for all stores
+ *   - MemoryStore, FileStore, PostgRESTStore - implementations
+ *   - createStore - factory function
+ * related:
+ *   - ./types.ts - StoreAdapter interface definition
+ *   - ./memory.ts - in-memory store (default)
+ *   - ./file.ts - JSON/CSV file store
+ *   - ./postgrest.ts - PostgREST API store
+ *   - ../interpreter/context.ts - stores registered in context
+ * ---
+ */
 export { applyStoreFilter } from './types.js';
 export { MemoryStore } from './memory.js';
 export { FileStore } from './file.js';
 export { PostgRESTStore, PostgRESTError } from './postgrest.js';
-export { createStore, resolveStoreType } from './factory.js';
+export { createStore, resolveStoreType, } from './factory.js';

package/dist/stores/memory.d.ts

@@ -9,6 +9,10 @@ export declare class MemoryStore implements StoreAdapter {
         key: string;
         value: Record<string, unknown>;
     }>): Promise<void>;
+    bulkUpsert(records: Array<{
+        key: string;
+        value: Record<string, unknown>;
+    }>): Promise<void>;
     update(key: string, value: Partial<Record<string, unknown>>): Promise<void>;
     delete(key: string): Promise<void>;
     list(filter?: StoreFilter): Promise<Record<string, unknown>[]>;

package/dist/stores/memory.js

@@ -1,4 +1,5 @@
 import { applyStoreFilter } from './types.js';
+import { deepMerge } from '../utils/deep-merge.js';
 export class MemoryStore {
     data = new Map();
     name;
@@ -16,14 +17,15 @@ export class MemoryStore {
             this.data.set(key, { ...value });
         }
     }
+    async bulkUpsert(records) {
+        for (const { key, value } of records) {
+            const existing = this.data.get(key);
+            this.data.set(key, existing ? deepMerge(existing, value) : { ...value });
+        }
+    }
     async update(key, value) {
         const existing = this.data.get(key);
-        if (existing) {
-            this.data.set(key, { ...existing, ...value });
-        }
-        else {
-            this.data.set(key, value);
-        }
+        this.data.set(key, existing ? deepMerge(existing, value) : { ...value });
     }
     async delete(key) {
         this.data.delete(key);

package/dist/stores/postgrest.d.ts

@@ -10,6 +10,17 @@ export interface PostgRESTOptions {
     primaryKey?: string;
     /** Optional schema (for Supabase, typically 'public') */
     schema?: string;
+    /**
+     * Per-request timeout in milliseconds (default: 30000). Every request is
+     * aborted after this elapses so a hung connection can't stall a mission.
+     */
+    timeoutMs?: number;
+    /**
+     * Opt-in guard for {@link PostgRESTStore.clear}, which issues a full-table
+     * DELETE. Defaults to false so a misconfigured store (e.g. a `sql` type
+     * resolving here by mistake) can't wipe a table.
+     */
+    allowFullTableClear?: boolean;
 }
 /**
  * PostgREST-compatible store adapter.
@@ -29,7 +40,24 @@ export declare class PostgRESTStore implements StoreAdapter {
     private baseUrl;
     private headers;
     private primaryKey;
+    private timeoutMs;
     constructor(options: PostgRESTOptions);
+    /** fetch wrapper that aborts the request once {@link timeoutMs} elapses. */
+    private request;
+    /**
+     * Reject a where-clause field unless it is a plain column identifier. This
+     * stops a key such as `or` or `id,or=(...)` from being concatenated into the
+     * query string and reinterpreted as a PostgREST operator.
+     */
+    private validateField;
+    /**
+     * Render a where-clause value as a PostgREST operand. Strings containing
+     * reserved characters are wrapped in a quoted operand (with `"` and `\`
+     * escaped) so they can't be parsed as list/group syntax.
+     */
+    private formatFilterValue;
+    /** Append a validated, escaped where clause to the given params. */
+    private applyWhere;
     get(key: string): Promise<Record<string, unknown> | null>;
     set(key: string, value: Record<string, unknown>): Promise<void>;
     update(key: string, value: Partial<Record<string, unknown>>): Promise<void>;