reqon-dsl 0.2.0

package/dist/stores/postgrest.js

@@ -0,0 +1,217 @@
+/**
+ * PostgREST-compatible store adapter.
+ * Works with Supabase, standalone PostgREST, or any PostgREST-compatible API.
+ *
+ * @example
+ * ```typescript
+ * const store = new PostgRESTStore({
+ *   url: 'https://xxx.supabase.co/rest/v1',
+ *   apiKey: process.env.SUPABASE_ANON_KEY,
+ *   table: 'users',
+ * });
+ * ```
+ */
+export class PostgRESTStore {
+    options;
+    baseUrl;
+    headers;
+    primaryKey;
+    constructor(options) {
+        this.options = options;
+        // Normalize URL (remove trailing slash)
+        const url = options.url.replace(/\/$/, '');
+        this.baseUrl = `${url}/${options.table}`;
+        this.primaryKey = options.primaryKey ?? 'id';
+        this.headers = {
+            'Content-Type': 'application/json',
+            'apikey': options.apiKey,
+            'Authorization': `Bearer ${options.apiKey}`,
+            'Prefer': 'return=representation',
+        };
+        if (options.schema) {
+            this.headers['Accept-Profile'] = options.schema;
+            this.headers['Content-Profile'] = options.schema;
+        }
+    }
+    async get(key) {
+        const url = `${this.baseUrl}?${this.primaryKey}=eq.${encodeURIComponent(key)}&limit=1`;
+        const response = await fetch(url, {
+            method: 'GET',
+            headers: this.headers,
+        });
+        if (!response.ok) {
+            throw new PostgRESTError(`Failed to get record: ${response.statusText}`, response.status);
+        }
+        const data = await response.json();
+        return Array.isArray(data) && data.length > 0 ? data[0] : null;
+    }
+    async set(key, value) {
+        // Upsert using PostgREST's on_conflict resolution
+        const record = { ...value, [this.primaryKey]: key };
+        const response = await fetch(this.baseUrl, {
+            method: 'POST',
+            headers: {
+                ...this.headers,
+                'Prefer': 'resolution=merge-duplicates,return=representation',
+            },
+            body: JSON.stringify(record),
+        });
+        if (!response.ok) {
+            const error = await this.parseError(response);
+            throw new PostgRESTError(`Failed to set record: ${error}`, response.status);
+        }
+    }
+    async update(key, value) {
+        const url = `${this.baseUrl}?${this.primaryKey}=eq.${encodeURIComponent(key)}`;
+        const response = await fetch(url, {
+            method: 'PATCH',
+            headers: this.headers,
+            body: JSON.stringify(value),
+        });
+        if (!response.ok) {
+            const error = await this.parseError(response);
+            throw new PostgRESTError(`Failed to update record: ${error}`, response.status);
+        }
+    }
+    async delete(key) {
+        const url = `${this.baseUrl}?${this.primaryKey}=eq.${encodeURIComponent(key)}`;
+        const response = await fetch(url, {
+            method: 'DELETE',
+            headers: this.headers,
+        });
+        if (!response.ok) {
+            const error = await this.parseError(response);
+            throw new PostgRESTError(`Failed to delete record: ${error}`, response.status);
+        }
+    }
+    async list(filter) {
+        const params = new URLSearchParams();
+        // Apply where clause
+        if (filter?.where) {
+            for (const [field, value] of Object.entries(filter.where)) {
+                if (value === null) {
+                    params.append(field, 'is.null');
+                }
+                else if (typeof value === 'string') {
+                    params.append(field, `eq.${value}`);
+                }
+                else if (typeof value === 'number' || typeof value === 'boolean') {
+                    params.append(field, `eq.${value}`);
+                }
+                else {
+                    // For complex values, try JSON
+                    params.append(field, `eq.${JSON.stringify(value)}`);
+                }
+            }
+        }
+        // Apply pagination
+        if (filter?.limit) {
+            params.append('limit', String(filter.limit));
+        }
+        if (filter?.offset) {
+            params.append('offset', String(filter.offset));
+        }
+        const queryString = params.toString();
+        const url = queryString ? `${this.baseUrl}?${queryString}` : this.baseUrl;
+        const response = await fetch(url, {
+            method: 'GET',
+            headers: this.headers,
+        });
+        if (!response.ok) {
+            const error = await this.parseError(response);
+            throw new PostgRESTError(`Failed to list records: ${error}`, response.status);
+        }
+        return response.json();
+    }
+    async clear() {
+        // Delete all records - PostgREST requires a filter, so we use a always-true condition
+        // This deletes where primary key is not null (i.e., all records)
+        const url = `${this.baseUrl}?${this.primaryKey}=not.is.null`;
+        const response = await fetch(url, {
+            method: 'DELETE',
+            headers: this.headers,
+        });
+        if (!response.ok) {
+            const error = await this.parseError(response);
+            throw new PostgRESTError(`Failed to clear records: ${error}`, response.status);
+        }
+    }
+    /**
+     * Bulk insert records (more efficient than individual sets)
+     */
+    async bulkInsert(records) {
+        if (records.length === 0)
+            return;
+        const response = await fetch(this.baseUrl, {
+            method: 'POST',
+            headers: {
+                ...this.headers,
+                'Prefer': 'resolution=merge-duplicates',
+            },
+            body: JSON.stringify(records),
+        });
+        if (!response.ok) {
+            const error = await this.parseError(response);
+            throw new PostgRESTError(`Failed to bulk insert: ${error}`, response.status);
+        }
+    }
+    /**
+     * Count records matching a filter
+     */
+    async count(filter) {
+        const params = new URLSearchParams();
+        params.append('select', 'count');
+        if (filter?.where) {
+            for (const [field, value] of Object.entries(filter.where)) {
+                if (value === null) {
+                    params.append(field, 'is.null');
+                }
+                else {
+                    params.append(field, `eq.${value}`);
+                }
+            }
+        }
+        const url = `${this.baseUrl}?${params.toString()}`;
+        const response = await fetch(url, {
+            method: 'GET',
+            headers: {
+                ...this.headers,
+                'Prefer': 'count=exact',
+            },
+        });
+        if (!response.ok) {
+            throw new PostgRESTError(`Failed to count records: ${response.statusText}`, response.status);
+        }
+        // Count is in the Content-Range header: "0-24/100"
+        const range = response.headers.get('Content-Range');
+        if (range) {
+            const match = range.match(/\/(\d+|\*)/);
+            if (match && match[1] !== '*') {
+                return parseInt(match[1], 10);
+            }
+        }
+        // Fallback: count the results
+        const data = await response.json();
+        return Array.isArray(data) ? data.length : 0;
+    }
+    async parseError(response) {
+        try {
+            const body = await response.json();
+            return body.message || body.error || response.statusText;
+        }
+        catch {
+            return response.statusText;
+        }
+    }
+}
+/**
+ * Error class for PostgREST operations
+ */
+export class PostgRESTError extends Error {
+    statusCode;
+    constructor(message, statusCode) {
+        super(message);
+        this.statusCode = statusCode;
+        this.name = 'PostgRESTError';
+    }
+}

package/dist/stores/stores.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/stores/stores.test.js

@@ -0,0 +1,158 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { rmSync, existsSync } from 'node:fs';
+import { FileStore } from './file.js';
+import { createStore, resolveStoreType } from './factory.js';
+const TEST_DIR = '.reqon-test-data';
+describe('FileStore', () => {
+    let store;
+    beforeEach(() => {
+        // Clean up test directory
+        if (existsSync(TEST_DIR)) {
+            rmSync(TEST_DIR, { recursive: true, force: true });
+        }
+        store = new FileStore('test-store', { baseDir: TEST_DIR });
+    });
+    afterEach(() => {
+        // Clean up after tests
+        if (existsSync(TEST_DIR)) {
+            rmSync(TEST_DIR, { recursive: true, force: true });
+        }
+    });
+    it('creates data directory and .gitignore', () => {
+        expect(existsSync(TEST_DIR)).toBe(true);
+        expect(existsSync(`${TEST_DIR}/.gitignore`)).toBe(true);
+    });
+    it('stores and retrieves data', async () => {
+        await store.set('item-1', { id: '1', name: 'Test Item' });
+        const retrieved = await store.get('item-1');
+        expect(retrieved).toEqual({ id: '1', name: 'Test Item' });
+    });
+    it('persists data to disk', async () => {
+        await store.set('item-1', { id: '1', name: 'Persisted' });
+        // Create new store instance pointing to same file
+        const newStore = new FileStore('test-store', { baseDir: TEST_DIR });
+        const retrieved = await newStore.get('item-1');
+        expect(retrieved).toEqual({ id: '1', name: 'Persisted' });
+    });
+    it('returns null for missing keys', async () => {
+        const result = await store.get('nonexistent');
+        expect(result).toBeNull();
+    });
+    it('updates existing records', async () => {
+        await store.set('item-1', { id: '1', name: 'Original', count: 0 });
+        await store.update('item-1', { count: 5 });
+        const retrieved = await store.get('item-1');
+        expect(retrieved).toEqual({ id: '1', name: 'Original', count: 5 });
+    });
+    it('deletes records', async () => {
+        await store.set('item-1', { id: '1' });
+        await store.delete('item-1');
+        const result = await store.get('item-1');
+        expect(result).toBeNull();
+    });
+    it('lists all records', async () => {
+        await store.set('a', { id: 'a', type: 'x' });
+        await store.set('b', { id: 'b', type: 'y' });
+        await store.set('c', { id: 'c', type: 'x' });
+        const all = await store.list();
+        expect(all).toHaveLength(3);
+    });
+    it('filters records with where clause', async () => {
+        await store.set('a', { id: 'a', type: 'x' });
+        await store.set('b', { id: 'b', type: 'y' });
+        await store.set('c', { id: 'c', type: 'x' });
+        const filtered = await store.list({ where: { type: 'x' } });
+        expect(filtered).toHaveLength(2);
+        expect(filtered.every((r) => r.type === 'x')).toBe(true);
+    });
+    it('clears all records', async () => {
+        await store.set('a', { id: 'a' });
+        await store.set('b', { id: 'b' });
+        await store.clear();
+        const all = await store.list();
+        expect(all).toHaveLength(0);
+    });
+    it('supports batch mode with flush', async () => {
+        const batchStore = new FileStore('batch-test', {
+            baseDir: TEST_DIR,
+            persist: 'batch',
+        });
+        await batchStore.set('item-1', { id: '1' });
+        await batchStore.set('item-2', { id: '2' });
+        // Before flush, new instance won't see changes
+        const beforeFlush = new FileStore('batch-test', { baseDir: TEST_DIR });
+        expect(await beforeFlush.get('item-1')).toBeNull();
+        // After flush, changes are persisted
+        batchStore.flush();
+        const afterFlush = new FileStore('batch-test', { baseDir: TEST_DIR });
+        expect(await afterFlush.get('item-1')).toEqual({ id: '1' });
+    });
+});
+describe('createStore factory', () => {
+    beforeEach(() => {
+        if (existsSync(TEST_DIR)) {
+            rmSync(TEST_DIR, { recursive: true, force: true });
+        }
+    });
+    afterEach(() => {
+        if (existsSync(TEST_DIR)) {
+            rmSync(TEST_DIR, { recursive: true, force: true });
+        }
+    });
+    it('creates memory store', async () => {
+        const store = createStore({ type: 'memory', name: 'test' });
+        await store.set('key', { value: 1 });
+        expect(await store.get('key')).toEqual({ value: 1 });
+    });
+    it('creates file store', async () => {
+        const store = createStore({
+            type: 'file',
+            name: 'test',
+            baseDir: TEST_DIR,
+        });
+        await store.set('key', { value: 1 });
+        // Verify persistence
+        const newStore = createStore({
+            type: 'file',
+            name: 'test',
+            baseDir: TEST_DIR,
+        });
+        expect(await newStore.get('key')).toEqual({ value: 1 });
+    });
+    it('falls back to file for sql in dev mode', async () => {
+        const store = createStore({
+            type: 'sql',
+            name: 'test',
+            baseDir: TEST_DIR,
+        });
+        await store.set('key', { value: 1 });
+        expect(await store.get('key')).toEqual({ value: 1 });
+    });
+    it('falls back to file for nosql in dev mode', async () => {
+        const store = createStore({
+            type: 'nosql',
+            name: 'test',
+            baseDir: TEST_DIR,
+        });
+        await store.set('key', { value: 1 });
+        expect(await store.get('key')).toEqual({ value: 1 });
+    });
+});
+describe('resolveStoreType', () => {
+    it('keeps memory as memory', () => {
+        expect(resolveStoreType('memory', true)).toBe('memory');
+        expect(resolveStoreType('memory', false)).toBe('memory');
+    });
+    it('maps sql to file in dev mode', () => {
+        expect(resolveStoreType('sql', true)).toBe('file');
+    });
+    it('maps nosql to file in dev mode', () => {
+        expect(resolveStoreType('nosql', true)).toBe('file');
+    });
+    it('keeps sql as sql in production mode', () => {
+        expect(resolveStoreType('sql', false)).toBe('sql');
+    });
+    it('keeps nosql as nosql in production mode', () => {
+        expect(resolveStoreType('nosql', false)).toBe('nosql');
+    });
+});

package/dist/stores/types.d.ts

@@ -0,0 +1,31 @@
+export interface StoreAdapter {
+    get(key: string): Promise<Record<string, unknown> | null>;
+    set(key: string, value: Record<string, unknown>): Promise<void>;
+    /** Bulk set multiple records at once - more efficient than individual sets */
+    bulkSet?(records: Array<{
+        key: string;
+        value: Record<string, unknown>;
+    }>): Promise<void>;
+    update(key: string, value: Partial<Record<string, unknown>>): Promise<void>;
+    delete(key: string): Promise<void>;
+    list(filter?: StoreFilter): Promise<Record<string, unknown>[]>;
+    count(filter?: StoreFilter): Promise<number>;
+    clear(): Promise<void>;
+}
+export interface StoreFilter {
+    where?: Record<string, unknown>;
+    limit?: number;
+    offset?: number;
+}
+export interface StoreConfig {
+    type: 'nosql' | 'sql' | 'memory' | 'file';
+    target: string;
+    connection?: string;
+    /** For file stores: 'json' or 'csv' */
+    format?: 'json' | 'csv';
+}
+/**
+ * Apply filter criteria to a list of records.
+ * Handles where clause, offset, and limit.
+ */
+export declare function applyStoreFilter<T extends Record<string, unknown>>(records: T[], filter?: StoreFilter): T[];

package/dist/stores/types.js

@@ -0,0 +1,26 @@
+/**
+ * Apply filter criteria to a list of records.
+ * Handles where clause, offset, and limit.
+ */
+export function applyStoreFilter(records, filter) {
+    if (!filter)
+        return records;
+    let results = records;
+    if (filter.where) {
+        const whereClause = filter.where;
+        results = results.filter((item) => {
+            for (const [key, value] of Object.entries(whereClause)) {
+                if (item[key] !== value)
+                    return false;
+            }
+            return true;
+        });
+    }
+    if (filter.offset) {
+        results = results.slice(filter.offset);
+    }
+    if (filter.limit) {
+        results = results.slice(0, filter.limit);
+    }
+    return results;
+}

package/dist/sync/index.d.ts

@@ -0,0 +1,4 @@
+export type { SyncCheckpoint, SyncState, SinceResolution, SinceDateFormat, } from './state.js';
+export { generateCheckpointKey, formatSinceDate, parseSinceDate, EPOCH, } from './state.js';
+export type { SyncStore } from './store.js';
+export { FileSyncStore, MemorySyncStore } from './store.js';

package/dist/sync/index.js

@@ -0,0 +1,2 @@
+export { generateCheckpointKey, formatSinceDate, parseSinceDate, EPOCH, } from './state.js';
+export { FileSyncStore, MemorySyncStore } from './store.js';

package/dist/sync/state.d.ts

@@ -0,0 +1,69 @@
+/**
+ * Sync State - Tracks last sync timestamps for incremental sync
+ *
+ * Enables "only fetch changed since last run" patterns:
+ * - get "/invoices" { since: lastSync }
+ * - call Xero.getInvoices { since: lastSync("invoices") }
+ */
+/**
+ * Sync checkpoint - records when a sync completed
+ */
+export interface SyncCheckpoint {
+    /** Unique key (e.g., "Xero:getInvoices" or "invoices") */
+    key: string;
+    /** Last successful sync timestamp */
+    syncedAt: Date;
+    /** Number of records fetched in last sync */
+    recordCount?: number;
+    /** Optional cursor for cursor-based pagination resume */
+    cursor?: string;
+    /** Mission that performed the sync */
+    mission?: string;
+    /** Execution ID that performed the sync */
+    executionId?: string;
+}
+/**
+ * Sync state for a mission/source
+ */
+export interface SyncState {
+    /** Mission name */
+    mission: string;
+    /** Source name */
+    source: string;
+    /** Checkpoints by key */
+    checkpoints: Map<string, SyncCheckpoint>;
+    /** When this state was last updated */
+    updatedAt: Date;
+}
+/**
+ * Options for resolving a "since" value
+ */
+export interface SinceResolution {
+    /** The resolved timestamp */
+    timestamp: Date;
+    /** Whether this is a fresh sync (no previous checkpoint) */
+    isFreshSync: boolean;
+    /** The checkpoint key used */
+    key: string;
+}
+/**
+ * Generate a checkpoint key for a fetch operation
+ */
+export declare function generateCheckpointKey(source: string, operationId?: string, endpoint?: string): string;
+/**
+ * Default "since" value for fresh syncs
+ * Uses Unix epoch (1970-01-01) to fetch all historical data
+ */
+export declare const EPOCH: Date;
+/**
+ * Common date formats for API "since" parameters
+ */
+export type SinceDateFormat = 'iso' | 'unix' | 'unix-ms' | 'date-only';
+/**
+ * Format a date for use in API requests
+ */
+export declare function formatSinceDate(date: Date, format?: SinceDateFormat): string;
+/**
+ * Parse a date from various API response formats
+ */
+export declare function parseSinceDate(value: unknown): Date | null;

package/dist/sync/state.js

@@ -0,0 +1,66 @@
+/**
+ * Sync State - Tracks last sync timestamps for incremental sync
+ *
+ * Enables "only fetch changed since last run" patterns:
+ * - get "/invoices" { since: lastSync }
+ * - call Xero.getInvoices { since: lastSync("invoices") }
+ */
+/**
+ * Generate a checkpoint key for a fetch operation
+ */
+export function generateCheckpointKey(source, operationId, endpoint) {
+    if (operationId) {
+        return `${source}:${operationId}`;
+    }
+    if (endpoint) {
+        // Normalize endpoint (remove query params, trailing slashes)
+        const normalized = endpoint.split('?')[0].replace(/\/+$/, '');
+        return `${source}:${normalized}`;
+    }
+    return source;
+}
+/**
+ * Default "since" value for fresh syncs
+ * Uses Unix epoch (1970-01-01) to fetch all historical data
+ */
+export const EPOCH = new Date(0);
+/**
+ * Format a date for use in API requests
+ */
+export function formatSinceDate(date, format = 'iso') {
+    switch (format) {
+        case 'iso':
+            return date.toISOString();
+        case 'unix':
+            return Math.floor(date.getTime() / 1000).toString();
+        case 'unix-ms':
+            return date.getTime().toString();
+        case 'date-only':
+            return date.toISOString().split('T')[0];
+        default:
+            return date.toISOString();
+    }
+}
+/**
+ * Parse a date from various API response formats
+ */
+export function parseSinceDate(value) {
+    if (value instanceof Date) {
+        return value;
+    }
+    if (typeof value === 'string') {
+        const parsed = new Date(value);
+        if (!isNaN(parsed.getTime())) {
+            return parsed;
+        }
+    }
+    if (typeof value === 'number') {
+        // Assume Unix timestamp in seconds if < year 3000 in seconds
+        if (value < 32503680000) {
+            return new Date(value * 1000);
+        }
+        // Otherwise assume milliseconds
+        return new Date(value);
+    }
+    return null;
+}

package/dist/sync/store.d.ts

@@ -0,0 +1,49 @@
+import type { SyncCheckpoint } from './state.js';
+/**
+ * Sync store interface - persists sync checkpoints
+ */
+export interface SyncStore {
+    /** Get last sync timestamp for a key */
+    getLastSync(key: string): Promise<Date>;
+    /** Get checkpoint details for a key */
+    getCheckpoint(key: string): Promise<SyncCheckpoint | null>;
+    /** Record a successful sync */
+    recordSync(checkpoint: SyncCheckpoint): Promise<void>;
+    /** List all checkpoints */
+    list(): Promise<SyncCheckpoint[]>;
+    /** Clear a specific checkpoint */
+    clear(key: string): Promise<void>;
+    /** Clear all checkpoints */
+    clearAll(): Promise<void>;
+}
+/**
+ * File-based sync store
+ * Stores sync state in .reqon-data/sync/{mission}.json
+ */
+export declare class FileSyncStore implements SyncStore {
+    private filePath;
+    private checkpoints;
+    private initialized;
+    constructor(mission: string, baseDir?: string);
+    private init;
+    private load;
+    private persist;
+    getLastSync(key: string): Promise<Date>;
+    getCheckpoint(key: string): Promise<SyncCheckpoint | null>;
+    recordSync(checkpoint: SyncCheckpoint): Promise<void>;
+    list(): Promise<SyncCheckpoint[]>;
+    clear(key: string): Promise<void>;
+    clearAll(): Promise<void>;
+}
+/**
+ * In-memory sync store (for testing)
+ */
+export declare class MemorySyncStore implements SyncStore {
+    private checkpoints;
+    getLastSync(key: string): Promise<Date>;
+    getCheckpoint(key: string): Promise<SyncCheckpoint | null>;
+    recordSync(checkpoint: SyncCheckpoint): Promise<void>;
+    list(): Promise<SyncCheckpoint[]>;
+    clear(key: string): Promise<void>;
+    clearAll(): Promise<void>;
+}