reqon-dsl 0.2.0

package/examples/temporal-comparison/README.md

@@ -0,0 +1,297 @@
+# Reqon vs Temporal: E-Commerce Order Reconciliation
+
+This example demonstrates a complex multi-vendor order reconciliation pipeline that syncs data from Shopify, Stripe, and ShipStation, validates for discrepancies, and stores results in a database.
+
+## The Scenario
+
+An e-commerce company needs to:
+1. Sync orders from Shopify (source of truth for orders)
+2. Sync payments from Stripe (source of truth for payments)
+3. Sync shipments from ShipStation (source of truth for fulfillment)
+4. Cross-reference and validate all three data sources
+5. Flag discrepancies (unpaid orders, missing shipments, refund mismatches)
+6. Store reconciled data with audit trail
+7. Handle rate limits, retries, and pagination across all APIs
+8. Run daily with incremental sync
+
+## The Numbers
+
+| Metric | Reqon | Temporal |
+|--------|-------|----------|
+| **Total Lines of Code** | ~280 | ~1,500+ |
+| **Files** | 1 | 12 |
+| **Dependencies** | 1 (reqon) | 8+ (@temporalio/*, axios, pg, stripe) |
+| **Infrastructure** | None | Temporal Server + PostgreSQL + Workers |
+| **Setup Time** | 0 minutes | 30+ minutes |
+| **Learning Curve** | Hours | Days/Weeks |
+
+## Side-by-Side Comparison
+
+### 1. API Pagination
+
+**Reqon** (2 lines):
+```vague
+paginate: cursor(page_info, 250, "link.next"),
+until: response.orders.length == 0,
+```
+
+**Temporal** (40+ lines):
+```typescript
+let cursor: string | null = null;
+do {
+  Context.current().heartbeat({ page: pageCount, ordersFound: allOrders.length });
+  const { orders, nextCursor } = await client.fetchOrdersPage(cursor, since);
+  for (const order of orders) {
+    allOrders.push(transformOrder(order));
+  }
+  cursor = nextCursor;
+  pageCount++;
+} while (cursor);
+```
+
+### 2. Rate Limiting
+
+**Reqon** (4 lines):
+```vague
+rateLimit: {
+  strategy: "pause",
+  maxWait: 120,
+  fallbackRpm: 40
+}
+```
+
+**Temporal** (50+ lines):
+```typescript
+class ShopifyClient {
+  private rateLimitState: RateLimitState | null = null;
+
+  private updateRateLimitState(headers: Record<string, string>): void {
+    // Parse headers, calculate remaining, track reset time...
+  }
+
+  private async waitForRateLimit(): Promise<void> {
+    if (!this.rateLimitState) return;
+    if (this.rateLimitState.remaining < 5) {
+      const waitTime = Math.max(0, this.rateLimitState.resetAt.getTime() - Date.now());
+      if (waitTime > 0) {
+        await new Promise((resolve) => setTimeout(resolve, waitTime));
+      }
+    }
+  }
+}
+```
+
+### 3. Schema Transformation
+
+**Reqon** (15 lines):
+```vague
+map order -> UnifiedOrder {
+  order_id: "shopify_" + .id,
+  customer_email: .customer.email,
+  total_amount: .total_price,
+  payment_status: match .financial_status {
+    "paid" => "captured",
+    "pending" => "pending",
+    "refunded" => "refunded",
+    _ => "unknown"
+  }
+}
+```
+
+**Temporal** (30+ lines):
+```typescript
+function transformOrder(order: ShopifyOrder): UnifiedOrder {
+  const paymentStatusMap: Record<string, string> = {
+    paid: 'captured',
+    partially_paid: 'partial',
+    pending: 'pending',
+    refunded: 'refunded',
+    partially_refunded: 'partial_refund',
+  };
+
+  return {
+    order_id: `shopify_${order.id}`,
+    customer_email: order.customer?.email || '',
+    total_amount: parseFloat(order.total_price),
+    payment_status: paymentStatusMap[order.financial_status] || 'unknown',
+    // ... 10 more fields
+  };
+}
+```
+
+### 4. Validation Rules
+
+**Reqon** (5 lines):
+```vague
+validate order {
+  assume payment_exists == true
+} or {
+  store { type: "missing_payment", ... } -> discrepancies { ... }
+}
+```
+
+**Temporal** (50+ lines):
+```typescript
+export async function validateOrdersHavePayments(
+  orders: UnifiedOrder[],
+  payments: UnifiedPayment[]
+): Promise<Discrepancy[]> {
+  const discrepancies: Discrepancy[] = [];
+  const paymentsByOrderId = new Map<string, UnifiedPayment[]>();
+
+  for (const payment of payments) {
+    const existing = paymentsByOrderId.get(payment.order_id) || [];
+    existing.push(payment);
+    paymentsByOrderId.set(payment.order_id, existing);
+  }
+
+  const threeDaysAgo = new Date(Date.now() - 3 * 24 * 60 * 60 * 1000);
+
+  for (const order of orders) {
+    if (order.payment_status === 'pending' && order.created_at < threeDaysAgo) {
+      const orderPayments = paymentsByOrderId.get(order.order_id);
+      if (!orderPayments || orderPayments.length === 0) {
+        discrepancies.push({
+          id: `disc_${order.order_id}_no_payment`,
+          // ... more fields
+        });
+      }
+    }
+  }
+  return discrepancies;
+}
+```
+
+### 5. Pipeline Orchestration with Parallel Execution
+
+**Reqon** (3 lines):
+```vague
+// Fetch from all three APIs in parallel, then validate
+run [SyncShopifyOrders, SyncStripePayments, SyncShipStationShipments]
+  then ValidateReconciliation
+```
+Brackets indicate parallel execution - all three sync actions run concurrently for maximum performance.
+
+**Temporal** (100+ lines):
+```typescript
+export async function orderReconciliationWorkflow(): Promise<ReconciliationResult> {
+  // Initialize state, queries, signals...
+  setHandler(getStatusQuery, () => state);
+  setHandler(cancelSignal, () => { cancelled = true; });
+  setHandler(pauseSignal, () => { paused = true; });
+  setHandler(resumeSignal, () => { paused = false; });
+
+  try {
+    state.status = 'syncing_shopify';
+    await checkPauseAndCancel();
+    const shopifyLastSync = await storage.getCheckpoint('shopify_orders_last_sync');
+    const orders = await shopify.fetchAllShopifyOrders(shopifyLastSync);
+    state.ordersProcessed = await storage.upsertOrders(orders);
+    await storage.setCheckpoint('shopify_orders_last_sync', new Date().toISOString());
+
+    // Repeat for Stripe...
+    // Repeat for ShipStation...
+    // Validation step...
+  } catch (error) {
+    state.status = 'failed';
+    throw error;
+  }
+}
+```
+
+### 6. Running the Pipeline
+
+**Reqon**:
+```bash
+# Run once
+reqon reconciliation.vague --auth ./credentials.json
+
+# Dry run
+reqon reconciliation.vague --dry-run
+
+# Resume from checkpoint
+reqon reconciliation.vague --resume exec-abc123
+
+# Schedule with cron
+0 2 * * * reqon reconciliation.vague
+```
+
+**Temporal**:
+```bash
+# 1. Start PostgreSQL
+docker-compose up -d postgresql
+
+# 2. Start Temporal Server
+docker-compose up -d temporal temporal-ui
+
+# 3. Wait for server to be ready
+sleep 30
+
+# 4. Start worker (in separate terminal)
+npx ts-node src/workers/main.ts
+
+# 5. Trigger workflow
+npx ts-node src/client/schedule.ts run
+
+# 6. (Optional) Create schedule
+npx ts-node src/client/schedule.ts schedule
+```
+
+## When to Use Which
+
+### Use Reqon When:
+- ✅ Your workflow is data synchronization / ETL
+- ✅ You need to fetch, transform, validate, and store data
+- ✅ You want minimal infrastructure
+- ✅ Business users need to understand the pipeline
+- ✅ You need quick iteration and prototyping
+- ✅ The "happy path" is the main path
+- ✅ Retry/backoff/rate-limiting are your main concerns
+
+### Use Temporal When:
+- ⚡ You need human-in-the-loop workflows
+- ⚡ You have complex branching/conditional logic
+- ⚡ You need workflow versioning/migration
+- ⚡ You need sub-second latency on signals
+- ⚡ You're building microservice orchestration
+- ⚡ You need the full power of a programming language
+- ⚡ You already have Temporal infrastructure
+
+## Files
+
+### Reqon Solution
+- `reconciliation.vague` - **The complete solution in ~280 lines**
+
+### Temporal Solution (11 files, ~1,500 lines)
+```
+temporal/
+├── types/
+│   └── index.ts          # Type definitions (130 lines)
+├── config/
+│   └── retry.ts          # Retry configuration (50 lines)
+├── activities/
+│   ├── index.ts          # Activity exports
+│   ├── shopify.ts        # Shopify API (220 lines)
+│   ├── stripe.ts         # Stripe API (150 lines)
+│   ├── shipstation.ts    # ShipStation API (180 lines)
+│   ├── validation.ts     # Validation logic (180 lines)
+│   └── storage.ts        # Database operations (160 lines)
+├── workflows/
+│   └── orderReconciliation.ts  # Main workflow (200 lines)
+├── workers/
+│   └── main.ts           # Worker setup (100 lines)
+└── client/
+    └── schedule.ts       # Scheduler client (180 lines)
+```
+
+## The Bottom Line
+
+For data synchronization pipelines, Reqon offers:
+
+1. **10x less code** - Focus on *what* you want, not *how* to do it
+2. **Zero infrastructure** - Run anywhere Node.js runs
+3. **Domain-specific** - Built specifically for fetch/transform/validate/store
+4. **Readable** - Business logic is visible, not buried in code
+5. **Batteries included** - Pagination, rate limiting, retries, checkpointing built-in
+
+Temporal is a powerful general-purpose workflow engine, but for the specific domain of data pipelines, it's like using a chainsaw to cut butter. Reqon is purpose-built for this domain and shows in every aspect of the developer experience.

package/examples/temporal-comparison/reconciliation.vague

@@ -0,0 +1,355 @@
+// E-Commerce Order Reconciliation Pipeline
+// Syncs and validates orders across Shopify, Stripe, and ShipStation
+// ~150 lines of declarative code vs ~1,500+ lines in Temporal
+
+mission OrderReconciliation {
+
+  // ============================================================
+  // SOURCES - API definitions with auth and rate limiting
+  // ============================================================
+
+  source Shopify {
+    auth: oauth2,
+    base: "https://mystore.myshopify.com/admin/api/2024-01",
+    headers: { "X-Shopify-Access-Token": "${SHOPIFY_TOKEN}" },
+    rateLimit: {
+      strategy: "pause",
+      maxWait: 120,
+      fallbackRpm: 40
+    }
+  }
+
+  source Stripe {
+    auth: bearer,
+    base: "https://api.stripe.com/v1",
+    rateLimit: {
+      strategy: "throttle",
+      maxWait: 60,
+      fallbackRpm: 100
+    }
+  }
+
+  source ShipStation {
+    auth: basic,
+    base: "https://ssapi.shipstation.com",
+    rateLimit: {
+      strategy: "pause",
+      maxWait: 30,
+      fallbackRpm: 40
+    }
+  }
+
+  // ============================================================
+  // STORES - Where we persist reconciled data
+  // ============================================================
+
+  store orders: sql("reconciled_orders")
+  store payments: sql("reconciled_payments")
+  store shipments: sql("reconciled_shipments")
+  store discrepancies: sql("audit_discrepancies")
+  store sync_state: sql("sync_checkpoints")
+
+  // ============================================================
+  // SCHEMAS - Unified data models
+  // ============================================================
+
+  schema UnifiedOrder {
+    order_id: string,
+    external_id: string,
+    source: string,
+    customer_email: string,
+    total_amount: decimal,
+    currency: string,
+    status: string,
+    created_at: date,
+    line_items_count: int,
+    payment_status: string,
+    fulfillment_status: string,
+    synced_at: date
+  }
+
+  schema UnifiedPayment {
+    payment_id: string,
+    order_id: string,
+    amount: decimal,
+    currency: string,
+    status: string,
+    method: string,
+    captured_at: date,
+    refunded_amount: decimal
+  }
+
+  schema UnifiedShipment {
+    shipment_id: string,
+    order_id: string,
+    carrier: string,
+    tracking_number: string,
+    status: string,
+    shipped_at: date,
+    delivered_at: date
+  }
+
+  schema Discrepancy {
+    id: string,
+    order_id: string,
+    type: string,
+    severity: string,
+    description: string,
+    shopify_value: string,
+    stripe_value: string,
+    shipstation_value: string,
+    detected_at: date
+  }
+
+  // ============================================================
+  // ACTION: Sync Shopify Orders (with incremental sync)
+  // ============================================================
+
+  action SyncShopifyOrders {
+    // Get last sync timestamp for incremental sync
+    get "/orders.json" {
+      source: "Shopify",
+      body: {
+        "status": "any",
+        "updated_at_min": sync_state.get("shopify_orders_last_sync"),
+        "limit": 250
+      },
+      paginate: cursor(page_info, 250, "link.next"),
+      until: response.orders.length == 0,
+      retry: {
+        maxAttempts: 5,
+        backoff: "exponential",
+        initialDelay: 2000,
+        maxDelay: 60000
+      }
+    },
+
+    // Transform each order to unified schema
+    for order in response.orders {
+      map order -> UnifiedOrder {
+        order_id: "shopify_" + .id,
+        external_id: .id,
+        source: "shopify",
+        customer_email: .customer.email,
+        total_amount: .total_price,
+        currency: .currency,
+        status: .status,
+        created_at: .created_at,
+        line_items_count: length(.line_items),
+        payment_status: match .financial_status {
+          "paid" => "captured",
+          "partially_paid" => "partial",
+          "pending" => "pending",
+          "refunded" => "refunded",
+          "partially_refunded" => "partial_refund",
+          _ => "unknown"
+        },
+        fulfillment_status: match .fulfillment_status {
+          "fulfilled" => "shipped",
+          "partial" => "partial",
+          null => "unfulfilled",
+          _ => .fulfillment_status
+        },
+        synced_at: now()
+      },
+
+      store response -> orders { key: .order_id, upsert: true }
+    },
+
+    // Update checkpoint
+    store { "shopify_orders_last_sync": now() } -> sync_state { key: "shopify_orders_last_sync" }
+  }
+
+  // ============================================================
+  // ACTION: Sync Stripe Payments
+  // ============================================================
+
+  action SyncStripePayments {
+    get "/payment_intents" {
+      source: "Stripe",
+      body: {
+        "limit": 100,
+        "created[gte]": sync_state.get("stripe_payments_last_sync"),
+        "expand[]": "data.charges"
+      },
+      paginate: cursor(starting_after, 100, "data[-1].id"),
+      until: response.has_more == false,
+      retry: {
+        maxAttempts: 3,
+        backoff: "exponential",
+        initialDelay: 1000
+      }
+    },
+
+    for payment in response.data where .metadata.shopify_order_id != null {
+      map payment -> UnifiedPayment {
+        payment_id: "stripe_" + .id,
+        order_id: "shopify_" + .metadata.shopify_order_id,
+        amount: .amount / 100,  // Stripe uses cents
+        currency: .currency,
+        status: match .status {
+          "succeeded" => "captured",
+          "requires_capture" => "authorized",
+          "canceled" => "voided",
+          _ => .status
+        },
+        method: .payment_method_types[0],
+        captured_at: .charges.data[0].created,
+        refunded_amount: .charges.data[0].amount_refunded / 100
+      },
+
+      store response -> payments { key: .payment_id, upsert: true }
+    },
+
+    store { "stripe_payments_last_sync": now() } -> sync_state { key: "stripe_payments_last_sync" }
+  }
+
+  // ============================================================
+  // ACTION: Sync ShipStation Shipments
+  // ============================================================
+
+  action SyncShipStationShipments {
+    get "/shipments" {
+      source: "ShipStation",
+      body: {
+        "pageSize": 500,
+        "createDateStart": sync_state.get("shipstation_last_sync"),
+        "sortBy": "CreateDate"
+      },
+      paginate: page(page, 500),
+      until: response.shipments.length == 0,
+      retry: {
+        maxAttempts: 4,
+        backoff: "exponential",
+        initialDelay: 3000
+      }
+    },
+
+    for shipment in response.shipments {
+      map shipment -> UnifiedShipment {
+        shipment_id: "ss_" + .shipmentId,
+        order_id: "shopify_" + .orderNumber,
+        carrier: .carrierCode,
+        tracking_number: .trackingNumber,
+        status: match .voided {
+          true => "voided",
+          _ => match .trackingNumber {
+            null => "label_created",
+            _ => "shipped"
+          }
+        },
+        shipped_at: .shipDate,
+        delivered_at: .deliveryDate
+      },
+
+      store response -> shipments { key: .shipment_id, upsert: true }
+    },
+
+    store { "shipstation_last_sync": now() } -> sync_state { key: "shipstation_last_sync" }
+  }
+
+  // ============================================================
+  // ACTION: Cross-Reference and Validate
+  // ============================================================
+
+  action ValidateReconciliation {
+    // Check for orders without payments
+    for order in orders where .payment_status == "pending" and .created_at < now() - days(3) {
+      let payment_exists = any of payments where .order_id == order.order_id,
+
+      validate order {
+        assume payment_exists == true
+      } or {
+        store {
+          id: "disc_" + order.order_id + "_no_payment",
+          order_id: order.order_id,
+          type: "missing_payment",
+          severity: "high",
+          description: "Order older than 3 days has no matching Stripe payment",
+          shopify_value: order.total_amount,
+          stripe_value: null,
+          shipstation_value: null,
+          detected_at: now()
+        } -> discrepancies { key: .id, upsert: true }
+      }
+    },
+
+    // Check for payment/order amount mismatches
+    for order in orders {
+      let matching_payments = payments where .order_id == order.order_id,
+      let total_paid = sum(matching_payments.amount),
+
+      validate order {
+        assume total_paid >= order.total_amount - 0.01,
+        assume total_paid <= order.total_amount + 0.01
+      } or {
+        store {
+          id: "disc_" + order.order_id + "_amount_mismatch",
+          order_id: order.order_id,
+          type: "amount_mismatch",
+          severity: match abs(order.total_amount - total_paid) {
+            x where x > 100 => "critical",
+            x where x > 10 => "high",
+            _ => "medium"
+          },
+          description: "Order total does not match payment total",
+          shopify_value: order.total_amount,
+          stripe_value: total_paid,
+          shipstation_value: null,
+          detected_at: now()
+        } -> discrepancies { key: .id, upsert: true }
+      }
+    },
+
+    // Check for shipped orders without shipment records
+    for order in orders where .fulfillment_status == "shipped" {
+      let shipment_exists = any of shipments where .order_id == order.order_id,
+
+      validate order {
+        assume shipment_exists == true
+      } or {
+        store {
+          id: "disc_" + order.order_id + "_no_shipment",
+          order_id: order.order_id,
+          type: "missing_shipment_record",
+          severity: "medium",
+          description: "Shopify shows fulfilled but no ShipStation shipment found",
+          shopify_value: order.fulfillment_status,
+          stripe_value: null,
+          shipstation_value: null,
+          detected_at: now()
+        } -> discrepancies { key: .id, upsert: true }
+      }
+    },
+
+    // Check for refund discrepancies
+    for payment in payments where .refunded_amount > 0 {
+      let order = first(orders where .order_id == payment.order_id),
+
+      validate payment {
+        assume order.payment_status == "refunded" or order.payment_status == "partial_refund"
+      } or {
+        store {
+          id: "disc_" + payment.order_id + "_refund_mismatch",
+          order_id: payment.order_id,
+          type: "refund_status_mismatch",
+          severity: "high",
+          description: "Stripe shows refund but Shopify payment status not updated",
+          shopify_value: order.payment_status,
+          stripe_value: payment.refunded_amount,
+          shipstation_value: null,
+          detected_at: now()
+        } -> discrepancies { key: .id, upsert: true }
+      }
+    }
+  }
+
+  // ============================================================
+  // PIPELINE: Parallel fetch, then sequential validation
+  // ============================================================
+
+  // Fetch from all three APIs in parallel for maximum performance
+  // ValidateReconciliation runs after all three complete
+  run [SyncShopifyOrders, SyncStripePayments, SyncShipStationShipments]
+    then ValidateReconciliation
+}

package/examples/temporal-comparison/temporal/activities/index.ts

@@ -0,0 +1,8 @@
+// Activity exports
+// Aggregate all activities for worker registration
+
+export * from './shopify';
+export * from './stripe';
+export * from './shipstation';
+export * from './validation';
+export * from './storage';