reqon-dsl 0.2.0

package/examples/temporal-comparison/temporal/activities/shipstation.ts

@@ -0,0 +1,225 @@
+// ShipStation API activities
+// Yet another 200+ lines of boilerplate
+
+import axios, { AxiosInstance, AxiosError } from 'axios';
+import { Context } from '@temporalio/activity';
+import { ShipStationShipment, UnifiedShipment } from '../types';
+
+// Custom errors
+export class ShipStationAuthError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'ShipStationAuthError';
+  }
+}
+
+export class ShipStationRateLimitError extends Error {
+  retryAfter: number;
+  constructor(message: string, retryAfter: number) {
+    super(message);
+    this.name = 'ShipStationRateLimitError';
+    this.retryAfter = retryAfter;
+  }
+}
+
+// ShipStation rate limit tracking
+interface RateLimitState {
+  remaining: number;
+  resetAt: Date;
+}
+
+class ShipStationClient {
+  private client: AxiosInstance;
+  private rateLimitState: RateLimitState | null = null;
+
+  constructor() {
+    const apiKey = process.env.SHIPSTATION_API_KEY;
+    const apiSecret = process.env.SHIPSTATION_API_SECRET;
+
+    if (!apiKey || !apiSecret) {
+      throw new ShipStationAuthError('Missing ShipStation credentials');
+    }
+
+    // ShipStation uses Basic Auth with API key:secret
+    const authToken = Buffer.from(`${apiKey}:${apiSecret}`).toString('base64');
+
+    this.client = axios.create({
+      baseURL: 'https://ssapi.shipstation.com',
+      headers: {
+        Authorization: `Basic ${authToken}`,
+        'Content-Type': 'application/json',
+      },
+    });
+
+    this.client.interceptors.response.use(
+      (response) => {
+        this.updateRateLimitState(response.headers);
+        return response;
+      },
+      async (error: AxiosError) => {
+        if (error.response?.status === 429) {
+          // ShipStation uses X-Rate-Limit-Reset header
+          const resetTime = error.response.headers['x-rate-limit-reset'];
+          const retryAfter = resetTime
+            ? new Date(resetTime).getTime() - Date.now()
+            : 30000;
+          throw new ShipStationRateLimitError('Rate limit exceeded', retryAfter);
+        }
+        if (error.response?.status === 401) {
+          throw new ShipStationAuthError('Invalid ShipStation credentials');
+        }
+        throw error;
+      }
+    );
+  }
+
+  private updateRateLimitState(headers: Record<string, string>): void {
+    const remaining = parseInt(headers['x-rate-limit-remaining'] || '40', 10);
+    const resetTime = headers['x-rate-limit-reset'];
+
+    this.rateLimitState = {
+      remaining,
+      resetAt: resetTime ? new Date(resetTime) : new Date(Date.now() + 60000),
+    };
+  }
+
+  private async waitForRateLimit(): Promise<void> {
+    if (!this.rateLimitState) return;
+
+    // ShipStation has very strict limits - 40 requests per minute
+    if (this.rateLimitState.remaining < 3) {
+      const waitTime = Math.max(
+        0,
+        this.rateLimitState.resetAt.getTime() - Date.now()
+      );
+      if (waitTime > 0) {
+        console.log(
+          `ShipStation rate limit low (${this.rateLimitState.remaining}), waiting ${waitTime}ms`
+        );
+        await new Promise((resolve) => setTimeout(resolve, waitTime));
+      }
+    }
+  }
+
+  async fetchShipmentsPage(
+    page: number,
+    since: string | null
+  ): Promise<{
+    shipments: ShipStationShipment[];
+    totalPages: number;
+  }> {
+    await this.waitForRateLimit();
+
+    const params: Record<string, string | number> = {
+      pageSize: 500,
+      page,
+      sortBy: 'CreateDate',
+    };
+
+    if (since) {
+      params.createDateStart = since;
+    }
+
+    const response = await this.client.get('/shipments', { params });
+
+    return {
+      shipments: response.data.shipments || [],
+      totalPages: response.data.pages || 1,
+    };
+  }
+}
+
+let shipStationClient: ShipStationClient | null = null;
+
+function getClient(): ShipStationClient {
+  if (!shipStationClient) {
+    shipStationClient = new ShipStationClient();
+  }
+  return shipStationClient;
+}
+
+// Transform ShipStation shipment to unified format
+function transformShipment(shipment: ShipStationShipment): UnifiedShipment {
+  let status: string;
+  if (shipment.voided) {
+    status = 'voided';
+  } else if (!shipment.trackingNumber) {
+    status = 'label_created';
+  } else {
+    status = 'shipped';
+  }
+
+  return {
+    shipment_id: `ss_${shipment.shipmentId}`,
+    order_id: `shopify_${shipment.orderNumber}`,
+    carrier: shipment.carrierCode,
+    tracking_number: shipment.trackingNumber,
+    status,
+    shipped_at: new Date(shipment.shipDate),
+    delivered_at: shipment.deliveryDate ? new Date(shipment.deliveryDate) : null,
+  };
+}
+
+// Activity: Fetch all ShipStation shipments with pagination
+export async function fetchAllShipStationShipments(
+  since: string | null
+): Promise<UnifiedShipment[]> {
+  const client = getClient();
+  const allShipments: UnifiedShipment[] = [];
+  let currentPage = 1;
+  let totalPages = 1;
+
+  do {
+    // Heartbeat for long-running pagination
+    Context.current().heartbeat({
+      page: currentPage,
+      totalPages,
+      shipmentsFound: allShipments.length,
+    });
+
+    const { shipments, totalPages: pages } = await client.fetchShipmentsPage(
+      currentPage,
+      since
+    );
+
+    totalPages = pages;
+
+    for (const shipment of shipments) {
+      allShipments.push(transformShipment(shipment));
+    }
+
+    console.log(
+      `ShipStation: Fetched page ${currentPage}/${totalPages}, total shipments: ${allShipments.length}`
+    );
+
+    currentPage++;
+  } while (currentPage <= totalPages);
+
+  return allShipments;
+}
+
+// Activity: Fetch shipment by tracking number
+export async function fetchShipmentByTracking(
+  trackingNumber: string
+): Promise<UnifiedShipment | null> {
+  const client = getClient();
+
+  const { shipments } = await (client as any).client.get('/shipments', {
+    params: { trackingNumber },
+  });
+
+  if (shipments?.length > 0) {
+    return transformShipment(shipments[0]);
+  }
+  return null;
+}
+
+// Activity: Get last sync timestamp
+export async function getShipStationLastSync(): Promise<string | null> {
+  return process.env.SHIPSTATION_LAST_SYNC || null;
+}
+
+// Activity: Update last sync timestamp
+export async function updateShipStationLastSync(timestamp: string): Promise<void> {
+  console.log(`Updated ShipStation last sync to: ${timestamp}`);
+}

package/examples/temporal-comparison/temporal/activities/shopify.ts

@@ -0,0 +1,257 @@
+// Shopify API activities
+// This single file handles what Reqon does with a few lines of DSL
+
+import axios, { AxiosInstance, AxiosError } from 'axios';
+import { Context } from '@temporalio/activity';
+import { ShopifyOrder, UnifiedOrder, PaginatedResponse } from '../types';
+
+// Custom error types for non-retryable errors
+export class ShopifyAuthError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'ShopifyAuthError';
+  }
+}
+
+export class ShopifyNotFoundError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'ShopifyNotFoundError';
+  }
+}
+
+export class ShopifyRateLimitError extends Error {
+  retryAfter: number;
+  constructor(message: string, retryAfter: number) {
+    super(message);
+    this.name = 'ShopifyRateLimitError';
+    this.retryAfter = retryAfter;
+  }
+}
+
+// Rate limiter state - in Temporal, we need to manage this ourselves
+interface RateLimitState {
+  remaining: number;
+  limit: number;
+  resetAt: Date;
+  lastRequest: Date;
+}
+
+class ShopifyClient {
+  private client: AxiosInstance;
+  private rateLimitState: RateLimitState | null = null;
+
+  constructor() {
+    const shopifyDomain = process.env.SHOPIFY_DOMAIN;
+    const accessToken = process.env.SHOPIFY_ACCESS_TOKEN;
+
+    if (!shopifyDomain || !accessToken) {
+      throw new ShopifyAuthError('Missing Shopify credentials');
+    }
+
+    this.client = axios.create({
+      baseURL: `https://${shopifyDomain}/admin/api/2024-01`,
+      headers: {
+        'X-Shopify-Access-Token': accessToken,
+        'Content-Type': 'application/json',
+      },
+    });
+
+    // Response interceptor for rate limit tracking
+    this.client.interceptors.response.use(
+      (response) => {
+        this.updateRateLimitState(response.headers);
+        return response;
+      },
+      async (error: AxiosError) => {
+        if (error.response?.status === 429) {
+          const retryAfter = parseInt(
+            error.response.headers['retry-after'] || '60',
+            10
+          );
+          throw new ShopifyRateLimitError(
+            'Rate limit exceeded',
+            retryAfter * 1000
+          );
+        }
+        if (error.response?.status === 401) {
+          throw new ShopifyAuthError('Invalid Shopify credentials');
+        }
+        if (error.response?.status === 404) {
+          throw new ShopifyNotFoundError('Resource not found');
+        }
+        throw error;
+      }
+    );
+  }
+
+  private updateRateLimitState(headers: Record<string, string>): void {
+    const remaining = parseInt(headers['x-shopify-shop-api-call-limit'] || '40/40', 10);
+    const [used, limit] = (headers['x-shopify-shop-api-call-limit'] || '0/40')
+      .split('/')
+      .map(Number);
+
+    this.rateLimitState = {
+      remaining: limit - used,
+      limit,
+      resetAt: new Date(Date.now() + 60000), // Shopify resets every minute
+      lastRequest: new Date(),
+    };
+  }
+
+  private async waitForRateLimit(): Promise<void> {
+    if (!this.rateLimitState) return;
+
+    // If we're getting low on requests, slow down
+    if (this.rateLimitState.remaining < 5) {
+      const waitTime = Math.max(
+        0,
+        this.rateLimitState.resetAt.getTime() - Date.now()
+      );
+      if (waitTime > 0) {
+        console.log(`Rate limit low (${this.rateLimitState.remaining}), waiting ${waitTime}ms`);
+        await new Promise((resolve) => setTimeout(resolve, waitTime));
+      }
+    }
+  }
+
+  async fetchOrdersPage(
+    cursor: string | null,
+    since: string | null
+  ): Promise<{
+    orders: ShopifyOrder[];
+    nextCursor: string | null;
+  }> {
+    await this.waitForRateLimit();
+
+    const params: Record<string, string> = {
+      status: 'any',
+      limit: '250',
+    };
+
+    if (since) {
+      params.updated_at_min = since;
+    }
+
+    if (cursor) {
+      params.page_info = cursor;
+    }
+
+    const response = await this.client.get('/orders.json', { params });
+
+    // Parse Link header for pagination
+    const linkHeader = response.headers.link || '';
+    const nextMatch = linkHeader.match(/<[^>]*page_info=([^>&]+)[^>]*>;\s*rel="next"/);
+    const nextCursor = nextMatch ? nextMatch[1] : null;
+
+    return {
+      orders: response.data.orders,
+      nextCursor,
+    };
+  }
+}
+
+// Singleton client instance
+let shopifyClient: ShopifyClient | null = null;
+
+function getClient(): ShopifyClient {
+  if (!shopifyClient) {
+    shopifyClient = new ShopifyClient();
+  }
+  return shopifyClient;
+}
+
+// Transform Shopify order to unified format
+// In Reqon, this is just `map order -> UnifiedOrder { ... }`
+function transformOrder(order: ShopifyOrder): UnifiedOrder {
+  const paymentStatusMap: Record<string, string> = {
+    paid: 'captured',
+    partially_paid: 'partial',
+    pending: 'pending',
+    refunded: 'refunded',
+    partially_refunded: 'partial_refund',
+  };
+
+  const fulfillmentStatusMap: Record<string, string> = {
+    fulfilled: 'shipped',
+    partial: 'partial',
+  };
+
+  return {
+    order_id: `shopify_${order.id}`,
+    external_id: order.id,
+    source: 'shopify',
+    customer_email: order.customer?.email || '',
+    total_amount: parseFloat(order.total_price),
+    currency: order.currency,
+    status: order.status,
+    created_at: new Date(order.created_at),
+    line_items_count: order.line_items?.length || 0,
+    payment_status: paymentStatusMap[order.financial_status] || 'unknown',
+    fulfillment_status:
+      order.fulfillment_status
+        ? fulfillmentStatusMap[order.fulfillment_status] || order.fulfillment_status
+        : 'unfulfilled',
+    synced_at: new Date(),
+  };
+}
+
+// Activity: Fetch all Shopify orders with pagination
+// This is what Reqon does automatically with `paginate: cursor(...)`
+export async function fetchAllShopifyOrders(
+  since: string | null
+): Promise<UnifiedOrder[]> {
+  const client = getClient();
+  const allOrders: UnifiedOrder[] = [];
+  let cursor: string | null = null;
+  let pageCount = 0;
+
+  do {
+    // Heartbeat to keep the activity alive during long pagination
+    Context.current().heartbeat({ page: pageCount, ordersFound: allOrders.length });
+
+    const { orders, nextCursor } = await client.fetchOrdersPage(cursor, since);
+
+    // Transform each order
+    for (const order of orders) {
+      allOrders.push(transformOrder(order));
+    }
+
+    cursor = nextCursor;
+    pageCount++;
+
+    // Log progress
+    console.log(`Shopify: Fetched page ${pageCount}, total orders: ${allOrders.length}`);
+  } while (cursor);
+
+  return allOrders;
+}
+
+// Activity: Fetch a single order by ID
+export async function fetchShopifyOrder(orderId: string): Promise<UnifiedOrder | null> {
+  const client = getClient();
+
+  try {
+    const response = await (client as any).client.get(`/orders/${orderId}.json`);
+    return transformOrder(response.data.order);
+  } catch (error) {
+    if (error instanceof ShopifyNotFoundError) {
+      return null;
+    }
+    throw error;
+  }
+}
+
+// Activity: Get last sync timestamp
+export async function getShopifyLastSync(): Promise<string | null> {
+  // This would query the database - simplified for example
+  // In Reqon: sync_state.get("shopify_orders_last_sync")
+  return process.env.SHOPIFY_LAST_SYNC || null;
+}
+
+// Activity: Update last sync timestamp
+export async function updateShopifyLastSync(timestamp: string): Promise<void> {
+  // This would update the database - simplified for example
+  // In Reqon: store { "shopify_orders_last_sync": now() } -> sync_state { key: ... }
+  console.log(`Updated Shopify last sync to: ${timestamp}`);
+}

package/examples/temporal-comparison/temporal/activities/storage.ts

@@ -0,0 +1,198 @@
+// Storage activities
+// In Reqon: `store response -> orders { key: .order_id, upsert: true }`
+// In Temporal: We need a full database abstraction layer
+
+import { Pool } from 'pg'; // Would use real Postgres
+import {
+  UnifiedOrder,
+  UnifiedPayment,
+  UnifiedShipment,
+  Discrepancy,
+  SyncCheckpoint,
+} from '../types';
+
+// In production, this would be a real database connection pool
+// For this example, we'll use in-memory maps (similar to Reqon's memory store)
+const ordersStore = new Map<string, UnifiedOrder>();
+const paymentsStore = new Map<string, UnifiedPayment>();
+const shipmentsStore = new Map<string, UnifiedShipment>();
+const discrepanciesStore = new Map<string, Discrepancy>();
+const checkpointsStore = new Map<string, SyncCheckpoint>();
+
+// Activity: Upsert orders
+export async function upsertOrders(orders: UnifiedOrder[]): Promise<number> {
+  let count = 0;
+
+  for (const order of orders) {
+    ordersStore.set(order.order_id, order);
+    count++;
+  }
+
+  console.log(`Upserted ${count} orders`);
+  return count;
+}
+
+// Activity: Upsert payments
+export async function upsertPayments(payments: UnifiedPayment[]): Promise<number> {
+  let count = 0;
+
+  for (const payment of payments) {
+    paymentsStore.set(payment.payment_id, payment);
+    count++;
+  }
+
+  console.log(`Upserted ${count} payments`);
+  return count;
+}
+
+// Activity: Upsert shipments
+export async function upsertShipments(shipments: UnifiedShipment[]): Promise<number> {
+  let count = 0;
+
+  for (const shipment of shipments) {
+    shipmentsStore.set(shipment.shipment_id, shipment);
+    count++;
+  }
+
+  console.log(`Upserted ${count} shipments`);
+  return count;
+}
+
+// Activity: Upsert discrepancies
+export async function upsertDiscrepancies(
+  discrepancies: Discrepancy[]
+): Promise<number> {
+  let count = 0;
+
+  for (const discrepancy of discrepancies) {
+    discrepanciesStore.set(discrepancy.id, discrepancy);
+    count++;
+  }
+
+  console.log(`Upserted ${count} discrepancies`);
+  return count;
+}
+
+// Activity: Get all orders
+export async function getAllOrders(): Promise<UnifiedOrder[]> {
+  return Array.from(ordersStore.values());
+}
+
+// Activity: Get all payments
+export async function getAllPayments(): Promise<UnifiedPayment[]> {
+  return Array.from(paymentsStore.values());
+}
+
+// Activity: Get all shipments
+export async function getAllShipments(): Promise<UnifiedShipment[]> {
+  return Array.from(shipmentsStore.values());
+}
+
+// Activity: Get checkpoint
+export async function getCheckpoint(key: string): Promise<string | null> {
+  const checkpoint = checkpointsStore.get(key);
+  return checkpoint?.value || null;
+}
+
+// Activity: Set checkpoint
+export async function setCheckpoint(key: string, value: string): Promise<void> {
+  checkpointsStore.set(key, {
+    key,
+    value,
+    updated_at: new Date(),
+  });
+  console.log(`Set checkpoint ${key} = ${value}`);
+}
+
+// Activity: Get orders by filter
+export async function getOrdersByStatus(
+  status: string
+): Promise<UnifiedOrder[]> {
+  return Array.from(ordersStore.values()).filter(
+    (order) => order.payment_status === status
+  );
+}
+
+// Activity: Get payments for order
+export async function getPaymentsForOrder(orderId: string): Promise<UnifiedPayment[]> {
+  return Array.from(paymentsStore.values()).filter(
+    (payment) => payment.order_id === orderId
+  );
+}
+
+// Activity: Get shipments for order
+export async function getShipmentsForOrder(
+  orderId: string
+): Promise<UnifiedShipment[]> {
+  return Array.from(shipmentsStore.values()).filter(
+    (shipment) => shipment.order_id === orderId
+  );
+}
+
+// Activity: Clear all data (for testing)
+export async function clearAllData(): Promise<void> {
+  ordersStore.clear();
+  paymentsStore.clear();
+  shipmentsStore.clear();
+  discrepanciesStore.clear();
+  checkpointsStore.clear();
+  console.log('Cleared all data');
+}
+
+// Activity: Get statistics
+export async function getStats(): Promise<{
+  orders: number;
+  payments: number;
+  shipments: number;
+  discrepancies: number;
+}> {
+  return {
+    orders: ordersStore.size,
+    payments: paymentsStore.size,
+    shipments: shipmentsStore.size,
+    discrepancies: discrepanciesStore.size,
+  };
+}
+
+// In a real implementation, these would be SQL queries like:
+/*
+async function upsertOrdersSQL(orders: UnifiedOrder[]): Promise<number> {
+  const pool = getPool();
+
+  const query = `
+    INSERT INTO reconciled_orders (
+      order_id, external_id, source, customer_email, total_amount,
+      currency, status, created_at, line_items_count, payment_status,
+      fulfillment_status, synced_at
+    ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
+    ON CONFLICT (order_id) DO UPDATE SET
+      customer_email = EXCLUDED.customer_email,
+      total_amount = EXCLUDED.total_amount,
+      status = EXCLUDED.status,
+      payment_status = EXCLUDED.payment_status,
+      fulfillment_status = EXCLUDED.fulfillment_status,
+      synced_at = EXCLUDED.synced_at
+  `;
+
+  let count = 0;
+  for (const order of orders) {
+    await pool.query(query, [
+      order.order_id,
+      order.external_id,
+      order.source,
+      order.customer_email,
+      order.total_amount,
+      order.currency,
+      order.status,
+      order.created_at,
+      order.line_items_count,
+      order.payment_status,
+      order.fulfillment_status,
+      order.synced_at,
+    ]);
+    count++;
+  }
+
+  return count;
+}
+*/