reqon-dsl 0.2.0 → 0.4.0

package/docusaurus/docs/examples.md

@@ -1,488 +0,0 @@
----
-sidebar_position: 20
----
-
-# Examples
-
-Real-world examples demonstrating Reqon's capabilities.
-
-## Basic API Sync
-
-Fetch and store data from a public API:
-
-```vague
-mission JSONPlaceholderSync {
-  source API {
-    auth: none,
-    base: "https://jsonplaceholder.typicode.com"
-  }
-
-  store posts: file("posts")
-  store users: file("users")
-
-  action FetchPosts {
-    get "/posts"
-
-    for post in response {
-      store post -> posts { key: .id }
-    }
-  }
-
-  action FetchUsers {
-    get "/users"
-
-    for user in response {
-      map user -> User {
-        id: .id,
-        name: .name,
-        email: lowercase(.email),
-        company: .company.name
-      }
-      store user -> users { key: .id }
-    }
-  }
-
-  run [FetchPosts, FetchUsers]
-}
-```
-
-## Xero Integration
-
-Sync invoices from Xero with OAuth2 authentication:
-
-```vague
-mission XeroInvoiceSync {
-  source Xero {
-    auth: oauth2,
-    base: "https://api.xero.com/api.xro/2.0",
-    headers: {
-      "Xero-Tenant-Id": env("XERO_TENANT_ID")
-    }
-  }
-
-  store invoices: file("xero-invoices")
-  store errors: file("sync-errors")
-
-  schema XeroInvoice {
-    InvoiceID: string,
-    InvoiceNumber: string,
-    Contact: { ContactID: string, Name: string },
-    Total: number,
-    Status: string
-  }
-
-  schema ErrorResponse {
-    error: string,
-    code: number
-  }
-
-  action FetchInvoices {
-    get "/Invoices" {
-      params: { page: 1 },
-      paginate: page(page, 100),
-      until: length(response.Invoices) == 0,
-      since: lastSync
-    }
-
-    match response {
-      ErrorResponse where .code == 401 -> jump RefreshToken then retry,
-      ErrorResponse -> queue errors { item: { error: response.error } },
-      _ -> continue
-    }
-
-    for invoice in response.Invoices {
-      map invoice -> StandardInvoice {
-        id: .InvoiceID,
-        number: .InvoiceNumber,
-        customerId: .Contact.ContactID,
-        customerName: .Contact.Name,
-        amount: .Total,
-        status: lowercase(.Status),
-        source: "xero"
-      }
-      store invoice -> invoices { key: .id, upsert: true }
-    }
-  }
-
-  action RefreshToken {
-    post "https://identity.xero.com/connect/token" {
-      body: {
-        grant_type: "refresh_token",
-        refresh_token: env("XERO_REFRESH_TOKEN"),
-        client_id: env("XERO_CLIENT_ID"),
-        client_secret: env("XERO_CLIENT_SECRET")
-      }
-    }
-  }
-
-  run FetchInvoices
-}
-```
-
-## Multi-Source Reconciliation
-
-Sync and reconcile data from multiple sources:
-
-```vague
-mission OrderReconciliation {
-  source Shopify { auth: bearer, base: "https://mystore.myshopify.com/admin/api/2024-01" }
-  source Stripe { auth: bearer, base: "https://api.stripe.com/v1" }
-  source Warehouse { auth: api_key, base: "https://warehouse.example.com/api" }
-
-  store shopifyOrders: memory("shopify")
-  store stripePayments: memory("stripe")
-  store warehouseShipments: memory("warehouse")
-  store reconciledOrders: file("reconciled")
-  store discrepancies: file("discrepancies")
-
-  action FetchShopify {
-    get Shopify "/orders.json" {
-      params: { status: "any", limit: 250 },
-      paginate: cursor(page_info, 250, "link.next"),
-      until: response.orders == null or length(response.orders) == 0
-    }
-
-    for order in response.orders {
-      store order -> shopifyOrders { key: .id }
-    }
-  }
-
-  action FetchStripe {
-    get Stripe "/payment_intents" {
-      params: { limit: 100 },
-      paginate: cursor(starting_after, 100, "data[length(data)-1].id"),
-      until: response.has_more == false
-    }
-
-    for payment in response.data {
-      store payment -> stripePayments { key: .id }
-    }
-  }
-
-  action FetchWarehouse {
-    get Warehouse "/shipments" {
-      paginate: offset(offset, 100),
-      until: length(response.shipments) == 0
-    }
-
-    for shipment in response.shipments {
-      store shipment -> warehouseShipments { key: .order_id }
-    }
-  }
-
-  action Reconcile {
-    for order in shopifyOrders {
-      // Find matching payment
-      for payment in stripePayments where .metadata.order_id == order.id {
-        // Find matching shipment
-        for shipment in warehouseShipments where .order_id == order.id {
-          map order -> ReconciledOrder {
-            orderId: order.id,
-            orderAmount: order.total_price,
-            paymentId: payment.id,
-            paymentAmount: payment.amount / 100,
-            shipmentId: shipment.id,
-            shipmentStatus: shipment.status,
-            amountMatch: order.total_price == payment.amount / 100,
-            isShipped: shipment.status == "delivered"
-          }
-
-          match order {
-            _ where order.amountMatch == false -> {
-              store order -> discrepancies { key: .orderId }
-            },
-            _ -> store order -> reconciledOrders { key: .orderId }
-          }
-        }
-      }
-    }
-  }
-
-  run [FetchShopify, FetchStripe, FetchWarehouse] then Reconcile
-}
-```
-
-## OpenAPI Integration
-
-Use OpenAPI spec for type-safe API calls:
-
-```vague
-mission PetstoreSync {
-  source Petstore from "https://petstore3.swagger.io/api/v3/openapi.json" {
-    auth: api_key,
-    validateResponses: true
-  }
-
-  store pets: file("pets")
-  store newPets: file("new-pets")
-
-  action FetchAllPets {
-    call Petstore.findPetsByStatus {
-      params: { status: "available" }
-    }
-
-    for pet in response {
-      validate pet {
-        assume .id is number,
-        assume .name is string
-      }
-      store pet -> pets { key: .id }
-    }
-  }
-
-  action CreatePet {
-    for pet in newPets {
-      call Petstore.addPet {
-        body: {
-          name: pet.name,
-          photoUrls: pet.photos or [],
-          status: "available"
-        }
-      }
-
-      match response {
-        { id: _ } -> {
-          store response -> pets { key: .id }
-          delete newPets[pet.id]
-        },
-        _ -> continue
-      }
-    }
-  }
-
-  run FetchAllPets
-}
-```
-
-## Error Handling with Dead Letter Queue
-
-Robust error handling with retry and DLQ:
-
-```vague
-mission RobustSync {
-  source API {
-    auth: bearer,
-    base: "https://api.example.com",
-    rateLimit: { requestsPerMinute: 60, strategy: "pause" },
-    circuitBreaker: { failureThreshold: 5, resetTimeout: 30000 }
-  }
-
-  store data: file("data")
-  store dlq: file("dead-letter-queue")
-  store processed: file("processed")
-
-  schema SuccessResponse { data: any }
-  schema RateLimitError { error: string, retryAfter: number }
-  schema AuthError { error: string, code: number }
-  schema ValidationError { error: string, details: array }
-
-  action FetchWithRetry {
-    get "/items" {
-      paginate: offset(offset, 100),
-      until: length(response.data) == 0,
-      retry: { maxAttempts: 3, backoff: exponential }
-    }
-
-    match response {
-      RateLimitError -> retry { delay: response.retryAfter * 1000 },
-      AuthError where .code == 401 -> jump RefreshAuth then retry,
-      AuthError -> abort response.error,
-      ValidationError -> {
-        store response -> dlq { key: uuid() }
-        skip
-      },
-      SuccessResponse -> continue,
-      _ -> abort "Unexpected response"
-    }
-
-    for item in response.data {
-      store item -> data { key: .id }
-    }
-  }
-
-  action ProcessItems {
-    for item in data where .processed != true {
-      get concat("/items/", item.id, "/process")
-
-      match response {
-        { error: e } -> {
-          queue dlq {
-            item: {
-              itemId: item.id,
-              error: e,
-              timestamp: now(),
-              retryCount: (item.retryCount or 0) + 1
-            }
-          }
-          skip
-        },
-        _ -> {
-          store { ...item, processed: true } -> processed { key: .id }
-        }
-      }
-    }
-  }
-
-  action RefreshAuth {
-    post "/auth/refresh" {
-      body: { refreshToken: env("REFRESH_TOKEN") }
-    }
-  }
-
-  run FetchWithRetry then ProcessItems
-}
-```
-
-## Scheduled Incremental Sync
-
-Regular incremental sync with scheduling:
-
-```vague
-mission ScheduledSync {
-  schedule: every 15 minutes
-  skipIfRunning: true
-
-  source API {
-    auth: oauth2,
-    base: "https://api.example.com"
-  }
-
-  store customers: sql("customers")
-  store orders: sql("orders")
-  store syncLog: file("sync-log")
-
-  action SyncCustomers {
-    store { action: "SyncCustomers", started: now() } -> syncLog
-
-    get "/customers" {
-      since: lastSync,
-      paginate: cursor(after, 100, "cursor.next"),
-      until: response.cursor.next == null
-    }
-
-    for customer in response.data {
-      store customer -> customers { key: .id, upsert: true }
-    }
-
-    store {
-      action: "SyncCustomers",
-      completed: now(),
-      count: length(response.data)
-    } -> syncLog
-  }
-
-  action SyncOrders {
-    store { action: "SyncOrders", started: now() } -> syncLog
-
-    get "/orders" {
-      since: lastSync,
-      paginate: cursor(after, 100, "cursor.next"),
-      until: response.cursor.next == null
-    }
-
-    for order in response.data {
-      store order -> orders { key: .id, upsert: true }
-    }
-
-    store {
-      action: "SyncOrders",
-      completed: now(),
-      count: length(response.data)
-    } -> syncLog
-  }
-
-  run [SyncCustomers, SyncOrders]
-}
-```
-
-## GitHub Repository Sync
-
-Sync repository data from GitHub:
-
-```vague
-mission GitHubSync {
-  source GitHub {
-    auth: bearer,
-    base: "https://api.github.com",
-    headers: {
-      "Accept": "application/vnd.github.v3+json"
-    }
-  }
-
-  store repos: file("repos")
-  store issues: file("issues")
-  store pullRequests: file("pull-requests")
-
-  action FetchRepos {
-    get "/user/repos" {
-      params: { per_page: 100, sort: "updated" },
-      paginate: page(page, 100),
-      until: length(response) < 100
-    }
-
-    for repo in response {
-      map repo -> Repository {
-        id: .id,
-        name: .name,
-        fullName: .full_name,
-        description: .description,
-        stars: .stargazers_count,
-        forks: .forks_count,
-        language: .language,
-        updatedAt: .updated_at
-      }
-      store repo -> repos { key: .id }
-    }
-  }
-
-  action FetchIssues {
-    for repo in repos where .open_issues_count > 0 {
-      get concat("/repos/", repo.fullName, "/issues") {
-        params: { state: "open", per_page: 100 },
-        paginate: page(page, 100),
-        until: length(response) < 100
-      }
-
-      for issue in response where .pull_request == null {
-        map issue -> Issue {
-          id: .id,
-          repoId: repo.id,
-          number: .number,
-          title: .title,
-          state: .state,
-          author: .user.login,
-          createdAt: .created_at
-        }
-        store issue -> issues { key: .id }
-      }
-    }
-  }
-
-  action FetchPRs {
-    for repo in repos {
-      get concat("/repos/", repo.fullName, "/pulls") {
-        params: { state: "open", per_page: 100 }
-      }
-
-      for pr in response {
-        map pr -> PullRequest {
-          id: .id,
-          repoId: repo.id,
-          number: .number,
-          title: .title,
-          author: .user.login,
-          branch: .head.ref,
-          createdAt: .created_at
-        }
-        store pr -> pullRequests { key: .id }
-      }
-    }
-  }
-
-  run FetchRepos then [FetchIssues, FetchPRs]
-}
-```
-
-## More Examples
-
-For more examples, see the [examples directory](https://github.com/mcclowes/reqon/tree/main/examples) in the Reqon repository.