reqon-dsl 0.2.0

package/docusaurus/docs/dsl-syntax/fetch.md

@@ -0,0 +1,293 @@
+---
+sidebar_position: 1
+---
+
+# Fetch Steps
+
+Fetch steps make HTTP requests to APIs. They're the primary way to retrieve data in Reqon.
+
+## HTTP Methods
+
+```vague
+// GET request
+get "/users"
+
+// POST request
+post "/users" { body: { name: "John" } }
+
+// PUT request
+put "/users/123" { body: { name: "Jane" } }
+
+// PATCH request
+patch "/users/123" { body: { email: "jane@example.com" } }
+
+// DELETE request
+delete "/users/123"
+```
+
+## Request Options
+
+### Query Parameters
+
+```vague
+get "/users" {
+  params: {
+    limit: 100,
+    offset: 0,
+    status: "active"
+  }
+}
+```
+
+### Request Body
+
+```vague
+post "/users" {
+  body: {
+    name: "John Doe",
+    email: "john@example.com",
+    roles: ["user", "admin"]
+  }
+}
+```
+
+### Custom Headers
+
+```vague
+get "/users" {
+  headers: {
+    "X-Custom-Header": "value",
+    "Accept": "application/json"
+  }
+}
+```
+
+## Pagination
+
+### Offset-Based
+
+```vague
+get "/users" {
+  paginate: offset(page, 100),
+  until: length(response) == 0
+}
+```
+
+Parameters:
+- `page` - Query parameter name for offset value
+- `100` - Page size
+
+### Page Number-Based
+
+```vague
+get "/users" {
+  paginate: page(pageNum, 50),
+  until: response.meta.hasNext == false
+}
+```
+
+Parameters:
+- `pageNum` - Query parameter name for page number
+- `50` - Page size
+
+### Cursor-Based
+
+```vague
+get "/users" {
+  paginate: cursor(cursor, 100, "meta.nextCursor"),
+  until: response.meta.nextCursor == null
+}
+```
+
+Parameters:
+- `cursor` - Query parameter name
+- `100` - Page size
+- `"meta.nextCursor"` - Path to next cursor in response
+
+See [Pagination](../http/pagination) for detailed documentation.
+
+## Termination Conditions
+
+The `until` option specifies when to stop paginating:
+
+```vague
+// Stop when empty response
+get "/users" {
+  paginate: offset(page, 100),
+  until: length(response) == 0
+}
+
+// Stop when no more pages
+get "/users" {
+  paginate: page(p, 50),
+  until: response.pagination.hasNext == false
+}
+
+// Stop when cursor is null
+get "/users" {
+  paginate: cursor(c, 100, "nextCursor"),
+  until: response.nextCursor == null
+}
+
+// Stop after N items
+get "/users" {
+  paginate: offset(page, 100),
+  until: length(response) == 0 or page > 10
+}
+```
+
+## Retry Configuration
+
+```vague
+get "/users" {
+  retry: {
+    maxAttempts: 3,
+    backoff: exponential,
+    initialDelay: 1000,
+    maxDelay: 30000
+  }
+}
+```
+
+Options:
+- `maxAttempts` - Maximum retry attempts
+- `backoff` - Strategy: `exponential`, `linear`, or `constant`
+- `initialDelay` - First retry delay in milliseconds
+- `maxDelay` - Maximum delay between retries
+
+See [Retry Strategies](../http/retry) for details.
+
+## Incremental Sync
+
+Fetch only changes since last run:
+
+```vague
+get "/users" {
+  since: lastSync
+}
+```
+
+This automatically adds a timestamp parameter to the request.
+
+See [Incremental Sync](../http/incremental-sync) for details.
+
+## Response Handling
+
+The `response` variable is automatically set after each fetch:
+
+```vague
+action FetchUsers {
+  get "/users"
+
+  // response contains the parsed JSON body
+  for user in response.data {
+    store user -> users { key: .id }
+  }
+}
+```
+
+### Response Structure
+
+```vague
+action InspectResponse {
+  get "/users"
+
+  // Access body data
+  store response.users -> users { key: .id }
+
+  // Check response metadata
+  validate {
+    assume response.total > 0
+  }
+}
+```
+
+## Named Source Requests
+
+When you have multiple sources, specify which to use:
+
+```vague
+mission MultiSource {
+  source Primary { auth: bearer, base: "https://primary.api.com" }
+  source Secondary { auth: bearer, base: "https://secondary.api.com" }
+
+  action FetchBoth {
+    // Default source (first defined)
+    get "/users"
+
+    // Explicit source
+    get Secondary "/backup-users"
+  }
+}
+```
+
+## Dynamic Paths
+
+Use expressions in paths:
+
+```vague
+action FetchUserOrders {
+  for user in users {
+    get concat("/users/", user.id, "/orders")
+    store response -> orders { key: .id }
+  }
+}
+```
+
+## OpenAPI Operation Calls
+
+When using OAS sources, use `call` syntax:
+
+```vague
+source Petstore from "./petstore.yaml" { auth: bearer }
+
+action FetchPets {
+  call Petstore.listPets {
+    params: { limit: 100 }
+  }
+
+  call Petstore.getPetById {
+    params: { petId: "123" }
+  }
+}
+```
+
+See [OpenAPI Integration](../category/openapi-integration) for details.
+
+## Complete Example
+
+```vague
+mission DataSync {
+  source API {
+    auth: bearer,
+    base: "https://api.example.com/v1"
+  }
+
+  store users: file("users")
+
+  action FetchAllUsers {
+    get "/users" {
+      params: { include: "profile" },
+      headers: { "Accept-Version": "2.0" },
+      paginate: offset(offset, 100),
+      until: length(response.users) == 0,
+      retry: {
+        maxAttempts: 3,
+        backoff: exponential,
+        initialDelay: 1000
+      },
+      since: lastSync
+    }
+
+    for user in response.users {
+      validate user {
+        assume .id is string,
+        assume .email is string
+      }
+      store user -> users { key: .id, upsert: true }
+    }
+  }
+
+  run FetchAllUsers
+}
+```

package/docusaurus/docs/dsl-syntax/for-loops.md

@@ -0,0 +1,324 @@
+---
+sidebar_position: 2
+---
+
+# For Loops
+
+For loops iterate over collections, allowing you to process each item individually.
+
+## Basic Syntax
+
+```vague
+for item in collection {
+  // steps to execute for each item
+}
+```
+
+## Iterating Over Response Data
+
+```vague
+action ProcessUsers {
+  get "/users"
+
+  for user in response.data {
+    store user -> users { key: .id }
+  }
+}
+```
+
+## Iterating Over Store Data
+
+```vague
+action ProcessStoredData {
+  for customer in customers {
+    get concat("/customers/", customer.id, "/orders")
+    store response -> orders { key: .id }
+  }
+}
+```
+
+## Filtering with Where
+
+Add conditions to filter items:
+
+```vague
+action ProcessActiveUsers {
+  get "/users"
+
+  // Single condition
+  for user in response.data where .status == "active" {
+    store user -> activeUsers { key: .id }
+  }
+}
+```
+
+### Multiple Conditions
+
+```vague
+action ProcessPremiumActiveUsers {
+  for user in users where .status == "active" and .tier == "premium" {
+    // Process premium active users
+  }
+}
+```
+
+### Comparison Operators
+
+```vague
+// Equality
+for item in items where .status == "pending" { }
+
+// Inequality
+for item in items where .status != "cancelled" { }
+
+// Numeric comparisons
+for item in items where .price > 100 { }
+for item in items where .quantity >= 10 { }
+for item in items where .discount < 0.5 { }
+for item in items where .stock <= 0 { }
+
+// String contains
+for item in items where .email contains "@example.com" { }
+
+// Type checking
+for item in items where .tags is array { }
+```
+
+### Complex Conditions
+
+```vague
+for order in orders where (.status == "pending" or .status == "processing") and .total > 100 {
+  // Process high-value pending/processing orders
+}
+```
+
+## Nested Loops
+
+```vague
+action ProcessOrderItems {
+  for order in orders {
+    for item in order.lineItems {
+      map item -> OrderItem {
+        orderId: order.id,
+        productId: item.productId,
+        quantity: item.quantity,
+        price: item.unitPrice
+      }
+      store item -> orderItems { key: concat(order.id, "-", item.productId) }
+    }
+  }
+}
+```
+
+## Variable Scope
+
+Loop variables are scoped to their block:
+
+```vague
+action ScopedVariables {
+  get "/users"
+
+  for user in response.users {
+    // user is available here
+
+    for order in user.orders {
+      // Both user and order are available
+
+      map order -> EnrichedOrder {
+        orderId: order.id,
+        userId: user.id,
+        userName: user.name
+      }
+    }
+    // order is no longer available
+  }
+  // user is no longer available
+}
+```
+
+## Accessing Loop Item Properties
+
+Use dot notation to access properties:
+
+```vague
+for user in users {
+  // Direct access
+  store user -> allUsers { key: .id }
+
+  // Nested access
+  validate user {
+    assume .profile.email is string
+  }
+
+  // In expressions
+  map user -> Output {
+    fullName: concat(.firstName, " ", .lastName),
+    domain: split(.email, "@")[1]
+  }
+}
+```
+
+## Iterating Over Paginated Results
+
+Combine pagination with iteration:
+
+```vague
+action FetchAllOrders {
+  get "/orders" {
+    paginate: offset(page, 100),
+    until: length(response.orders) == 0
+  }
+
+  // This runs after ALL pages are fetched
+  for order in response.orders {
+    store order -> orders { key: .id }
+  }
+}
+```
+
+For processing each page separately:
+
+```vague
+action ProcessPagesSequentially {
+  get "/orders" {
+    paginate: offset(page, 100),
+    until: length(response.orders) == 0
+  }
+
+  // Pagination accumulates all results in response
+  // Then the for loop processes them
+  for order in response.orders {
+    match order {
+      { status: "urgent" } -> {
+        get concat("/orders/", order.id, "/expedite")
+      },
+      _ -> continue
+    }
+    store order -> orders { key: .id }
+  }
+}
+```
+
+## Breaking Out of Loops
+
+Use `skip` in match to skip to the next iteration:
+
+```vague
+for user in users {
+  match user {
+    { status: "inactive" } -> skip,
+    { status: "banned" } -> skip,
+    _ -> continue
+  }
+
+  // Only runs for active, non-banned users
+  store user -> activeUsers { key: .id }
+}
+```
+
+## Error Handling in Loops
+
+Handle errors per-item:
+
+```vague
+for user in users {
+  get concat("/users/", user.id, "/details")
+
+  match response {
+    { error: _ } -> {
+      // Log error and continue
+      store { userId: user.id, error: response.error } -> errors { key: user.id }
+      skip
+    },
+    _ -> continue
+  }
+
+  store response -> userDetails { key: user.id }
+}
+```
+
+## Performance Considerations
+
+### Batch Operations
+
+Instead of individual requests:
+
+```vague
+// Less efficient: one request per user
+for user in users {
+  get concat("/users/", user.id)
+}
+```
+
+Consider batching if the API supports it:
+
+```vague
+// More efficient: batch request
+post "/users/batch" {
+  body: { ids: users.map(.id) }
+}
+```
+
+### Parallel Processing
+
+For independent operations, consider parallel actions:
+
+```vague
+run [FetchOrders, FetchProducts, FetchCustomers] then MergeData
+```
+
+## Complete Example
+
+```vague
+mission OrderProcessing {
+  source API { auth: bearer, base: "https://api.example.com" }
+
+  store orders: file("orders")
+  store enrichedOrders: file("enriched-orders")
+  store errors: file("errors")
+
+  action ProcessOrders {
+    get "/orders" {
+      paginate: offset(page, 100),
+      until: length(response.data) == 0
+    }
+
+    for order in response.data where .status != "cancelled" {
+      // Validate order
+      validate order {
+        assume .id is string,
+        assume .total > 0,
+        assume .items is array
+      }
+
+      // Fetch customer details
+      get concat("/customers/", order.customerId)
+
+      match response {
+        { error: _ } -> {
+          store { orderId: order.id, error: "Customer not found" } -> errors
+          skip
+        },
+        _ -> continue
+      }
+
+      // Enrich order with customer data
+      map order -> EnrichedOrder {
+        id: order.id,
+        total: order.total,
+        status: order.status,
+        customer: {
+          id: response.id,
+          name: response.name,
+          email: response.email
+        },
+        items: order.items
+      }
+
+      store order -> enrichedOrders { key: .id }
+    }
+  }
+
+  run ProcessOrders
+}
+```