quickbase-js 0.1.0-beta.12 → 0.1.0-beta.14

package/dist/esm/quickbase.js

@@ -7478,6 +7478,8 @@ class TempTokenStrategy {
             const newToken = await this.getToken(dbid);
             if (newToken) {
                 this.tokenCache.set(dbid, newToken);
+                if (debug)
+                    console.log(`[${methodName || "method"}] Retrying with token: ${newToken.substring(0, 10)}...`);
                 return newToken;
             }
             return null;
@@ -7555,6 +7557,8 @@ class SsoTokenStrategy {
         const newToken = await this.refreshSsoToken(debug || this.debug);
         if (newToken) {
             this.currentToken = newToken;
+            if (debug || this.debug)
+                console.log(`[${methodName || "method"}] Retrying with token: ${newToken.substring(0, 10)}...`);
             return newToken;
         }
         return null;
@@ -7640,20 +7644,318 @@ class RateLimitError extends Error {
     }
 }
 
-// src/invokeMethod.ts
-async function invokeMethod(methodName, params, methodMap, baseHeaders, authStrategy, rateLimiter, transformDates, debug, convertDates, attempt = 0, maxAttempts = rateLimiter.maxRetries + 1) {
+/**
+ * Paginates records from a Quick Base API endpoint, handling both skip-based and token-based pagination.
+ * @template K - The method name keyof QuickbaseClient.
+ * @param methodName - The Quick Base API method to invoke (e.g., "runQuery", "getUsers").
+ * @param params - Parameters for the API method, including optional skip and top values.
+ * @param methodMap - Mapping of method names to their implementation details.
+ * @param baseHeaders - Base HTTP headers for API requests.
+ * @param authStrategy - Strategy for handling authentication tokens.
+ * @param rateLimiter - Rate limiting mechanism to respect API limits.
+ * @param transformDates - Function to transform date strings in the response.
+ * @param debug - If true, logs detailed pagination information to the console.
+ * @param convertDates - If true, converts date strings to Date objects in the response.
+ * @param initialResponse - Optional initial response to start pagination from (used by invokeMethod).
+ * @returns A promise resolving to the complete paginated response with all records.
+ * @throws {Error} If no response is received during pagination.
+ */
+async function paginateRecords(methodName, params, methodMap, baseHeaders, authStrategy, rateLimiter, transformDates, debug, convertDates, initialResponse) {
+    if (debug)
+        console.log("[paginateRecords] Starting with params:", params);
+    /** Extracts the key containing the data array from the response (e.g., "data" or "users"). */
+    const getDataKey = (response) => Object.keys(response).find((key) => Array.isArray(response[key])) || "data";
+    const dataKey = initialResponse ? getDataKey(initialResponse) : "data";
+    let allRecords = initialResponse ? initialResponse[dataKey] : [];
+    let lastResponse = initialResponse;
+    let requestCount = 0;
+    /**
+     * Determines the pagination strategy based on response metadata.
+     * @param response - The API response to analyze.
+     * @returns "skip" for skip-based, "token" for token-based, or "none" if no pagination is detected.
+     */
+    const determinePaginationType = (response) => {
+        if (response.metadata.totalRecords !== undefined &&
+            response.metadata.skip !== undefined) {
+            if (debug)
+                console.log("[paginateRecords] Detected skip-based pagination");
+            return "skip";
+        }
+        if (response.metadata.nextPageToken !== undefined ||
+            response.metadata.nextToken !== undefined) {
+            if (debug)
+                console.log("[paginateRecords] Detected token-based pagination");
+            return "token";
+        }
+        if (debug)
+            console.log("[paginateRecords] No clear pagination type detected, assuming single-page response");
+        return "none"; // No pagination indicators
+    };
+    let paginationType = initialResponse
+        ? determinePaginationType(initialResponse)
+        : "none"; // Start with no pagination assumption
+    let continuation = initialResponse
+        ? paginationType === "skip"
+            ? {
+                type: "skip",
+                value: (initialResponse.metadata.skip ?? 0) +
+                    (initialResponse.metadata.numRecords || allRecords.length),
+            }
+            : paginationType === "token"
+                ? {
+                    type: "token",
+                    value: initialResponse.metadata.nextPageToken ||
+                        initialResponse.metadata.nextToken ||
+                        "",
+                    key: initialResponse.metadata.nextPageToken !== undefined
+                        ? "nextPageToken"
+                        : "nextToken",
+                }
+                : null
+        : null; // No continuation until type is determined
+    if (debug) {
+        console.log("[paginateRecords] Initial state:", {
+            dataKey,
+            paginationType,
+            continuation,
+            hasInitialResponse: !!initialResponse,
+            totalRecordsSoFar: allRecords.length,
+        });
+        if (initialResponse && initialResponse[dataKey].length > 0) {
+            console.log("[paginateRecords] Initial response IDs: first:", initialResponse[dataKey][0]["3"]?.value, "last:", initialResponse[dataKey][initialResponse[dataKey].length - 1]["3"]
+                ?.value);
+        }
+    }
+    /** Fetches the next page of records using the provided parameters. */
+    const fetchNextPage = async (paginatedParams) => {
+        if (debug)
+            console.log("[paginateRecords] Fetching with params:", paginatedParams);
+        const response = await invokeMethod(methodName, paginatedParams, methodMap, baseHeaders, authStrategy, rateLimiter, transformDates, debug, convertDates, false, 0, rateLimiter.maxRetries + 1, true);
+        requestCount++;
+        return response;
+    };
+    /** Constructs the parameters for the next API request based on the pagination type. */
+    const getPaginatedParams = () => {
+        const hasBody = "body" in params && params.body !== undefined;
+        if (paginationType === "skip" && continuation?.type === "skip") {
+            const paginatedOptions = {
+                ...(hasBody && "options" in params.body ? params.body.options : {}),
+                skip: continuation.value,
+            };
+            return hasBody
+                ? {
+                    ...params,
+                    body: {
+                        ...params.body,
+                        options: paginatedOptions,
+                    },
+                }
+                : { ...params, skip: continuation.value };
+        }
+        else if (paginationType === "token" && continuation?.type === "token") {
+            if (continuation.value === "" && !lastResponse) {
+                return { ...params }; // Initial call, no pagination fields
+            }
+            const tokenKey = continuation.key;
+            return hasBody
+                ? {
+                    ...params,
+                    body: {
+                        ...params.body,
+                        [tokenKey]: continuation.value,
+                    },
+                }
+                : { ...params, [tokenKey]: continuation.value };
+        }
+        return params; // No pagination or initial call
+    };
+    if (!initialResponse) {
+        const firstParams = getPaginatedParams();
+        const firstResponse = await fetchNextPage(firstParams);
+        const actualDataKey = getDataKey(firstResponse);
+        allRecords = firstResponse[actualDataKey];
+        lastResponse = firstResponse;
+        paginationType = determinePaginationType(firstResponse);
+        continuation =
+            paginationType === "skip"
+                ? {
+                    type: "skip",
+                    value: (firstResponse.metadata.skip ?? 0) +
+                        (firstResponse.metadata.numRecords || allRecords.length),
+                }
+                : paginationType === "token"
+                    ? {
+                        type: "token",
+                        value: firstResponse.metadata.nextPageToken ??
+                            firstResponse.metadata.nextToken ??
+                            "",
+                        key: firstResponse.metadata.nextPageToken !== undefined
+                            ? "nextPageToken"
+                            : "nextToken",
+                    }
+                    : null;
+    }
+    while (true) {
+        const hasMore = (paginationType === "skip" &&
+            continuation?.type === "skip" &&
+            lastResponse.metadata.totalRecords !== undefined &&
+            continuation.value < lastResponse.metadata.totalRecords &&
+            allRecords.length < lastResponse.metadata.totalRecords) ||
+            (paginationType === "token" &&
+                continuation?.type === "token" &&
+                continuation.value !== "");
+        if (!hasMore) {
+            if (debug) {
+                console.log("[paginateRecords] Stopping:", paginationType === "skip"
+                    ? "All skip-based records fetched or no more data"
+                    : paginationType === "token"
+                        ? "Token exhausted"
+                        : "No pagination required");
+            }
+            break;
+        }
+        const paginatedParams = getPaginatedParams();
+        const response = await fetchNextPage(paginatedParams);
+        const actualDataKey = getDataKey(response);
+        allRecords = allRecords.concat(response[actualDataKey]);
+        lastResponse = response;
+        if (debug && response[actualDataKey].length > 0) {
+            console.log("[paginateRecords] Fetched records this iteration:", response[actualDataKey].length, "IDs: first:", response[actualDataKey][0]["3"]?.value, "last:", response[actualDataKey][response[actualDataKey].length - 1]["3"]?.value, "Continuation after fetch:", continuation);
+        }
+        if (paginationType === "skip") {
+            const newSkip = continuation?.type === "skip"
+                ? continuation.value + response[actualDataKey].length
+                : (response.metadata.skip ?? 0) + response[actualDataKey].length;
+            continuation = { type: "skip", value: newSkip };
+            if (response[actualDataKey].length === 0) {
+                if (debug &&
+                    lastResponse.metadata.totalRecords &&
+                    newSkip < lastResponse.metadata.totalRecords) {
+                    console.warn("[paginateRecords] Warning: Empty response received but records remain", { skip: newSkip, totalRecords: lastResponse.metadata.totalRecords });
+                }
+                break;
+            }
+        }
+        else if (paginationType === "token") {
+            const newToken = response.metadata.nextPageToken ?? response.metadata.nextToken ?? "";
+            continuation = {
+                type: "token",
+                value: newToken,
+                key: response.metadata.nextPageToken !== undefined
+                    ? "nextPageToken"
+                    : "nextToken",
+            };
+            if (continuation.value === "") {
+                break;
+            }
+            else if (response[actualDataKey].length === 0 && newToken !== "") {
+                if (debug)
+                    console.warn("[paginateRecords] Warning: Empty response with non-empty token, possible API inconsistency", { token: newToken });
+            }
+        }
+    }
+    if (!lastResponse) {
+        throw new Error("[paginateRecords] No response received during pagination");
+    }
+    const finalDataKey = getDataKey(lastResponse);
+    const finalResponse = {
+        [finalDataKey]: allRecords,
+        fields: lastResponse.fields,
+        metadata: {
+            totalRecords: paginationType === "skip"
+                ? lastResponse.metadata.totalRecords
+                : undefined,
+            numRecords: allRecords.length,
+            numFields: lastResponse.metadata.numFields,
+            skip: paginationType === "skip" ? 0 : undefined,
+            top: paginationType === "skip" ? undefined : undefined,
+            ...(paginationType === "token" &&
+                continuation?.type === "token" &&
+                continuation.key
+                ? { [continuation.key]: "" }
+                : {}),
+        },
+    };
+    if (debug) {
+        console.log("[paginateRecords] Final response summary:", {
+            totalRecords: finalResponse.metadata.totalRecords,
+            numRecords: finalResponse.metadata.numRecords,
+            numFields: finalResponse.metadata.numFields,
+            skip: finalResponse.metadata.skip,
+            top: finalResponse.metadata.top,
+            ...(paginationType === "token" &&
+                continuation?.type === "token" &&
+                continuation.key
+                ? { [continuation.key]: finalResponse.metadata[continuation.key] }
+                : {}),
+        });
+        console.log("[paginateRecords] Pagination stats:", {
+            totalRequests: requestCount,
+        });
+    }
+    return finalResponse;
+}
+/**
+ * Checks if a response is paginatable based on its structure.
+ * @param response - The response to check.
+ * @returns True if the response has a data array and pagination metadata, false otherwise.
+ */
+function isPaginatable(response) {
+    return (response &&
+        typeof response === "object" &&
+        Object.keys(response).some((key) => Array.isArray(response[key])) &&
+        "metadata" in response &&
+        typeof response.metadata === "object" &&
+        (("totalRecords" in response.metadata &&
+            "numRecords" in response.metadata &&
+            "skip" in response.metadata) ||
+            "nextPageToken" in response.metadata ||
+            "nextToken" in response.metadata));
+}
+
+/**
+ * Invokes a Quick Base API method with the specified parameters, handling authentication, rate limiting, and pagination.
+ * @template K - The method name keyof QuickbaseClient.
+ * @param methodName - The API method to invoke (e.g., "runQuery", "changesetSolution").
+ * @param params - Parameters for the API method, potentially including skip, top, or a body.
+ * @param methodMap - Mapping of method names to their implementation details.
+ * @param baseHeaders - Base HTTP headers for API requests.
+ * @param authStrategy - Strategy for handling authentication tokens.
+ * @param rateLimiter - Rate limiting mechanism to respect API limits.
+ * @param transformDates - Function to transform date strings in the response.
+ * @param debug - If true, logs detailed invocation information.
+ * @param convertDates - If true, converts date strings to Date objects in the response.
+ * @param autoPaginate - If true, automatically paginates multi-page responses (default: true).
+ * @param attempt - Current retry attempt (default: 0).
+ * @param maxAttempts - Maximum number of retry attempts (default: rateLimiter.maxRetries + 1).
+ * @param isPaginating - Indicates if this call is part of an ongoing pagination (default: false).
+ * @returns A promise resolving to the API method's response.
+ * @throws {Error} If the method is not found or retries are exhausted.
+ */
+async function invokeMethod(methodName, params, methodMap, baseHeaders, authStrategy, rateLimiter, transformDates, debug, convertDates, autoPaginate = true, attempt = 0, maxAttempts = rateLimiter.maxRetries + 1, isPaginating = false) {
     const methodInfo = methodMap[methodName];
     if (!methodInfo)
         throw new Error(`Method ${methodName} not found`);
     const hasBody = "body" in params && params.body !== undefined;
     const body = hasBody ? params.body : undefined;
+    // Safely check for options only if body is an object
+    const options = hasBody && body && typeof body === "object" && "options" in body
+        ? body.options
+        : undefined;
+    const adjustedParams = {
+        ...params,
+        skip: params.skip ?? (options && "skip" in options ? options.skip : undefined),
+        top: params.top ?? (options && "top" in options ? options.top : undefined),
+    };
     const restParams = hasBody
-        ? Object.fromEntries(Object.entries(params).filter(([key]) => key !== "body" && key !== "startTime"))
-        : { ...params, startTime: undefined };
+        ? Object.fromEntries(Object.entries(adjustedParams).filter(([key]) => key !== "body" && key !== "startTime"))
+        : { ...adjustedParams, startTime: undefined };
     const requestParameters = {
         ...restParams,
         ...(hasBody ? { generated: body } : {}),
     };
+    if (debug)
+        console.log("[invokeMethod] Adjusted params for pagination:", adjustedParams);
     const requestOptions = {
         credentials: methodName === "getTempTokenDBID" ? "include" : "omit",
         method: methodInfo.httpMethod,
@@ -7676,12 +7978,22 @@ async function invokeMethod(methodName, params, methodMap, baseHeaders, authStra
                 ?.toLowerCase();
             if (contentType?.includes("application/json")) {
                 const jsonResponse = await rawResponse.json();
+                if (autoPaginate && !isPaginating && isPaginatable(jsonResponse)) {
+                    if (debug)
+                        console.log("[invokeMethod] Entering pagination with params:", adjustedParams);
+                    return paginateRecords(methodName, adjustedParams, methodMap, baseHeaders, authStrategy, rateLimiter, transformDates, debug, convertDates, jsonResponse);
+                }
                 return transformDates(jsonResponse, convertDates);
             }
             return rawResponse;
         }
         if (rawResponse && typeof rawResponse.value === "function") {
             const response = await rawResponse.value();
+            if (autoPaginate && !isPaginating && isPaginatable(response)) {
+                if (debug)
+                    console.log("[invokeMethod] Entering pagination with params:", adjustedParams);
+                return paginateRecords(methodName, adjustedParams, methodMap, baseHeaders, authStrategy, rateLimiter, transformDates, debug, convertDates, response);
+            }
             return transformDates(response, convertDates);
         }
         return transformDates(rawResponse, convertDates);
@@ -7718,11 +8030,18 @@ async function invokeMethod(methodName, params, methodMap, baseHeaders, authStra
         try {
             await rateLimiter.throttle();
             acquired = true;
-            const postThrottleTime = Date.now(); // Capture post-throttle time
+            const postThrottleTime = Date.now();
             if (params.startTime !== undefined)
-                params.startTime = postThrottleTime; // Update startTime for testing
-            const responsePromise = methodInfo.method(requestParameters, requestOptions);
-            rateLimiter.release(); // Release slot immediately after starting fetch
+                params.startTime = postThrottleTime;
+            const headers = { ...baseHeaders };
+            const finalRequest = { ...requestParameters, headers };
+            if (debug)
+                console.log("[invokeMethod] Sending request:", finalRequest);
+            const responsePromise = methodInfo.method(finalRequest, {
+                ...requestOptions,
+                headers,
+            });
+            rateLimiter.release();
             const response = await responsePromise;
             return await processResponse(response);
         }
@@ -7768,8 +8087,6 @@ async function invokeMethod(methodName, params, methodMap, baseHeaders, authStra
                 token = newToken;
                 authStrategy.applyHeaders(baseHeaders, token);
                 requestOptions.headers = { ...baseHeaders };
-                if (debug)
-                    console.log(`[${methodName}] Retrying with token: ${token.substring(0, 10)}...`);
                 attempt++;
                 continue;
             }
@@ -7779,8 +8096,37 @@ async function invokeMethod(methodName, params, methodMap, baseHeaders, authStra
     throw new Error(`API Error: Exhausted retries after ${maxAttempts} attempts`);
 }
 
-// src/ThrottleBucket.ts
-class ConcurrentThrottleBucket {
+// src/Semaphore.ts
+class Semaphore {
+    permits;
+    waiting = [];
+    constructor(maxPermits) {
+        this.permits = maxPermits;
+    }
+    async acquire() {
+        if (this.permits > 0) {
+            this.permits -= 1;
+            return;
+        }
+        return new Promise((resolve, reject) => {
+            this.waiting.push({ resolve, reject });
+        });
+    }
+    release() {
+        this.permits += 1;
+        if (this.waiting.length > 0 && this.permits > 0) {
+            const { resolve } = this.waiting.shift();
+            this.permits -= 1;
+            resolve();
+        }
+    }
+    available() {
+        return this.permits;
+    }
+}
+
+// src/FlowThrottleBucket.ts
+class FlowThrottleBucket {
     tokens;
     maxTokens;
     refillRate;
@@ -7820,31 +8166,46 @@ class ConcurrentThrottleBucket {
         this.semaphore.release();
     }
 }
-class Semaphore {
-    permits;
-    waiting = [];
-    constructor(maxPermits) {
-        this.permits = maxPermits;
+
+// src/BurstAwareThrottleBucket.ts
+class BurstAwareThrottleBucket {
+    maxTokens;
+    windowSeconds;
+    requestTimestamps = [];
+    semaphore;
+    constructor(options) {
+        this.maxTokens = options.maxTokens;
+        this.windowSeconds = options.windowSeconds;
+        this.semaphore = new Semaphore(this.maxTokens);
+    }
+    countRequestsInWindow() {
+        const now = Date.now();
+        this.requestTimestamps = this.requestTimestamps.filter((ts) => now - ts < this.windowSeconds * 1000);
+        return this.requestTimestamps.length;
+    }
+    availableTokens() {
+        const windowCount = this.countRequestsInWindow();
+        return Math.max(0, Math.min(100 - windowCount, this.maxTokens - windowCount));
     }
     async acquire() {
-        if (this.permits > 0) {
-            this.permits -= 1;
-            return;
+        while (true) {
+            const available = this.availableTokens();
+            if (available > 0 && this.semaphore.available() > 0) {
+                await this.semaphore.acquire();
+                this.requestTimestamps.push(Date.now());
+                return;
+            }
+            const now = Date.now();
+            const oldestTimestamp = this.requestTimestamps[0];
+            let waitTime = 100;
+            if (oldestTimestamp) {
+                waitTime = Math.max(0, this.windowSeconds * 1000 - (now - oldestTimestamp));
+            }
+            await new Promise((resolve) => setTimeout(resolve, waitTime));
         }
-        return new Promise((resolve, reject) => {
-            this.waiting.push({ resolve, reject });
-        });
     }
     release() {
-        this.permits += 1;
-        if (this.waiting.length > 0 && this.permits > 0) {
-            const { resolve } = this.waiting.shift();
-            this.permits -= 1;
-            resolve();
-        }
-    }
-    available() {
-        return this.permits;
+        this.semaphore.release();
     }
 }
 
@@ -7852,8 +8213,7 @@ class RateLimiter {
     throttleBucket;
     maxRetries;
     retryDelay;
-    constructor(throttleBucket, // Updated type
-    maxRetries = 3, retryDelay = 1000) {
+    constructor(throttleBucket, maxRetries = 3, retryDelay = 1000) {
         this.throttleBucket = throttleBucket;
         this.maxRetries = maxRetries;
         this.retryDelay = retryDelay;
@@ -7877,20 +8237,23 @@ class RateLimiter {
 }
 
 function quickbase(config) {
-    const { realm, userToken, tempToken, useTempTokens, useSso, samlToken, fetchApi, debug, convertDates = true, tempTokenLifespan = 290000, throttle = { rate: 5, burst: 3 }, // Updated default to match passing test
-    maxRetries = 3, retryDelay = 1000, tokenCache: providedTokenCache, baseUrl = "https://api.quickbase.com/v1", } = config;
+    const { realm, userToken, tempToken, useTempTokens = false, useSso = false, samlToken, fetchApi, debug, convertDates = true, tempTokenLifespan = 290000, throttle = { type: "flow", rate: 5, burst: 50 }, // Changed rate from 6 to 5
+    maxRetries = 3, retryDelay = 1000, tokenCache: providedTokenCache, baseUrl = "https://api.quickbase.com/v1", autoPaginate = true, } = config;
     const tokenCache = providedTokenCache || new TokenCache(tempTokenLifespan);
-    const throttleBucket = throttle
-        ? new ConcurrentThrottleBucket(throttle.rate, throttle.burst)
-        : null;
+    const throttleOptions = throttle;
+    const throttleBucket = throttleOptions.type === "burst-aware"
+        ? new BurstAwareThrottleBucket({
+            maxTokens: throttleOptions.burst || 100,
+            windowSeconds: throttleOptions.windowSeconds || 10,
+        })
+        : new FlowThrottleBucket(throttleOptions.rate || 5, // Changed rate from 6 to 5
+        throttleOptions.burst || 50);
     const rateLimiter = new RateLimiter(throttleBucket, maxRetries, retryDelay);
-    const defaultFetch = typeof globalThis.window !== "undefined"
-        ? globalThis.window.fetch.bind(globalThis.window)
-        : undefined;
-    const effectiveFetch = fetchApi || defaultFetch;
-    if (!effectiveFetch) {
-        throw new Error("No fetch implementation available");
+    const defaultFetch = globalThis.fetch || globalThis.window?.fetch?.bind(globalThis.window);
+    if (!defaultFetch && !fetchApi) {
+        throw new Error("No fetch implementation available. Please provide fetchApi in a Node.js environment without native fetch.");
     }
+    const effectiveFetch = fetchApi || defaultFetch;
     const authStrategy = useSso
         ? new SsoTokenStrategy(samlToken || "", realm, effectiveFetch, debug, baseUrl)
         : useTempTokens
@@ -7945,19 +8308,23 @@ function quickbase(config) {
     const methodMap = buildMethodMap();
     const proxyHandler = {
         get: (_, prop) => {
-            console.log("[proxy] Accessing:", prop, "in methodMap:", prop in methodMap);
+            if (debug) {
+                console.log("[proxy] Accessing:", prop, "in methodMap:", prop in methodMap);
+            }
             if (prop in methodMap) {
                 const methodName = prop;
-                return (params) => invokeMethod(methodName, params, methodMap, baseHeaders, authStrategy, rateLimiter, transformDates, debug, convertDates);
+                return (params) => invokeMethod(methodName, params, methodMap, baseHeaders, authStrategy, rateLimiter, transformDates, debug, convertDates, autoPaginate);
+            }
+            if (debug) {
+                console.log("[proxy] Method not found:", prop);
             }
-            console.log("[proxy] Method not found:", prop);
             return undefined;
         },
     };
     const proxy = new Proxy({}, proxyHandler);
     if (debug) {
         console.log("[createClient] Config:", config);
-        console.log("[createClient] Proxy created:", proxy);
+        console.log("[createClient] Returning proxy");
     }
     return proxy;
 }