quickbase-js 0.1.0-beta.12 → 0.1.0-beta.13

package/dist/umd/quickbase.umd.js

@@ -7484,6 +7484,8 @@
                 const newToken = await this.getToken(dbid);
                 if (newToken) {
                     this.tokenCache.set(dbid, newToken);
+                    if (debug)
+                        console.log(`[${methodName || "method"}] Retrying with token: ${newToken.substring(0, 10)}...`);
                     return newToken;
                 }
                 return null;
@@ -7561,6 +7563,8 @@
             const newToken = await this.refreshSsoToken(debug || this.debug);
             if (newToken) {
                 this.currentToken = newToken;
+                if (debug || this.debug)
+                    console.log(`[${methodName || "method"}] Retrying with token: ${newToken.substring(0, 10)}...`);
                 return newToken;
             }
             return null;
@@ -7646,20 +7650,318 @@
         }
     }
 
-    // src/invokeMethod.ts
-    async function invokeMethod(methodName, params, methodMap, baseHeaders, authStrategy, rateLimiter, transformDates, debug, convertDates, attempt = 0, maxAttempts = rateLimiter.maxRetries + 1) {
+    /**
+     * Paginates records from a Quick Base API endpoint, handling both skip-based and token-based pagination.
+     * @template K - The method name keyof QuickbaseClient.
+     * @param methodName - The Quick Base API method to invoke (e.g., "runQuery", "getUsers").
+     * @param params - Parameters for the API method, including optional skip and top values.
+     * @param methodMap - Mapping of method names to their implementation details.
+     * @param baseHeaders - Base HTTP headers for API requests.
+     * @param authStrategy - Strategy for handling authentication tokens.
+     * @param rateLimiter - Rate limiting mechanism to respect API limits.
+     * @param transformDates - Function to transform date strings in the response.
+     * @param debug - If true, logs detailed pagination information to the console.
+     * @param convertDates - If true, converts date strings to Date objects in the response.
+     * @param initialResponse - Optional initial response to start pagination from (used by invokeMethod).
+     * @returns A promise resolving to the complete paginated response with all records.
+     * @throws {Error} If no response is received during pagination.
+     */
+    async function paginateRecords(methodName, params, methodMap, baseHeaders, authStrategy, rateLimiter, transformDates, debug, convertDates, initialResponse) {
+        if (debug)
+            console.log("[paginateRecords] Starting with params:", params);
+        /** Extracts the key containing the data array from the response (e.g., "data" or "users"). */
+        const getDataKey = (response) => Object.keys(response).find((key) => Array.isArray(response[key])) || "data";
+        const dataKey = initialResponse ? getDataKey(initialResponse) : "data";
+        let allRecords = initialResponse ? initialResponse[dataKey] : [];
+        let lastResponse = initialResponse;
+        let requestCount = 0;
+        /**
+         * Determines the pagination strategy based on response metadata.
+         * @param response - The API response to analyze.
+         * @returns "skip" for skip-based, "token" for token-based, or "none" if no pagination is detected.
+         */
+        const determinePaginationType = (response) => {
+            if (response.metadata.totalRecords !== undefined &&
+                response.metadata.skip !== undefined) {
+                if (debug)
+                    console.log("[paginateRecords] Detected skip-based pagination");
+                return "skip";
+            }
+            if (response.metadata.nextPageToken !== undefined ||
+                response.metadata.nextToken !== undefined) {
+                if (debug)
+                    console.log("[paginateRecords] Detected token-based pagination");
+                return "token";
+            }
+            if (debug)
+                console.log("[paginateRecords] No clear pagination type detected, assuming single-page response");
+            return "none"; // No pagination indicators
+        };
+        let paginationType = initialResponse
+            ? determinePaginationType(initialResponse)
+            : "none"; // Start with no pagination assumption
+        let continuation = initialResponse
+            ? paginationType === "skip"
+                ? {
+                    type: "skip",
+                    value: (initialResponse.metadata.skip ?? 0) +
+                        (initialResponse.metadata.numRecords || allRecords.length),
+                }
+                : paginationType === "token"
+                    ? {
+                        type: "token",
+                        value: initialResponse.metadata.nextPageToken ||
+                            initialResponse.metadata.nextToken ||
+                            "",
+                        key: initialResponse.metadata.nextPageToken !== undefined
+                            ? "nextPageToken"
+                            : "nextToken",
+                    }
+                    : null
+            : null; // No continuation until type is determined
+        if (debug) {
+            console.log("[paginateRecords] Initial state:", {
+                dataKey,
+                paginationType,
+                continuation,
+                hasInitialResponse: !!initialResponse,
+                totalRecordsSoFar: allRecords.length,
+            });
+            if (initialResponse && initialResponse[dataKey].length > 0) {
+                console.log("[paginateRecords] Initial response IDs: first:", initialResponse[dataKey][0]["3"]?.value, "last:", initialResponse[dataKey][initialResponse[dataKey].length - 1]["3"]
+                    ?.value);
+            }
+        }
+        /** Fetches the next page of records using the provided parameters. */
+        const fetchNextPage = async (paginatedParams) => {
+            if (debug)
+                console.log("[paginateRecords] Fetching with params:", paginatedParams);
+            const response = await invokeMethod(methodName, paginatedParams, methodMap, baseHeaders, authStrategy, rateLimiter, transformDates, debug, convertDates, false, 0, rateLimiter.maxRetries + 1, true);
+            requestCount++;
+            return response;
+        };
+        /** Constructs the parameters for the next API request based on the pagination type. */
+        const getPaginatedParams = () => {
+            const hasBody = "body" in params && params.body !== undefined;
+            if (paginationType === "skip" && continuation?.type === "skip") {
+                const paginatedOptions = {
+                    ...(hasBody && "options" in params.body ? params.body.options : {}),
+                    skip: continuation.value,
+                };
+                return hasBody
+                    ? {
+                        ...params,
+                        body: {
+                            ...params.body,
+                            options: paginatedOptions,
+                        },
+                    }
+                    : { ...params, skip: continuation.value };
+            }
+            else if (paginationType === "token" && continuation?.type === "token") {
+                if (continuation.value === "" && !lastResponse) {
+                    return { ...params }; // Initial call, no pagination fields
+                }
+                const tokenKey = continuation.key;
+                return hasBody
+                    ? {
+                        ...params,
+                        body: {
+                            ...params.body,
+                            [tokenKey]: continuation.value,
+                        },
+                    }
+                    : { ...params, [tokenKey]: continuation.value };
+            }
+            return params; // No pagination or initial call
+        };
+        if (!initialResponse) {
+            const firstParams = getPaginatedParams();
+            const firstResponse = await fetchNextPage(firstParams);
+            const actualDataKey = getDataKey(firstResponse);
+            allRecords = firstResponse[actualDataKey];
+            lastResponse = firstResponse;
+            paginationType = determinePaginationType(firstResponse);
+            continuation =
+                paginationType === "skip"
+                    ? {
+                        type: "skip",
+                        value: (firstResponse.metadata.skip ?? 0) +
+                            (firstResponse.metadata.numRecords || allRecords.length),
+                    }
+                    : paginationType === "token"
+                        ? {
+                            type: "token",
+                            value: firstResponse.metadata.nextPageToken ??
+                                firstResponse.metadata.nextToken ??
+                                "",
+                            key: firstResponse.metadata.nextPageToken !== undefined
+                                ? "nextPageToken"
+                                : "nextToken",
+                        }
+                        : null;
+        }
+        while (true) {
+            const hasMore = (paginationType === "skip" &&
+                continuation?.type === "skip" &&
+                lastResponse.metadata.totalRecords !== undefined &&
+                continuation.value < lastResponse.metadata.totalRecords &&
+                allRecords.length < lastResponse.metadata.totalRecords) ||
+                (paginationType === "token" &&
+                    continuation?.type === "token" &&
+                    continuation.value !== "");
+            if (!hasMore) {
+                if (debug) {
+                    console.log("[paginateRecords] Stopping:", paginationType === "skip"
+                        ? "All skip-based records fetched or no more data"
+                        : paginationType === "token"
+                            ? "Token exhausted"
+                            : "No pagination required");
+                }
+                break;
+            }
+            const paginatedParams = getPaginatedParams();
+            const response = await fetchNextPage(paginatedParams);
+            const actualDataKey = getDataKey(response);
+            allRecords = allRecords.concat(response[actualDataKey]);
+            lastResponse = response;
+            if (debug && response[actualDataKey].length > 0) {
+                console.log("[paginateRecords] Fetched records this iteration:", response[actualDataKey].length, "IDs: first:", response[actualDataKey][0]["3"]?.value, "last:", response[actualDataKey][response[actualDataKey].length - 1]["3"]?.value, "Continuation after fetch:", continuation);
+            }
+            if (paginationType === "skip") {
+                const newSkip = continuation?.type === "skip"
+                    ? continuation.value + response[actualDataKey].length
+                    : (response.metadata.skip ?? 0) + response[actualDataKey].length;
+                continuation = { type: "skip", value: newSkip };
+                if (response[actualDataKey].length === 0) {
+                    if (debug &&
+                        lastResponse.metadata.totalRecords &&
+                        newSkip < lastResponse.metadata.totalRecords) {
+                        console.warn("[paginateRecords] Warning: Empty response received but records remain", { skip: newSkip, totalRecords: lastResponse.metadata.totalRecords });
+                    }
+                    break;
+                }
+            }
+            else if (paginationType === "token") {
+                const newToken = response.metadata.nextPageToken ?? response.metadata.nextToken ?? "";
+                continuation = {
+                    type: "token",
+                    value: newToken,
+                    key: response.metadata.nextPageToken !== undefined
+                        ? "nextPageToken"
+                        : "nextToken",
+                };
+                if (continuation.value === "") {
+                    break;
+                }
+                else if (response[actualDataKey].length === 0 && newToken !== "") {
+                    if (debug)
+                        console.warn("[paginateRecords] Warning: Empty response with non-empty token, possible API inconsistency", { token: newToken });
+                }
+            }
+        }
+        if (!lastResponse) {
+            throw new Error("[paginateRecords] No response received during pagination");
+        }
+        const finalDataKey = getDataKey(lastResponse);
+        const finalResponse = {
+            [finalDataKey]: allRecords,
+            fields: lastResponse.fields,
+            metadata: {
+                totalRecords: paginationType === "skip"
+                    ? lastResponse.metadata.totalRecords
+                    : undefined,
+                numRecords: allRecords.length,
+                numFields: lastResponse.metadata.numFields,
+                skip: paginationType === "skip" ? 0 : undefined,
+                top: paginationType === "skip" ? undefined : undefined,
+                ...(paginationType === "token" &&
+                    continuation?.type === "token" &&
+                    continuation.key
+                    ? { [continuation.key]: "" }
+                    : {}),
+            },
+        };
+        if (debug) {
+            console.log("[paginateRecords] Final response summary:", {
+                totalRecords: finalResponse.metadata.totalRecords,
+                numRecords: finalResponse.metadata.numRecords,
+                numFields: finalResponse.metadata.numFields,
+                skip: finalResponse.metadata.skip,
+                top: finalResponse.metadata.top,
+                ...(paginationType === "token" &&
+                    continuation?.type === "token" &&
+                    continuation.key
+                    ? { [continuation.key]: finalResponse.metadata[continuation.key] }
+                    : {}),
+            });
+            console.log("[paginateRecords] Pagination stats:", {
+                totalRequests: requestCount,
+            });
+        }
+        return finalResponse;
+    }
+    /**
+     * Checks if a response is paginatable based on its structure.
+     * @param response - The response to check.
+     * @returns True if the response has a data array and pagination metadata, false otherwise.
+     */
+    function isPaginatable(response) {
+        return (response &&
+            typeof response === "object" &&
+            Object.keys(response).some((key) => Array.isArray(response[key])) &&
+            "metadata" in response &&
+            typeof response.metadata === "object" &&
+            (("totalRecords" in response.metadata &&
+                "numRecords" in response.metadata &&
+                "skip" in response.metadata) ||
+                "nextPageToken" in response.metadata ||
+                "nextToken" in response.metadata));
+    }
+
+    /**
+     * Invokes a Quick Base API method with the specified parameters, handling authentication, rate limiting, and pagination.
+     * @template K - The method name keyof QuickbaseClient.
+     * @param methodName - The API method to invoke (e.g., "runQuery", "changesetSolution").
+     * @param params - Parameters for the API method, potentially including skip, top, or a body.
+     * @param methodMap - Mapping of method names to their implementation details.
+     * @param baseHeaders - Base HTTP headers for API requests.
+     * @param authStrategy - Strategy for handling authentication tokens.
+     * @param rateLimiter - Rate limiting mechanism to respect API limits.
+     * @param transformDates - Function to transform date strings in the response.
+     * @param debug - If true, logs detailed invocation information.
+     * @param convertDates - If true, converts date strings to Date objects in the response.
+     * @param autoPaginate - If true, automatically paginates multi-page responses (default: true).
+     * @param attempt - Current retry attempt (default: 0).
+     * @param maxAttempts - Maximum number of retry attempts (default: rateLimiter.maxRetries + 1).
+     * @param isPaginating - Indicates if this call is part of an ongoing pagination (default: false).
+     * @returns A promise resolving to the API method's response.
+     * @throws {Error} If the method is not found or retries are exhausted.
+     */
+    async function invokeMethod(methodName, params, methodMap, baseHeaders, authStrategy, rateLimiter, transformDates, debug, convertDates, autoPaginate = true, attempt = 0, maxAttempts = rateLimiter.maxRetries + 1, isPaginating = false) {
         const methodInfo = methodMap[methodName];
         if (!methodInfo)
             throw new Error(`Method ${methodName} not found`);
         const hasBody = "body" in params && params.body !== undefined;
         const body = hasBody ? params.body : undefined;
+        // Safely check for options only if body is an object
+        const options = hasBody && body && typeof body === "object" && "options" in body
+            ? body.options
+            : undefined;
+        const adjustedParams = {
+            ...params,
+            skip: params.skip ?? (options && "skip" in options ? options.skip : undefined),
+            top: params.top ?? (options && "top" in options ? options.top : undefined),
+        };
         const restParams = hasBody
-            ? Object.fromEntries(Object.entries(params).filter(([key]) => key !== "body" && key !== "startTime"))
-            : { ...params, startTime: undefined };
+            ? Object.fromEntries(Object.entries(adjustedParams).filter(([key]) => key !== "body" && key !== "startTime"))
+            : { ...adjustedParams, startTime: undefined };
         const requestParameters = {
             ...restParams,
             ...(hasBody ? { generated: body } : {}),
         };
+        if (debug)
+            console.log("[invokeMethod] Adjusted params for pagination:", adjustedParams);
         const requestOptions = {
             credentials: methodName === "getTempTokenDBID" ? "include" : "omit",
             method: methodInfo.httpMethod,
@@ -7682,12 +7984,22 @@
                     ?.toLowerCase();
                 if (contentType?.includes("application/json")) {
                     const jsonResponse = await rawResponse.json();
+                    if (autoPaginate && !isPaginating && isPaginatable(jsonResponse)) {
+                        if (debug)
+                            console.log("[invokeMethod] Entering pagination with params:", adjustedParams);
+                        return paginateRecords(methodName, adjustedParams, methodMap, baseHeaders, authStrategy, rateLimiter, transformDates, debug, convertDates, jsonResponse);
+                    }
                     return transformDates(jsonResponse, convertDates);
                 }
                 return rawResponse;
             }
             if (rawResponse && typeof rawResponse.value === "function") {
                 const response = await rawResponse.value();
+                if (autoPaginate && !isPaginating && isPaginatable(response)) {
+                    if (debug)
+                        console.log("[invokeMethod] Entering pagination with params:", adjustedParams);
+                    return paginateRecords(methodName, adjustedParams, methodMap, baseHeaders, authStrategy, rateLimiter, transformDates, debug, convertDates, response);
+                }
                 return transformDates(response, convertDates);
             }
             return transformDates(rawResponse, convertDates);
@@ -7724,11 +8036,18 @@
             try {
                 await rateLimiter.throttle();
                 acquired = true;
-                const postThrottleTime = Date.now(); // Capture post-throttle time
+                const postThrottleTime = Date.now();
                 if (params.startTime !== undefined)
-                    params.startTime = postThrottleTime; // Update startTime for testing
-                const responsePromise = methodInfo.method(requestParameters, requestOptions);
-                rateLimiter.release(); // Release slot immediately after starting fetch
+                    params.startTime = postThrottleTime;
+                const headers = { ...baseHeaders };
+                const finalRequest = { ...requestParameters, headers };
+                if (debug)
+                    console.log("[invokeMethod] Sending request:", finalRequest);
+                const responsePromise = methodInfo.method(finalRequest, {
+                    ...requestOptions,
+                    headers,
+                });
+                rateLimiter.release();
                 const response = await responsePromise;
                 return await processResponse(response);
             }
@@ -7774,8 +8093,6 @@
                     token = newToken;
                     authStrategy.applyHeaders(baseHeaders, token);
                     requestOptions.headers = { ...baseHeaders };
-                    if (debug)
-                        console.log(`[${methodName}] Retrying with token: ${token.substring(0, 10)}...`);
                     attempt++;
                     continue;
                 }
@@ -7785,8 +8102,37 @@
         throw new Error(`API Error: Exhausted retries after ${maxAttempts} attempts`);
     }
 
-    // src/ThrottleBucket.ts
-    class ConcurrentThrottleBucket {
+    // src/Semaphore.ts
+    class Semaphore {
+        permits;
+        waiting = [];
+        constructor(maxPermits) {
+            this.permits = maxPermits;
+        }
+        async acquire() {
+            if (this.permits > 0) {
+                this.permits -= 1;
+                return;
+            }
+            return new Promise((resolve, reject) => {
+                this.waiting.push({ resolve, reject });
+            });
+        }
+        release() {
+            this.permits += 1;
+            if (this.waiting.length > 0 && this.permits > 0) {
+                const { resolve } = this.waiting.shift();
+                this.permits -= 1;
+                resolve();
+            }
+        }
+        available() {
+            return this.permits;
+        }
+    }
+
+    // src/FlowThrottleBucket.ts
+    class FlowThrottleBucket {
         tokens;
         maxTokens;
         refillRate;
@@ -7826,31 +8172,46 @@
             this.semaphore.release();
         }
     }
-    class Semaphore {
-        permits;
-        waiting = [];
-        constructor(maxPermits) {
-            this.permits = maxPermits;
+
+    // src/BurstAwareThrottleBucket.ts
+    class BurstAwareThrottleBucket {
+        maxTokens;
+        windowSeconds;
+        requestTimestamps = [];
+        semaphore;
+        constructor(options) {
+            this.maxTokens = options.maxTokens;
+            this.windowSeconds = options.windowSeconds;
+            this.semaphore = new Semaphore(this.maxTokens);
+        }
+        countRequestsInWindow() {
+            const now = Date.now();
+            this.requestTimestamps = this.requestTimestamps.filter((ts) => now - ts < this.windowSeconds * 1000);
+            return this.requestTimestamps.length;
+        }
+        availableTokens() {
+            const windowCount = this.countRequestsInWindow();
+            return Math.max(0, Math.min(100 - windowCount, this.maxTokens - windowCount));
         }
         async acquire() {
-            if (this.permits > 0) {
-                this.permits -= 1;
-                return;
+            while (true) {
+                const available = this.availableTokens();
+                if (available > 0 && this.semaphore.available() > 0) {
+                    await this.semaphore.acquire();
+                    this.requestTimestamps.push(Date.now());
+                    return;
+                }
+                const now = Date.now();
+                const oldestTimestamp = this.requestTimestamps[0];
+                let waitTime = 100;
+                if (oldestTimestamp) {
+                    waitTime = Math.max(0, this.windowSeconds * 1000 - (now - oldestTimestamp));
+                }
+                await new Promise((resolve) => setTimeout(resolve, waitTime));
             }
-            return new Promise((resolve, reject) => {
-                this.waiting.push({ resolve, reject });
-            });
         }
         release() {
-            this.permits += 1;
-            if (this.waiting.length > 0 && this.permits > 0) {
-                const { resolve } = this.waiting.shift();
-                this.permits -= 1;
-                resolve();
-            }
-        }
-        available() {
-            return this.permits;
+            this.semaphore.release();
         }
     }
 
@@ -7858,8 +8219,7 @@
         throttleBucket;
         maxRetries;
         retryDelay;
-        constructor(throttleBucket, // Updated type
-        maxRetries = 3, retryDelay = 1000) {
+        constructor(throttleBucket, maxRetries = 3, retryDelay = 1000) {
             this.throttleBucket = throttleBucket;
             this.maxRetries = maxRetries;
             this.retryDelay = retryDelay;
@@ -7883,20 +8243,23 @@
     }
 
     function quickbase(config) {
-        const { realm, userToken, tempToken, useTempTokens, useSso, samlToken, fetchApi, debug, convertDates = true, tempTokenLifespan = 290000, throttle = { rate: 5, burst: 3 }, // Updated default to match passing test
-        maxRetries = 3, retryDelay = 1000, tokenCache: providedTokenCache, baseUrl = "https://api.quickbase.com/v1", } = config;
+        const { realm, userToken, tempToken, useTempTokens = false, useSso = false, samlToken, fetchApi, debug, convertDates = true, tempTokenLifespan = 290000, throttle = { type: "flow", rate: 5, burst: 50 }, // Changed rate from 6 to 5
+        maxRetries = 3, retryDelay = 1000, tokenCache: providedTokenCache, baseUrl = "https://api.quickbase.com/v1", autoPaginate = true, } = config;
         const tokenCache = providedTokenCache || new TokenCache(tempTokenLifespan);
-        const throttleBucket = throttle
-            ? new ConcurrentThrottleBucket(throttle.rate, throttle.burst)
-            : null;
+        const throttleOptions = throttle;
+        const throttleBucket = throttleOptions.type === "burst-aware"
+            ? new BurstAwareThrottleBucket({
+                maxTokens: throttleOptions.burst || 100,
+                windowSeconds: throttleOptions.windowSeconds || 10,
+            })
+            : new FlowThrottleBucket(throttleOptions.rate || 5, // Changed rate from 6 to 5
+            throttleOptions.burst || 50);
         const rateLimiter = new RateLimiter(throttleBucket, maxRetries, retryDelay);
-        const defaultFetch = typeof globalThis.window !== "undefined"
-            ? globalThis.window.fetch.bind(globalThis.window)
-            : undefined;
-        const effectiveFetch = fetchApi || defaultFetch;
-        if (!effectiveFetch) {
-            throw new Error("No fetch implementation available");
+        const defaultFetch = globalThis.fetch || globalThis.window?.fetch?.bind(globalThis.window);
+        if (!defaultFetch && !fetchApi) {
+            throw new Error("No fetch implementation available. Please provide fetchApi in a Node.js environment without native fetch.");
         }
+        const effectiveFetch = fetchApi || defaultFetch;
         const authStrategy = useSso
             ? new SsoTokenStrategy(samlToken || "", realm, effectiveFetch, debug, baseUrl)
             : useTempTokens
@@ -7951,19 +8314,23 @@
         const methodMap = buildMethodMap();
         const proxyHandler = {
             get: (_, prop) => {
-                console.log("[proxy] Accessing:", prop, "in methodMap:", prop in methodMap);
+                if (debug) {
+                    console.log("[proxy] Accessing:", prop, "in methodMap:", prop in methodMap);
+                }
                 if (prop in methodMap) {
                     const methodName = prop;
-                    return (params) => invokeMethod(methodName, params, methodMap, baseHeaders, authStrategy, rateLimiter, transformDates, debug, convertDates);
+                    return (params) => invokeMethod(methodName, params, methodMap, baseHeaders, authStrategy, rateLimiter, transformDates, debug, convertDates, autoPaginate);
+                }
+                if (debug) {
+                    console.log("[proxy] Method not found:", prop);
                 }
-                console.log("[proxy] Method not found:", prop);
                 return undefined;
             },
         };
         const proxy = new Proxy({}, proxyHandler);
         if (debug) {
             console.log("[createClient] Config:", config);
-            console.log("[createClient] Proxy created:", proxy);
+            console.log("[createClient] Returning proxy");
         }
         return proxy;
     }