retoolrpc 0.1.3 → 0.1.5

package/dist/rpc-86842ea7.mjs

@@ -0,0 +1,534 @@
+import { v4 } from 'uuid';
+import { dedent } from 'ts-dedent';
+import fetch from 'node-fetch';
+import AbortControllerFallback from 'abort-controller';
+
+const AGENT_SERVER_ERROR = 'AgentServerError';
+function createAgentServerError(error) {
+    if (error instanceof Error) {
+        const agentError = {
+            name: error.name,
+            message: error.message,
+            stack: error.stack,
+        };
+        if ('code' in error) {
+            if (typeof error.code === 'number') {
+                agentError.code = error.code;
+            }
+            else if (error.code === 'string') {
+                agentError.code = parseInt(error.code, 10);
+            }
+        }
+        if ('details' in error) {
+            agentError.details = error.details;
+        }
+        return agentError;
+    }
+    if (typeof error === 'string') {
+        return {
+            name: AGENT_SERVER_ERROR,
+            message: error,
+        };
+    }
+    return {
+        name: AGENT_SERVER_ERROR,
+        message: 'Unknown agent server error',
+    };
+}
+class FunctionNotFoundError extends Error {
+    constructor(functionName) {
+        super(`Function "${functionName}" not found on remote agent server.`);
+        this.name = 'FunctionNotFoundError';
+    }
+}
+class InvalidArgumentsError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'InvalidArgumentsError';
+    }
+}
+
+function pick(obj, keys) {
+    const result = {};
+    for (const key of keys) {
+        if (key in obj) {
+            result[key] = obj[key];
+        }
+    }
+    return result;
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function isFalsyArgumentValue(value) {
+    return value === null || value === undefined || value === '';
+}
+function isBooleanString(value) {
+    if (typeof value === 'string') {
+        const lowercaseValue = value.toLowerCase();
+        return lowercaseValue === 'true' || lowercaseValue === 'false';
+    }
+    return false;
+}
+function isNumberString(value) {
+    if (typeof value === 'string') {
+        // Use a regular expression to check if the string is a valid number
+        return /^-?\d+(\.\d+)?$/.test(value);
+    }
+    return false;
+}
+function isClientError(status) {
+    return status >= 400 && status < 500;
+}
+
+class ArgumentParser {
+    constructor(schema) {
+        this.schema = schema;
+    }
+    parse(argumentsToParse) {
+        const parsedArguments = { ...argumentsToParse };
+        const parsedErrors = [];
+        for (const argName in this.schema) {
+            const argDefinition = this.schema[argName];
+            const argValue = argumentsToParse[argName];
+            const falsyArgValue = isFalsyArgumentValue(argValue);
+            if (falsyArgValue) {
+                if (argDefinition.required) {
+                    parsedErrors.push(`Argument "${argName}" is required but missing.`);
+                    continue;
+                }
+            }
+            if (!falsyArgValue) {
+                if (argDefinition.array) {
+                    if (!Array.isArray(argValue)) {
+                        parsedErrors.push(`Argument "${argName}" should be an array.`);
+                        continue;
+                    }
+                    const parseValueTypeItems = argValue.map((item) => this.parseValueType(item, argDefinition.type));
+                    if (!parseValueTypeItems.every((item) => item.isValidType)) {
+                        parsedErrors.push(`Argument "${argName}" should be an array of type "${argDefinition.type}".`);
+                    }
+                    parsedArguments[argName] = parseValueTypeItems.map((item) => item.parsedValue);
+                }
+                else {
+                    const parsedValueTypeItem = this.parseValueType(argValue, argDefinition.type);
+                    if (!parsedValueTypeItem.isValidType) {
+                        parsedErrors.push(`Argument "${argName}" should be of type "${argDefinition.type}".`);
+                    }
+                    parsedArguments[argName] = parsedValueTypeItem.parsedValue;
+                }
+            }
+        }
+        return { parsedErrors, parsedArguments };
+    }
+    parseValueType(value, expectedType) {
+        switch (expectedType) {
+            case 'string':
+                // For string type, we just need to convert to string.
+                return {
+                    isValidType: true,
+                    parsedValue: typeof value === 'object' ? JSON.stringify(value) : String(value), // Need to do this because String(object) returns "[object Object]".
+                };
+            case 'boolean':
+                // For boolean type, we need to check if the value is a boolean or a boolean string.
+                if (typeof value === 'boolean') {
+                    return {
+                        isValidType: true,
+                        parsedValue: value,
+                    };
+                }
+                if (isBooleanString(value)) {
+                    return {
+                        isValidType: true,
+                        parsedValue: value.toLowerCase() === 'true',
+                    };
+                }
+                return {
+                    isValidType: false,
+                    parsedValue: value,
+                };
+            case 'number':
+                // For number type, we need to check if the value is a number or a number string.
+                if (typeof value === 'number') {
+                    return {
+                        isValidType: true,
+                        parsedValue: value,
+                    };
+                }
+                if (isNumberString(value)) {
+                    return {
+                        isValidType: true,
+                        parsedValue: parseFloat(value),
+                    };
+                }
+                return {
+                    isValidType: false,
+                    parsedValue: value,
+                };
+            case 'dict':
+                // For dict type, we need to check if the value is a record.
+                if (isRecord(value)) {
+                    return {
+                        isValidType: true,
+                        parsedValue: value,
+                    };
+                }
+                return {
+                    isValidType: false,
+                    parsedValue: value,
+                };
+            case 'json':
+                // For json type, we need to check if the value is a valid JSON string.
+                try {
+                    const parsedJSONValue = JSON.parse(JSON.stringify(value));
+                    return {
+                        isValidType: true,
+                        parsedValue: parsedJSONValue,
+                    };
+                }
+                catch {
+                    return {
+                        isValidType: false,
+                        parsedValue: value,
+                    };
+                }
+            default:
+                throw new Error(`Unknown argument type "${expectedType}".`);
+        }
+    }
+}
+function parseFunctionArguments(args, schema) {
+    if (!isRecord(args)) {
+        throw new Error(`The given arguments are invalid.`);
+    }
+    const argumentParser = new ArgumentParser(schema);
+    const { parsedArguments, parsedErrors } = argumentParser.parse(args);
+    if (parsedErrors.length > 0) {
+        const invalidArgumentsError = dedent `
+      Invalid parameter(s) found:
+      ${parsedErrors.join('\n')}
+    `;
+        throw new InvalidArgumentsError(invalidArgumentsError);
+    }
+    return pick(parsedArguments, Object.keys(schema));
+}
+
+const CONNECTION_ERROR_INITIAL_TIMEOUT_MS = 50;
+const CONNECTION_ERROR_RETRY_MAX_MS = 1000 * 60 * 10; // 10 minutes
+function sleep(ms) {
+    return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    });
+}
+async function loopWithBackoff(pollingIntervalMs, logger, callback) {
+    let delayTimeMs = CONNECTION_ERROR_INITIAL_TIMEOUT_MS;
+    let lastLoopTimestamp = Date.now();
+    while (true) {
+        try {
+            const result = await callback();
+            const currentTimestamp = Date.now();
+            const loopDurationMs = currentTimestamp - lastLoopTimestamp;
+            lastLoopTimestamp = currentTimestamp;
+            logger.debug(`Loop time: ${loopDurationMs}ms, delay time: ${delayTimeMs}ms, polling interval: ${pollingIntervalMs}ms`);
+            if (result !== 'continue') {
+                return result;
+            }
+            await sleep(pollingIntervalMs);
+            delayTimeMs = Math.max(delayTimeMs / 2, CONNECTION_ERROR_INITIAL_TIMEOUT_MS);
+        }
+        catch (err) {
+            logger.error('Error running RPC agent', err);
+            await sleep(delayTimeMs);
+            delayTimeMs = Math.min(delayTimeMs * 2, CONNECTION_ERROR_RETRY_MAX_MS);
+        }
+    }
+}
+
+const LOG_LEVEL_RANKINGS = {
+    debug: 0,
+    info: 1,
+    warn: 2,
+    error: 3,
+};
+class Logger {
+    constructor(options) {
+        this.currentLogLevel = options.logLevel || 'info'; // Default to 'info' if logLevel is not specified
+    }
+    shouldLog(level) {
+        return LOG_LEVEL_RANKINGS[level] >= LOG_LEVEL_RANKINGS[this.currentLogLevel] && process.env.NODE_ENV !== 'test';
+    }
+    debug(...messages) {
+        if (this.shouldLog('debug')) {
+            console.log(...messages);
+        }
+    }
+    info(...messages) {
+        if (this.shouldLog('info')) {
+            console.log(...messages);
+        }
+    }
+    warn(...messages) {
+        if (this.shouldLog('warn')) {
+            console.log(...messages);
+        }
+    }
+    error(...messages) {
+        if (this.shouldLog('error')) {
+            console.log(...messages);
+        }
+    }
+}
+
+const RetoolRPCVersion = '0.1.5';
+
+// AbortController was added in node v14.17.0 globally, but we need to polyfill it for older versions
+const AbortController = globalThis.AbortController || AbortControllerFallback;
+class RetoolAPI {
+    constructor({ hostUrl, apiKey, pollingTimeoutMs }) {
+        this._hostUrl = hostUrl;
+        this._apiKey = apiKey;
+        this._pollingTimeoutMs = pollingTimeoutMs;
+    }
+    async popQuery(options) {
+        const abortController = new AbortController();
+        setTimeout(() => {
+            abortController.abort();
+        }, this._pollingTimeoutMs);
+        try {
+            return await fetch(`${this._hostUrl}/api/v1/retoolrpc/popQuery`, {
+                method: 'POST',
+                headers: {
+                    Authorization: `Bearer ${this._apiKey}`,
+                    'Content-Type': 'application/json',
+                    'User-Agent': `RetoolRPC/${RetoolRPCVersion} (Javascript)`,
+                },
+                body: JSON.stringify(options),
+                // Had to cast to RequestInit['signal'] because of a bug in the types
+                // https://github.com/jasonkuhrt/graphql-request/issues/481
+                signal: abortController.signal,
+            });
+        }
+        catch (error) {
+            if (abortController.signal.aborted) {
+                throw new Error(`Polling timeout after ${this._pollingTimeoutMs}ms`);
+            }
+            throw error;
+        }
+    }
+    async registerAgent(options) {
+        return fetch(`${this._hostUrl}/api/v1/retoolrpc/registerAgent`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this._apiKey}`,
+                'Content-Type': 'application/json',
+                'User-Agent': `RetoolRPC/${RetoolRPCVersion} (Javascript)`,
+            },
+            body: JSON.stringify(options),
+        });
+    }
+    async postQueryResponse(options) {
+        return fetch(`${this._hostUrl}/api/v1/retoolrpc/postQueryResponse`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this._apiKey}`,
+                'Content-Type': 'application/json',
+                'User-Agent': `RetoolRPC/${RetoolRPCVersion} (Javascript)`,
+            },
+            body: JSON.stringify(options),
+        });
+    }
+}
+
+const MINIMUM_POLLING_INTERVAL_MS = 100;
+const DEFAULT_POLLING_INTERVAL_MS = 1000;
+const DEFAULT_POLLING_TIMEOUT_MS = 5000;
+const DEFAULT_ENVIRONMENT_NAME = 'production';
+const DEFAULT_VERSION = '0.0.1';
+/**
+ * Represents the Retool RPC for interacting with Retool functions and contexts.
+ */
+class RetoolRPC {
+    /**
+     * Creates an instance of the RetoolRPC class.
+     */
+    constructor(config) {
+        var _a;
+        this._functions = {};
+        this._apiKey = config.apiToken;
+        this._hostUrl = config.host.replace(/\/$/, ''); // Remove trailing / from host
+        this._resourceId = config.resourceId;
+        this._environmentName = config.environmentName || DEFAULT_ENVIRONMENT_NAME;
+        this._pollingIntervalMs = config.pollingIntervalMs
+            ? Math.max(config.pollingIntervalMs, MINIMUM_POLLING_INTERVAL_MS)
+            : DEFAULT_POLLING_INTERVAL_MS;
+        this._pollingTimeoutMs = config.pollingTimeoutMs || DEFAULT_POLLING_TIMEOUT_MS;
+        this._version = config.version || DEFAULT_VERSION;
+        this._agentUuid = config.agentUuid || v4();
+        this._retoolApi = new RetoolAPI({
+            hostUrl: this._hostUrl,
+            apiKey: this._apiKey,
+            pollingTimeoutMs: this._pollingTimeoutMs || DEFAULT_POLLING_TIMEOUT_MS,
+        });
+        this._logger = (_a = config.logger) !== null && _a !== void 0 ? _a : new Logger({ logLevel: config.logLevel });
+        this._logger.debug('Retool RPC Configuration', {
+            apiKey: this._apiKey,
+            hostUrl: this._hostUrl,
+            resourceId: this._resourceId,
+            environmentName: this._environmentName,
+            agentUuid: this._agentUuid,
+            version: this._version,
+            pollingIntervalMs: this._pollingIntervalMs,
+        });
+    }
+    /**
+     * Asynchronously starts listening for incoming Retool function invocations.
+     */
+    async listen() {
+        this._logger.info('Starting RPC agent');
+        const registerResult = await loopWithBackoff(this._pollingIntervalMs, this._logger, () => this.registerAgent());
+        if (registerResult === 'done') {
+            this._logger.info('Agent registered');
+            this._logger.info('Starting processing query');
+            loopWithBackoff(this._pollingIntervalMs, this._logger, () => this.fetchQueryAndExecute());
+        }
+    }
+    /**
+     * Registers a Retool function with the specified function definition.
+     */
+    register(spec) {
+        this._functions[spec.name] = {
+            arguments: spec.arguments,
+            permissions: spec.permissions,
+            implementation: spec.implementation,
+        };
+        return spec.implementation;
+    }
+    /**
+     * Executes a Retool function with the specified arguments and context.
+     */
+    async executeFunction(functionName, functionArguments, context) {
+        this._logger.info(`Executing function: ${functionName}, context: ${context}`);
+        if (functionName === '__testConnection__') {
+            return { result: this.testConnection(context), arguments: {} };
+        }
+        const fnSpec = this._functions[functionName];
+        if (!fnSpec) {
+            throw new FunctionNotFoundError(functionName);
+        }
+        const parsedArguments = parseFunctionArguments(functionArguments, fnSpec.arguments);
+        this._logger.debug('Parsed arguments: ', parsedArguments);
+        const result = await fnSpec.implementation(parsedArguments, context);
+        // Consider truncating large arguments
+        return { result, arguments: parsedArguments };
+    }
+    /**
+     * Tests the current connection to the Retool server.
+     */
+    testConnection(context) {
+        return {
+            success: true,
+            version: this._version,
+            agentUuid: this._agentUuid,
+            context,
+        };
+    }
+    /**
+     * Registers the agent with the Retool server.
+     */
+    async registerAgent() {
+        const functionsMetadata = {};
+        for (const functionName in this._functions) {
+            functionsMetadata[functionName] = {
+                arguments: this._functions[functionName].arguments,
+                permissions: this._functions[functionName].permissions,
+            };
+        }
+        const registerAgentResponse = await this._retoolApi.registerAgent({
+            resourceId: this._resourceId,
+            environmentName: this._environmentName,
+            version: this._version,
+            agentUuid: this._agentUuid,
+            operations: functionsMetadata,
+        });
+        if (!registerAgentResponse.ok) {
+            if (isClientError(registerAgentResponse.status)) {
+                this._logger.error(`Error registering agent: ${registerAgentResponse.status} ${await registerAgentResponse.text()}}`);
+                // client error, stop the client
+                return 'stop';
+            }
+            throw new Error(`Error connecting to retool server: ${registerAgentResponse.status} ${await registerAgentResponse.text()}}`);
+        }
+        const { versionHash } = await registerAgentResponse.json();
+        this._versionHash = versionHash;
+        this._logger.info(`Agent registered with versionHash: ${versionHash}`);
+        return 'done';
+    }
+    /**
+     * Fetches a query from the Retool server and executes it.
+     */
+    async fetchQueryAndExecute() {
+        const pendingQueryFetch = await this._retoolApi.popQuery({
+            resourceId: this._resourceId,
+            environmentName: this._environmentName,
+            agentUuid: this._agentUuid,
+            versionHash: this._versionHash,
+        });
+        if (!pendingQueryFetch.ok) {
+            if (isClientError(pendingQueryFetch.status)) {
+                this._logger.error(`Error fetching query (${pendingQueryFetch.status}): ${await pendingQueryFetch.text()}`);
+                return 'stop';
+            }
+            throw new Error(`Server error when fetching query: ${pendingQueryFetch.status}. Retrying...`);
+        }
+        const { query } = await pendingQueryFetch.json();
+        if (query) {
+            this._logger.debug('Executing query', query); // This might contain sensitive information
+            const agentReceivedQueryAt = new Date().toISOString();
+            const queryUuid = query.queryUuid;
+            const { method, parameters, context } = query.queryInfo;
+            let status;
+            let executionResponse = undefined;
+            let executionArguments = undefined;
+            let agentError = undefined;
+            this.executeFunction(method, parameters, context)
+                .then((executionResult) => {
+                executionResponse = executionResult.result;
+                executionArguments = executionResult.arguments;
+                status = 'success';
+            })
+                .catch((err) => {
+                agentError = createAgentServerError(err);
+                status = 'error';
+            })
+                .finally(() => {
+                this._retoolApi
+                    .postQueryResponse({
+                    resourceId: this._resourceId,
+                    environmentName: this._environmentName,
+                    versionHash: this._versionHash,
+                    agentUuid: this._agentUuid,
+                    queryUuid,
+                    status,
+                    data: executionResponse,
+                    metadata: {
+                        packageLanguage: 'javascript',
+                        packageVersion: RetoolRPCVersion,
+                        agentReceivedQueryAt,
+                        agentFinishedQueryAt: new Date().toISOString(),
+                        parameters: executionArguments,
+                    },
+                    error: agentError,
+                })
+                    .then(async (updateQueryResponse) => {
+                    this._logger.debug('Update query response status: ', updateQueryResponse.status, await updateQueryResponse.text());
+                })
+                    .catch((err) => {
+                    this._logger.error(`Error updating query response: `, err);
+                });
+            });
+        }
+        return 'continue';
+    }
+}
+
+export { RetoolRPC as R };

package/dist/src/rpc.d.ts

@@ -26,7 +26,7 @@ export declare class RetoolRPC {
     /**
      * Registers a Retool function with the specified function definition.
      */
-    register<TArgs extends Arguments>(spec: RegisterFunctionSpec<TArgs>): void;
+    register<TArgs extends Arguments, TReturn>(spec: RegisterFunctionSpec<TArgs, TReturn>): RegisterFunctionSpec<TArgs, TReturn>['implementation'];
     /**
      * Executes a Retool function with the specified arguments and context.
      */

package/dist/src/types.d.ts

@@ -55,13 +55,13 @@ export type TransformedArguments<TArgs extends Arguments> = {
     [TArg in keyof TArgs]: TransformedArgument<TArgs[TArg]>;
 };
 /** Represents the specification for registering a Retool function. */
-export type RegisterFunctionSpec<TArgs extends Arguments> = {
+export type RegisterFunctionSpec<TArgs extends Arguments, TReturn> = {
     /** The name of the function. */
     name: string;
     /** The arguments of the function. */
     arguments: Pick<TArgs, keyof TArgs>;
     /** The implementation of the function. */
-    implementation: (args: TransformedArguments<TArgs>, context: RetoolContext) => Promise<any>;
+    implementation: (args: TransformedArguments<TArgs>, context: RetoolContext) => Promise<TReturn>;
     /** The permissions configuration for the function. */
     permissions?: {
         /** The list of group names that have permission to execute the function. */

package/dist/src/version.d.ts

@@ -1 +1 @@
-export declare const RetoolRPCVersion = "0.1.3";
+export declare const RetoolRPCVersion = "0.1.5";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "retoolrpc",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "description": "TypeScript package for Retool RPC",
   "keywords": [],
   "homepage": "https://github.com/tryretool/retoolrpc#readme",
@@ -12,10 +12,14 @@
     "url": "git+https://github.com/tryretool/retoolrpc.git"
   },
   "license": "MIT",
+  "main": "./dist/cjs/index.js",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
   "exports": {
     ".": {
       "import": "./dist/index.mjs",
-      "require": "./dist/cjs/index.js"
+      "require": "./dist/cjs/index.js",
+      "types": "./dist/index.d.ts"
     },
     "./package.json": "./package.json"
   },
@@ -30,8 +34,6 @@
     "test:api": "tsc --project tsconfig.json"
   },
   "dependencies": {
-    "@types/node-fetch": "^2.6.4",
-    "@types/uuid": "^9.0.2",
     "abort-controller": "^3.0.0",
     "node-fetch": "^2.6",
     "ts-dedent": "^2.2.0",
@@ -41,7 +43,9 @@
     "@rollup/plugin-typescript": "^8.2.0",
     "@types/fs-extra": "^11.0.1",
     "@types/node": "15.12.1",
+    "@types/node-fetch": "^2.6.4",
     "@types/semver": "^7.5.1",
+    "@types/uuid": "^9.0.2",
     "cross-env": "7.0.3",
     "fs-extra": "^11.1.1",
     "nock": "^13.3.2",

package/src/rpc.spec.ts

@@ -2,11 +2,11 @@ import fs from 'fs-extra'
 import nock from 'nock'
 import crypto from 'crypto'
 import { dedent } from 'ts-dedent'
-import { describe, expect, beforeEach, afterEach, test, vi } from 'vitest'
+import { describe, expect, beforeEach, afterEach, test, vi, expectTypeOf } from 'vitest'
 import { v4 as uuidv4 } from 'uuid'
 
 import { RetoolRPC } from './rpc'
-import { Arguments } from './types'
+import { Arguments, RetoolContext, TransformedArguments } from './types'
 import { parseFunctionArguments } from './utils/schema'
 import { RetoolRPCVersion } from './version'
 
@@ -751,6 +751,25 @@ describe('RetoolRPC', () => {
       )
     })
   })
+
+  test('returns the implementation when registering', async () => {
+    const fn = rpcAgent.register({
+      name: 'test',
+      arguments: {},
+      implementation: async () => {
+        return 1
+      }
+    })
+
+    type ExpectedImplementation = (args: TransformedArguments<Arguments>, context: RetoolContext) => Promise<number>
+    expectTypeOf(fn).toEqualTypeOf<ExpectedImplementation>()
+    
+
+    const result = await fn({}, context);
+    expect(result).toEqual(1)
+    expectTypeOf(result).toEqualTypeOf(1)
+  })
+
 })
 
 describe('RetoolRPCVersion', () => {

package/src/rpc.ts

@@ -36,7 +36,7 @@ export class RetoolRPC {
   private _version: string
   private _agentUuid: string
   private _versionHash: string | undefined
-  private _functions: Record<string, Omit<RegisterFunctionSpec<any>, 'name'>> = {}
+  private _functions: Record<string, Omit<RegisterFunctionSpec<any, any>, 'name'>> = {}
   private _retoolApi: RetoolAPI
   private _logger: LoggerService
 
@@ -89,12 +89,13 @@ export class RetoolRPC {
   /**
    * Registers a Retool function with the specified function definition.
    */
-  register<TArgs extends Arguments>(spec: RegisterFunctionSpec<TArgs>): void {
+  register<TArgs extends Arguments, TReturn>(spec: RegisterFunctionSpec<TArgs, TReturn>): RegisterFunctionSpec<TArgs, TReturn>['implementation'] {
     this._functions[spec.name] = {
       arguments: spec.arguments,
       permissions: spec.permissions,
       implementation: spec.implementation,
     }
+    return spec.implementation;
   }
 
   /**
@@ -145,7 +146,7 @@ export class RetoolRPC {
    * Registers the agent with the Retool server.
    */
   private async registerAgent(): Promise<AgentServerStatus> {
-    const functionsMetadata: Record<string, Pick<RegisterFunctionSpec<any>, 'arguments' | 'permissions'>> = {}
+    const functionsMetadata: Record<string, Pick<RegisterFunctionSpec<any, any>, 'arguments' | 'permissions'>> = {}
     for (const functionName in this._functions) {
       functionsMetadata[functionName] = {
         arguments: this._functions[functionName].arguments,