retoolrpc 0.1.0

package/dist/sdk.js

@@ -0,0 +1,440 @@
+import fetch from 'node-fetch';
+import { setTimeout } from 'timers/promises';
+import * as uuid from 'uuid';
+import { createLogger, transports } from 'winston';
+import { dedent } from 'ts-dedent';
+
+const RetoolSDKVersion = '0.0.35';
+
+const AGENT_SERVER_ERROR = 'AgentServerError';
+function createAgentServerError(error) {
+    if (error instanceof Error) {
+        const agentError = {
+            name: error.name,
+            message: error.message,
+            stack: error.stack,
+        };
+        if ('code' in error) {
+            if (typeof error.code === 'number') {
+                agentError.code = error.code;
+            }
+            else if (error.code === 'string') {
+                agentError.code = parseInt(error.code, 10);
+            }
+        }
+        if ('details' in error) {
+            agentError.details = error.details;
+        }
+        return agentError;
+    }
+    if (typeof error === 'string') {
+        return {
+            name: AGENT_SERVER_ERROR,
+            message: error,
+        };
+    }
+    return {
+        name: AGENT_SERVER_ERROR,
+        message: 'Unknown agent server error',
+    };
+}
+class FunctionNotFoundError extends Error {
+    constructor(functionName) {
+        super(`Function "${functionName}" not found on remote agent server.`);
+        this.name = 'FunctionNotFoundError';
+    }
+}
+class InvalidArgumentsError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'InvalidArgumentsError';
+    }
+}
+
+function pick(obj, keys) {
+    const result = {};
+    for (const key of keys) {
+        if (key in obj) {
+            result[key] = obj[key];
+        }
+    }
+    return result;
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function isFalsyArgumentValue(value) {
+    return value === null || value === undefined || value === '';
+}
+function isBooleanString(value) {
+    if (typeof value === 'string') {
+        const lowercaseValue = value.toLowerCase();
+        return lowercaseValue === 'true' || lowercaseValue === 'false';
+    }
+    return false;
+}
+function isNumberString(value) {
+    if (typeof value === 'string') {
+        // Use a regular expression to check if the string is a valid number
+        return /^-?\d+(\.\d+)?$/.test(value);
+    }
+    return false;
+}
+function isClientError(status) {
+    return status >= 400 && status < 500;
+}
+
+class ArgumentParser {
+    constructor(schema) {
+        this.schema = schema;
+    }
+    parse(argsToParse) {
+        const parsedArgs = { ...argsToParse };
+        const parsedErrors = [];
+        for (const argName in this.schema) {
+            const argDefinition = this.schema[argName];
+            const argValue = argsToParse[argName];
+            const falsyArgValue = isFalsyArgumentValue(argValue);
+            if (falsyArgValue) {
+                if (argDefinition.required) {
+                    parsedErrors.push(`Argument "${argName}" is required but missing.`);
+                    continue;
+                }
+            }
+            if (!falsyArgValue) {
+                if (argDefinition.array) {
+                    if (!Array.isArray(argValue)) {
+                        parsedErrors.push(`Argument "${argName}" should be an array.`);
+                        continue;
+                    }
+                    const parseValueTypeItems = argValue.map((item) => this.parseValueType(item, argDefinition.type));
+                    if (!parseValueTypeItems.every((item) => item.isValidType)) {
+                        parsedErrors.push(`Argument "${argName}" should be an array of type "${argDefinition.type}".`);
+                    }
+                    parsedArgs[argName] = parseValueTypeItems.map((item) => item.parsedValue);
+                }
+                else {
+                    const parsedValueTypeItem = this.parseValueType(argValue, argDefinition.type);
+                    if (!parsedValueTypeItem.isValidType) {
+                        parsedErrors.push(`Argument "${argName}" should be of type "${argDefinition.type}".`);
+                    }
+                    parsedArgs[argName] = parsedValueTypeItem.parsedValue;
+                }
+            }
+        }
+        return { parsedErrors, parsedArgs };
+    }
+    parseValueType(value, expectedType) {
+        switch (expectedType) {
+            case 'string':
+                if (typeof value === 'string') {
+                    return {
+                        isValidType: true,
+                        parsedValue: value,
+                    };
+                }
+                return {
+                    isValidType: false,
+                    parsedValue: value,
+                };
+            case 'boolean':
+                if (typeof value === 'boolean') {
+                    return {
+                        isValidType: true,
+                        parsedValue: value,
+                    };
+                }
+                if (isBooleanString(value)) {
+                    return {
+                        isValidType: true,
+                        parsedValue: value.toLowerCase() === 'true',
+                    };
+                }
+                return {
+                    isValidType: false,
+                    parsedValue: value,
+                };
+            case 'number':
+                if (typeof value === 'number') {
+                    return {
+                        isValidType: true,
+                        parsedValue: value,
+                    };
+                }
+                if (isNumberString(value)) {
+                    return {
+                        isValidType: true,
+                        parsedValue: parseFloat(value),
+                    };
+                }
+                return {
+                    isValidType: false,
+                    parsedValue: value,
+                };
+            case 'json':
+                try {
+                    const parsedJSONValue = JSON.parse(JSON.stringify(value));
+                    return {
+                        isValidType: true,
+                        parsedValue: parsedJSONValue,
+                    };
+                }
+                catch {
+                    return {
+                        isValidType: false,
+                        parsedValue: value,
+                    };
+                }
+            default:
+                const _expectedType = expectedType;
+                throw new Error(`Unknown argument type "${_expectedType}".`);
+        }
+    }
+}
+function parseFunctionArgs(args, schema) {
+    if (!isRecord(args)) {
+        throw new Error(`The given arguments are invalid.`);
+    }
+    const argumentParser = new ArgumentParser(schema);
+    const { parsedArgs, parsedErrors } = argumentParser.parse(args);
+    if (parsedErrors.length > 0) {
+        const invalidArgumentsError = dedent `
+      Invalid parameter(s) found:
+      ${parsedErrors.join('\n')}
+    `;
+        throw new InvalidArgumentsError(invalidArgumentsError);
+    }
+    return pick(parsedArgs, Object.keys(schema));
+}
+
+const logger = createLogger({
+    levels: {
+        error: 0,
+        warn: 1,
+        info: 2,
+        verbose: 3,
+        debug: 4, // log things like sql queries executed by the ORM
+    },
+    transports: [
+        new transports.Console({
+            level: 'info',
+        }),
+    ],
+    exitOnError: false,
+    silent: process.env.NODE_ENV === 'test',
+});
+const CONNECTION_ERROR_INITIAL_TIMEOUT_MS = 50;
+const MINIMUM_POLLING_INTERVAL_MS = 100;
+const CONNECTION_ERROR_RETRY_MAX_MS = 1000 * 60 * 10; // 10 minutes
+const POLLING_TIMEOUT_MS = 5 * 1000; // 5 seconds
+async function pollingTimeout() {
+    await setTimeout(POLLING_TIMEOUT_MS);
+    throw new Error(`Polling timeout after ${POLLING_TIMEOUT_MS}ms`);
+}
+async function loopWithBackoff(pollingIntervalMs, fn) {
+    let delayTimeMs = CONNECTION_ERROR_INITIAL_TIMEOUT_MS;
+    let lastLoopTime = Date.now();
+    while (true) {
+        if (logger.transports[0].level === 'debug') {
+            const currentTime = Date.now();
+            const loopDurationMs = currentTime - lastLoopTime;
+            logger.debug(`Loop time: ${loopDurationMs}ms, delay time: ${delayTimeMs}ms, polling interval: ${pollingIntervalMs}ms`);
+            lastLoopTime = currentTime;
+        }
+        try {
+            const result = await Promise.race([fn(), pollingTimeout()]);
+            if (result && result !== 'continue') {
+                return result;
+            }
+            await setTimeout(pollingIntervalMs);
+            delayTimeMs = Math.max(delayTimeMs / 2, CONNECTION_ERROR_INITIAL_TIMEOUT_MS);
+        }
+        catch (error) {
+            logger.error('Error running sdk agent', error);
+            await setTimeout(delayTimeMs);
+            delayTimeMs *= 2;
+            delayTimeMs = Math.min(delayTimeMs, CONNECTION_ERROR_RETRY_MAX_MS);
+        }
+    }
+}
+/**
+ * Represents the Retool SDK for interacting with Retool functions and contexts.
+ */
+class RetoolSDK {
+    /**
+     * Creates an instance of the RetoolSDK class.
+     */
+    constructor(config) {
+        this._functions = {};
+        this._apiKey = config.apiToken;
+        this._hostUrl = config.host.replace(/\/$/, ''); // Remove trailing / from host
+        this._resourceId = config.resourceId;
+        this._environmentName = config.environmentName || 'production';
+        this._pollingIntervalMs = config.pollingIntervalMs
+            ? Math.max(config.pollingIntervalMs, MINIMUM_POLLING_INTERVAL_MS)
+            : 1000;
+        this._version = config.version || '0.0.1';
+        this._agentUuid = config.agentUuid || uuid.v4();
+        if (config.logLevel) {
+            logger.transports[0].level = config.logLevel;
+        }
+        logger.debug({
+            apiKey: this._apiKey,
+            hostUrl: this._hostUrl,
+            resourceId: this._resourceId,
+            environmentName: this._environmentName,
+            agentUuid: this._agentUuid,
+            version: this._version,
+            pollingIntervalMs: this._pollingIntervalMs,
+        });
+    }
+    /**
+     * Asynchronously starts listening for incoming Retool function invocations.
+     */
+    async listen() {
+        logger.info('Starting sdk agent');
+        const registerResult = await loopWithBackoff(this._pollingIntervalMs, () => this._registerAgent());
+        if (registerResult === 'done') {
+            logger.info('Agent registered');
+            logger.info('Starting processing query');
+            loopWithBackoff(this._pollingIntervalMs, () => this._fetchQueryAndExecute());
+        }
+    }
+    /**
+     * Registers a Retool function with the specified function definition.
+     */
+    register({ name, args, impl }) {
+        this._functions[name] = {
+            args,
+            impl,
+        };
+    }
+    async executeFunction(functionName, args, context) {
+        logger.info(`Executing function: ${functionName}, context: ${context}`);
+        if (functionName === '__testConnection__') {
+            return { result: this._testConnection(args, context), args: {} };
+        }
+        const fnSpec = this._functions[functionName];
+        if (!fnSpec) {
+            throw new FunctionNotFoundError(functionName);
+        }
+        const parsedArgs = parseFunctionArgs(args, fnSpec.args);
+        logger.debug('Parsed args: ', parsedArgs);
+        const result = await fnSpec.impl(parsedArgs, context);
+        // TODO: consider truncating large arguments
+        return { result, args: parsedArgs };
+    }
+    _testConnection(_args, context) {
+        return {
+            success: true,
+            version: this._version,
+            agentUuid: this._agentUuid,
+            context,
+        };
+    }
+    async _registerAgent() {
+        const functionsMetadata = {};
+        for (const functionName in this._functions) {
+            functionsMetadata[functionName] = {
+                args: this._functions[functionName].args,
+            };
+        }
+        const response = await fetch(`${this._hostUrl}/api/v1/retoolsdk/registerAgent`, {
+            method: 'POST',
+            // TODO: add ApiToken
+            headers: {
+                Authorization: `Bearer ${this._apiKey}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                resourceId: this._resourceId,
+                environmentName: this._environmentName,
+                version: this._version,
+                agentUuid: this._agentUuid,
+                operations: functionsMetadata,
+            }),
+        });
+        if (!response.ok) {
+            if (isClientError(response.status)) {
+                logger.error(`Error registering agent: ${response.status} ${await response.text()}}`);
+                // client error, stop the client
+                return 'stop';
+            }
+            throw new Error(`Error connecting to retool server: ${response.status} ${await response.text()}}`);
+        }
+        const { versionHash } = await response.json();
+        this._versionHash = versionHash;
+        logger.info(`Agent registered with versionHash ${versionHash}`);
+        return 'done';
+    }
+    async _fetchQueryAndExecute() {
+        const pendingQueryFetch = await fetch(`${this._hostUrl}/api/v1/retoolsdk/popQuery`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this._apiKey}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                resourceId: this._resourceId,
+                environmentName: this._environmentName,
+                agentUuid: this._agentUuid,
+                versionHash: this._versionHash,
+            }),
+        });
+        if (!pendingQueryFetch.ok) {
+            if (isClientError(pendingQueryFetch.status)) {
+                logger.error(`Error fetching query (${pendingQueryFetch.status}): ${await pendingQueryFetch.text()}`);
+                return 'stop';
+            }
+            throw new Error(`Server error when fetching query: ${pendingQueryFetch.status}. Retrying...`);
+        }
+        const body = await pendingQueryFetch.json();
+        if (body.query) {
+            logger.debug('execute query', body.query); // This might contain sensitive information
+            const agentReceivedQueryAt = new Date().toISOString();
+            const queryUuid = body.query.queryUuid;
+            const { method, parameters, context } = body.query.queryInfo;
+            let status;
+            let executionResponse = undefined;
+            let executionArgs = undefined;
+            let agentError = undefined;
+            try {
+                const executionResult = await this.executeFunction(method, parameters, context);
+                executionResponse = executionResult.result;
+                executionArgs = executionResult.args;
+                status = 'success';
+            }
+            catch (err) {
+                status = 'error';
+                agentError = createAgentServerError(err);
+            }
+            const agentFinishedQueryAt = new Date().toISOString();
+            const updateQueryResponse = await fetch(`${this._hostUrl}/api/v1/retoolsdk/postQueryResponse`, {
+                method: 'POST',
+                headers: {
+                    Authorization: `Bearer ${this._apiKey}`,
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify({
+                    resourceId: this._resourceId,
+                    environmentName: this._environmentName,
+                    versionHash: this._versionHash,
+                    agentUuid: this._agentUuid,
+                    queryUuid,
+                    status,
+                    data: executionResponse,
+                    metadata: {
+                        agentReceivedQueryAt,
+                        agentFinishedQueryAt,
+                        parameters: executionArgs,
+                    },
+                    error: agentError,
+                }),
+            });
+            logger.debug('update query response status: ', updateQueryResponse.status, await updateQueryResponse.text());
+        }
+        return 'continue';
+    }
+}
+
+export { RetoolSDK, RetoolSDKVersion };

package/dist/src/addons/sequelize.d.ts

@@ -0,0 +1,13 @@
+import { RetoolRPC } from '../rpc';
+type Constructor<T = {}> = new (...args: any[]) => T;
+type RegisterModelArgs = {
+    model: any;
+    readAttributes?: string[];
+    writeAttributes?: string[];
+    findByAttributes?: string[];
+};
+export declare class MyMixinInterface {
+    registerModel(args: RegisterModelArgs): void;
+}
+export declare function sequelizeMixin<TBase extends Constructor<RetoolRPC>>(Base: TBase): Constructor<MyMixinInterface> & TBase;
+export {};

package/dist/src/example.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/rpc.d.ts

@@ -0,0 +1,56 @@
+import { RetoolRPCConfig, Arguments, RetoolContext, RegisterFunctionSpec, AgentServerStatus } from './types';
+/**
+ * Represents the Retool RPC for interacting with Retool functions and contexts.
+ */
+export declare class RetoolRPC {
+    private _apiKey;
+    private _hostUrl;
+    private _resourceId;
+    private _environmentName;
+    private _pollingIntervalMs;
+    private _version;
+    private _agentUuid;
+    private _versionHash;
+    private _functions;
+    private _retoolApi;
+    private _logger;
+    /**
+     * Creates an instance of the RetoolRPC class.
+     */
+    constructor(config: RetoolRPCConfig);
+    /**
+     * Asynchronously starts listening for incoming Retool function invocations.
+     */
+    listen(): Promise<void>;
+    /**
+     * Registers a Retool function with the specified function definition.
+     */
+    register<TArgs extends Arguments>(spec: RegisterFunctionSpec<TArgs>): void;
+    /**
+     * Executes a Retool function with the specified arguments and context.
+     */
+    executeFunction(functionName: string, functionArguments: unknown, context: RetoolContext): Promise<{
+        result: any;
+        arguments: Record<string, unknown>;
+    }>;
+    /**
+     * Tests the current connection to the Retool server.
+     */
+    private testConnection;
+    /**
+     * Registers the agent with the Retool server.
+     */
+    private registerAgent;
+    /**
+     * Fetches a query from the Retool server and executes it.
+     */
+    private fetchQueryAndExecute;
+    /**
+     * Export registerAgent as a protected method for testing purpose only. Lookup TestRetoolRPC for usage.
+     */
+    protected protectedRegisterAgent(): Promise<AgentServerStatus>;
+    /**
+     * Export fetchQueryAndExecute as a protected method for testing purpose only. Lookup TestRetoolRPC for usage.
+     */
+    protected protectedFetchQueryAndExecute(): Promise<AgentServerStatus>;
+}

package/dist/src/rpc.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/sdk.d.ts

@@ -0,0 +1,57 @@
+import { RetoolSDKConfig, Arguments, RetoolContext, RegisterFunctionSpec, LoopCallbackResult } from './types';
+export { RetoolSDKVersion } from './version';
+/**
+ * Represents the Retool SDK for interacting with Retool functions and contexts.
+ */
+export declare class RetoolSDK {
+    private _apiKey;
+    private _hostUrl;
+    private _resourceId;
+    private _environmentName;
+    private _pollingIntervalMs;
+    private _version;
+    private _agentUuid;
+    private _versionHash;
+    private _functions;
+    private _retoolApi;
+    private _logger;
+    /**
+     * Creates an instance of the RetoolSDK class.
+     */
+    constructor(config: RetoolSDKConfig);
+    /**
+     * Asynchronously starts listening for incoming Retool function invocations.
+     */
+    listen(): Promise<void>;
+    /**
+     * Registers a Retool function with the specified function definition.
+     */
+    register<TArgs extends Arguments>(spec: RegisterFunctionSpec<TArgs>): void;
+    /**
+     * Executes a Retool function with the specified arguments and context.
+     */
+    executeFunction(functionName: string, args: unknown, context: RetoolContext): Promise<{
+        result: any;
+        args: import("./types").TransformedArguments<Arguments>;
+    }>;
+    /**
+     * Tests the current connection to the Retool server.
+     */
+    private testConnection;
+    /**
+     * Registers the agent with the Retool server.
+     */
+    private registerAgent;
+    /**
+     * Fetches a query from the Retool server and executes it.
+     */
+    private fetchQueryAndExecute;
+    /**
+     * Export for testing purpose only.
+     */
+    _registerAgent(): Promise<LoopCallbackResult>;
+    /**
+     * Export for testing purpose only.
+     */
+    _fetchQueryAndExecute(): Promise<LoopCallbackResult>;
+}

package/dist/src/sdk.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/types.d.ts

@@ -0,0 +1,93 @@
+/**
+ * Configuration options for the Retool RPC.
+ */
+export type RetoolRPCConfig = {
+    /** The host URL of the Retool instance. */
+    host: string;
+    /** The API token for authenticating with the Retool instance. */
+    apiToken: string;
+    /** The ID of the resource in Retool (UUID format). */
+    resourceId: string;
+    /** The optional environment identifier. Defaults to `production` */
+    environmentName?: string;
+    /**
+     * The version of the schema being used. If the resource enforce explicit versioning,
+     * this property must be set, and must be updated with each schema change.
+     * */
+    version?: string;
+    /** The optional polling interval in milliseconds. Defaults to 1000. Minimum is 100. */
+    pollingIntervalMs?: number;
+    /** The optional UUID of the agent. Will be automatically generated by default */
+    agentUuid?: string;
+    /** The optional log level. */
+    logLevel?: 'debug' | 'info' | 'warn' | 'error';
+};
+/** Represents the type of the argument. Right now we are supporting only string, boolean, number, dict, and json. */
+export type ArgumentType = 'string' | 'boolean' | 'number' | 'dict' | 'json';
+/** Represents an argument definition for a Retool function. */
+export type Argument = {
+    /** The type of the argument. */
+    type: ArgumentType;
+    /** Specifies whether the argument is expected to be an array. */
+    array?: boolean;
+    /** The description of the argument. */
+    description?: string;
+    /** Specifies whether the argument is required. */
+    required?: boolean;
+};
+/** Recursive JSON type */
+export type Json = string | number | boolean | null | Json[] | {
+    [property: string]: Json;
+};
+/** Represents a map of argument name to argument type. */
+export type ArgumentTypeMap<T extends Argument> = T['type'] extends 'string' ? string : T['type'] extends 'number' ? number : T['type'] extends 'boolean' ? boolean : T['type'] extends 'dict' ? Record<string, any> : T['type'] extends 'json' ? Json : never;
+/** Represents a map of argument names to argument definitions. */
+export type Arguments = Record<string, Argument>;
+/** Represents a map of argument name to argument type. */
+export type TransformedArgument<TArg extends Argument> = TArg['array'] extends true ? Array<ArgumentTypeMap<TArg>> : ArgumentTypeMap<TArg>;
+/** Represents a map of argument names to argument types. */
+export type TransformedArguments<TArgs extends Arguments> = {
+    [TArg in keyof TArgs]: TransformedArgument<TArgs[TArg]>;
+};
+/** Represents the specification for registering a Retool function. */
+export type RegisterFunctionSpec<TArgs extends Arguments> = {
+    /** The name of the function. */
+    name: string;
+    /** The arguments of the function. */
+    arguments: Pick<TArgs, keyof TArgs>;
+    /** The implementation of the function. */
+    implementation: (args: TransformedArguments<TArgs>, context: RetoolContext) => Promise<any>;
+    /** The permissions configuration for the function. */
+    permissions?: {
+        /** The list of group names that have permission to execute the function. */
+        groupNames?: string[];
+        /** The list of user emails that have permission to execute the function. */
+        userEmails?: string[];
+    };
+};
+/** Additional context information that is passed to the Retool function. */
+export type RetoolContext = {
+    /** Name of the user who is executing the function. */
+    userName?: string;
+    /** Email address of the user who is executing the function. */
+    userEmail?: string;
+    /** A list of group names of the user who is executing the function. */
+    userGroups?: string[];
+    /** The organization name of the user who is executing the function. */
+    organizationName?: string;
+};
+/** Agent server error that is thrown by Retool RPC. */
+export type AgentServerError = {
+    /** The error name. */
+    name: string;
+    /** The error message. */
+    message: string;
+    /** The error stack track. */
+    stack?: string;
+    /** The error status code. */
+    code?: number;
+    /** The error additional details. */
+    details?: unknown;
+};
+/** Represents the current status of a function execution. */
+export type AgentServerStatus = 'continue' | 'stop' | 'done';