retoolrpc 0.1.0

package/src/rpc.ts

@@ -0,0 +1,262 @@
+import { v4 as uuidv4 } from 'uuid'
+
+import {
+  RetoolRPCConfig,
+  Arguments,
+  RetoolContext,
+  AgentServerError,
+  RegisterFunctionSpec,
+  AgentServerStatus,
+} from './types'
+
+import { createAgentServerError, FunctionNotFoundError } from './utils/errors'
+import { parseFunctionArguments } from './utils/schema'
+import { isClientError } from './utils/helpers'
+import { loopWithBackoff } from './utils/polling'
+import { Logger } from './utils/logger'
+import { RetoolAPI } from './utils/api'
+import { RetoolRPCVersion } from './version'
+
+const MINIMUM_POLLING_INTERVAL_MS = 100
+const DEFAULT_POLLING_INTERVAL_MS = 1000
+const DEFAULT_ENVIRONMENT_NAME = 'production'
+const DEFAULT_VERSION = '0.0.1'
+
+/**
+ * Represents the Retool RPC for interacting with Retool functions and contexts.
+ */
+export class RetoolRPC {
+  private _apiKey: string
+  private _hostUrl: string
+  private _resourceId: string
+  private _environmentName: string
+  private _pollingIntervalMs: number
+  private _version: string
+  private _agentUuid: string
+  private _versionHash: string | undefined
+  private _functions: Record<string, Omit<RegisterFunctionSpec<any>, 'name'>> = {}
+  private _retoolApi: RetoolAPI
+  private _logger: Logger
+
+  /**
+   * Creates an instance of the RetoolRPC class.
+   */
+  constructor(config: RetoolRPCConfig) {
+    this._apiKey = config.apiToken
+    this._hostUrl = config.host.replace(/\/$/, '') // Remove trailing / from host
+    this._resourceId = config.resourceId
+    this._environmentName = config.environmentName || DEFAULT_ENVIRONMENT_NAME
+    this._pollingIntervalMs = config.pollingIntervalMs
+      ? Math.max(config.pollingIntervalMs, MINIMUM_POLLING_INTERVAL_MS)
+      : DEFAULT_POLLING_INTERVAL_MS
+    this._version = config.version || DEFAULT_VERSION
+    this._agentUuid = config.agentUuid || uuidv4()
+
+    this._retoolApi = new RetoolAPI({ hostUrl: this._hostUrl, apiKey: this._apiKey })
+    this._logger = new Logger({ logLevel: config.logLevel })
+
+    this._logger.debug('Retool RPC Configuration', {
+      apiKey: this._apiKey,
+      hostUrl: this._hostUrl,
+      resourceId: this._resourceId,
+      environmentName: this._environmentName,
+      agentUuid: this._agentUuid,
+      version: this._version,
+      pollingIntervalMs: this._pollingIntervalMs,
+    })
+  }
+
+  /**
+   * Asynchronously starts listening for incoming Retool function invocations.
+   */
+  async listen(): Promise<void> {
+    this._logger.info('Starting RPC agent')
+    const registerResult = await loopWithBackoff(this._pollingIntervalMs, this._logger, () => this.registerAgent())
+    if (registerResult === 'done') {
+      this._logger.info('Agent registered')
+      this._logger.info('Starting processing query')
+      loopWithBackoff(this._pollingIntervalMs, this._logger, () => this.fetchQueryAndExecute())
+    }
+  }
+
+  /**
+   * Registers a Retool function with the specified function definition.
+   */
+  register<TArgs extends Arguments>(spec: RegisterFunctionSpec<TArgs>): void {
+    this._functions[spec.name] = {
+      arguments: spec.arguments,
+      permissions: spec.permissions,
+      implementation: spec.implementation,
+    }
+  }
+
+  /**
+   * Executes a Retool function with the specified arguments and context.
+   */
+  async executeFunction(
+    functionName: string,
+    functionArguments: unknown,
+    context: RetoolContext,
+  ): Promise<{ result: any; arguments: Record<string, unknown> }> {
+    this._logger.info(`Executing function: ${functionName}, context: ${context}`)
+    if (functionName === '__testConnection__') {
+      return { result: this.testConnection(context), arguments: {} }
+    }
+
+    const fnSpec = this._functions[functionName]
+    if (!fnSpec) {
+      throw new FunctionNotFoundError(functionName)
+    }
+
+    const parsedArguments = parseFunctionArguments(functionArguments, fnSpec.arguments)
+    this._logger.debug('Parsed arguments: ', parsedArguments)
+
+    const result = await fnSpec.implementation(parsedArguments, context)
+
+    // Consider truncating large arguments
+    return { result, arguments: parsedArguments }
+  }
+
+  /**
+   * Tests the current connection to the Retool server.
+   */
+  private testConnection(context: RetoolContext): {
+    success: true
+    version: string
+    agentUuid: string
+    context: RetoolContext
+  } {
+    return {
+      success: true,
+      version: this._version,
+      agentUuid: this._agentUuid,
+      context,
+    }
+  }
+
+  /**
+   * Registers the agent with the Retool server.
+   */
+  private async registerAgent(): Promise<AgentServerStatus> {
+    const functionsMetadata: Record<string, Pick<RegisterFunctionSpec<any>, 'arguments' | 'permissions'>> = {}
+    for (const functionName in this._functions) {
+      functionsMetadata[functionName] = {
+        arguments: this._functions[functionName].arguments,
+        permissions: this._functions[functionName].permissions,
+      }
+    }
+
+    const registerAgentResponse = await this._retoolApi.registerAgent({
+      resourceId: this._resourceId,
+      environmentName: this._environmentName,
+      version: this._version,
+      agentUuid: this._agentUuid,
+      operations: functionsMetadata,
+    })
+
+    if (!registerAgentResponse.ok) {
+      if (isClientError(registerAgentResponse.status)) {
+        this._logger.error(
+          `Error registering agent: ${registerAgentResponse.status} ${await registerAgentResponse.text()}}`,
+        )
+
+        // client error, stop the client
+        return 'stop'
+      }
+
+      throw new Error(
+        `Error connecting to retool server: ${registerAgentResponse.status} ${await registerAgentResponse.text()}}`,
+      )
+    }
+
+    const { versionHash } = await registerAgentResponse.json()
+    this._versionHash = versionHash
+    this._logger.info(`Agent registered with versionHash: ${versionHash}`)
+
+    return 'done'
+  }
+
+  /**
+   * Fetches a query from the Retool server and executes it.
+   */
+  private async fetchQueryAndExecute(): Promise<AgentServerStatus> {
+    const pendingQueryFetch = await this._retoolApi.popQuery({
+      resourceId: this._resourceId,
+      environmentName: this._environmentName,
+      agentUuid: this._agentUuid,
+      versionHash: this._versionHash,
+    })
+
+    if (!pendingQueryFetch.ok) {
+      if (isClientError(pendingQueryFetch.status)) {
+        this._logger.error(`Error fetching query (${pendingQueryFetch.status}): ${await pendingQueryFetch.text()}`)
+        return 'stop'
+      }
+
+      throw new Error(`Server error when fetching query: ${pendingQueryFetch.status}. Retrying...`)
+    }
+
+    const { query } = await pendingQueryFetch.json()
+    if (query) {
+      this._logger.debug('Executing query', query) // This might contain sensitive information
+
+      const agentReceivedQueryAt = new Date().toISOString()
+
+      const queryUuid: string = query.queryUuid
+      const { method, parameters, context } = query.queryInfo
+
+      let status: 'success' | 'error'
+      let executionResponse: unknown = undefined
+      let executionArguments: Record<string, unknown> | undefined = undefined
+      let agentError: AgentServerError | undefined = undefined
+
+      try {
+        const executionResult = await this.executeFunction(method, parameters, context)
+        executionResponse = executionResult.result
+        executionArguments = executionResult.arguments
+        status = 'success'
+      } catch (err: unknown) {
+        agentError = createAgentServerError(err)
+        status = 'error'
+      }
+
+      const agentFinishedQueryAt = new Date().toISOString()
+
+      const updateQueryResponse = await this._retoolApi.postQueryResponse({
+        resourceId: this._resourceId,
+        environmentName: this._environmentName,
+        versionHash: this._versionHash,
+        agentUuid: this._agentUuid,
+        queryUuid,
+        status,
+        data: executionResponse,
+        metadata: {
+          packageLanguage: 'javascript',
+          packageVersion: RetoolRPCVersion,
+          agentReceivedQueryAt,
+          agentFinishedQueryAt,
+          parameters: executionArguments,
+        },
+        error: agentError,
+      })
+
+      this._logger.debug('Update query response status: ', updateQueryResponse.status, await updateQueryResponse.text())
+    }
+
+    return 'continue'
+  }
+
+  /**
+   * Export registerAgent as a protected method for testing purpose only. Lookup TestRetoolRPC for usage.
+   */
+  protected async protectedRegisterAgent() {
+    return this.registerAgent()
+  }
+
+  /**
+   * Export fetchQueryAndExecute as a protected method for testing purpose only. Lookup TestRetoolRPC for usage.
+   */
+  protected async protectedFetchQueryAndExecute() {
+    return this.fetchQueryAndExecute()
+  }
+}

package/src/types.ts

@@ -0,0 +1,114 @@
+/**
+ * Configuration options for the Retool RPC.
+ */
+export type RetoolRPCConfig = {
+  /** The host URL of the Retool instance. */
+  host: string
+  /** The API token for authenticating with the Retool instance. */
+  apiToken: string
+  /** The ID of the resource in Retool (UUID format). */
+  resourceId: string
+  /** The optional environment identifier. Defaults to `production` */
+  environmentName?: string
+  /**
+   * The version of the schema being used. If the resource enforce explicit versioning,
+   * this property must be set, and must be updated with each schema change.
+   * */
+  version?: string
+  /** The optional polling interval in milliseconds. Defaults to 1000. Minimum is 100. */
+  pollingIntervalMs?: number
+  /** The optional UUID of the agent. Will be automatically generated by default */
+  agentUuid?: string
+  /** The optional log level. */
+  logLevel?: 'debug' | 'info' | 'warn' | 'error'
+}
+
+/** Represents the type of the argument. Right now we are supporting only string, boolean, number, dict, and json. */
+export type ArgumentType = 'string' | 'boolean' | 'number' | 'dict' | 'json'
+
+/** Represents an argument definition for a Retool function. */
+export type Argument = {
+  /** The type of the argument. */
+  type: ArgumentType
+  /** Specifies whether the argument is expected to be an array. */
+  array?: boolean
+  /** The description of the argument. */
+  description?: string
+  /** Specifies whether the argument is required. */
+  required?: boolean
+}
+
+/** Recursive JSON type */
+export type Json = string | number | boolean | null | Json[] | { [property: string]: Json }
+
+/** Represents a map of argument name to argument type. */
+export type ArgumentTypeMap<T extends Argument> = T['type'] extends 'string'
+  ? string
+  : T['type'] extends 'number'
+  ? number
+  : T['type'] extends 'boolean'
+  ? boolean
+  : T['type'] extends 'dict'
+  ? Record<string, any>
+  : T['type'] extends 'json'
+  ? Json
+  : never
+
+/** Represents a map of argument names to argument definitions. */
+export type Arguments = Record<string, Argument>
+
+/** Represents a map of argument name to argument type. */
+export type TransformedArgument<TArg extends Argument> = TArg['array'] extends true
+  ? Array<ArgumentTypeMap<TArg>>
+  : ArgumentTypeMap<TArg>
+
+/** Represents a map of argument names to argument types. */
+export type TransformedArguments<TArgs extends Arguments> = {
+  [TArg in keyof TArgs]: TransformedArgument<TArgs[TArg]>
+}
+
+/** Represents the specification for registering a Retool function. */
+export type RegisterFunctionSpec<TArgs extends Arguments> = {
+  /** The name of the function. */
+  name: string
+  /** The arguments of the function. */
+  arguments: Pick<TArgs, keyof TArgs>
+  /** The implementation of the function. */
+  implementation: (args: TransformedArguments<TArgs>, context: RetoolContext) => Promise<any>
+  /** The permissions configuration for the function. */
+  permissions?: {
+    /** The list of group names that have permission to execute the function. */
+    groupNames?: string[]
+    /** The list of user emails that have permission to execute the function. */
+    userEmails?: string[]
+  }
+}
+
+/** Additional context information that is passed to the Retool function. */
+export type RetoolContext = {
+  /** Name of the user who is executing the function. */
+  userName?: string
+  /** Email address of the user who is executing the function. */
+  userEmail?: string
+  /** A list of group names of the user who is executing the function. */
+  userGroups?: string[]
+  /** The organization name of the user who is executing the function. */
+  organizationName?: string
+}
+
+/** Agent server error that is thrown by Retool RPC. */
+export type AgentServerError = {
+  /** The error name. */
+  name: string
+  /** The error message. */
+  message: string
+  /** The error stack track. */
+  stack?: string
+  /** The error status code. */
+  code?: number
+  /** The error additional details. */
+  details?: unknown
+}
+
+/** Represents the current status of a function execution. */
+export type AgentServerStatus = 'continue' | 'stop' | 'done'

package/src/utils/api.ts

@@ -0,0 +1,105 @@
+import fetch, { RequestInit } from 'node-fetch'
+
+import AbortControllerFallback from 'abort-controller'
+
+// AbortController was added in node v14.17.0 globally, but we need to polyfill it for older versions
+const AbortController = globalThis.AbortController || AbortControllerFallback
+
+import { AgentServerError } from '../types'
+import { RetoolRPCVersion } from '../version'
+
+const POLLING_TIMEOUT_MS = 5 * 1000 // 5 seconds
+
+type PopQueryRequest = {
+  resourceId: string
+  environmentName: string
+  agentUuid: string
+  versionHash: string | undefined
+}
+
+type RegisterAgentRequest = {
+  resourceId: string
+  environmentName: string
+  version: string
+  agentUuid: string
+  operations: Record<string, unknown>
+}
+
+type PostQueryResponseRequest = {
+  resourceId: string
+  environmentName: string
+  versionHash: string | undefined
+  agentUuid: string
+  queryUuid: string
+  metadata: {
+    packageVersion: string
+    packageLanguage: 'python' | 'javascript'
+    agentReceivedQueryAt: string
+    agentFinishedQueryAt: string
+    parameters: Record<string, unknown> | undefined
+  }
+  status: 'success' | 'error' // Should be a discriminated union instead
+  data: unknown
+  error: AgentServerError | undefined
+}
+
+export class RetoolAPI {
+  private _hostUrl: string
+  private _apiKey: string
+
+  constructor({ hostUrl, apiKey }: { hostUrl: string; apiKey: string }) {
+    this._hostUrl = hostUrl
+    this._apiKey = apiKey
+  }
+
+  async popQuery(options: PopQueryRequest) {
+    const abortController = new AbortController()
+    setTimeout(() => {
+      abortController.abort()
+    }, POLLING_TIMEOUT_MS)
+
+    try {
+      return await fetch(`${this._hostUrl}/api/v1/retoolrpc/popQuery`, {
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${this._apiKey}`,
+          'Content-Type': 'application/json',
+          'User-Agent': `RetoolRPC/${RetoolRPCVersion} (Javascript)`,
+        },
+        body: JSON.stringify(options),
+        // Had to cast to RequestInit['signal'] because of a bug in the types
+        // https://github.com/jasonkuhrt/graphql-request/issues/481
+        signal: abortController.signal as RequestInit['signal'],
+      })
+    } catch (error: any) {
+      if (abortController.signal.aborted) {
+        throw new Error(`Polling timeout after ${POLLING_TIMEOUT_MS}ms`)
+      }
+      throw error
+    }
+  }
+
+  async registerAgent(options: RegisterAgentRequest) {
+    return fetch(`${this._hostUrl}/api/v1/retoolrpc/registerAgent`, {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${this._apiKey}`,
+        'Content-Type': 'application/json',
+        'User-Agent': `RetoolRPC/${RetoolRPCVersion} (Javascript)`,
+      },
+      body: JSON.stringify(options),
+    })
+  }
+
+  async postQueryResponse(options: PostQueryResponseRequest) {
+    return fetch(`${this._hostUrl}/api/v1/retoolrpc/postQueryResponse`, {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${this._apiKey}`,
+        'Content-Type': 'application/json',
+        'User-Agent': `RetoolRPC/${RetoolRPCVersion} (Javascript)`,
+      },
+      body: JSON.stringify(options),
+    })
+  }
+}

package/src/utils/errors.ts

@@ -0,0 +1,53 @@
+import { AgentServerError } from '../types'
+
+const AGENT_SERVER_ERROR = 'AgentServerError'
+
+export function createAgentServerError(error: unknown): AgentServerError {
+  if (error instanceof Error) {
+    const agentError: AgentServerError = {
+      name: error.name,
+      message: error.message,
+      stack: error.stack,
+    }
+
+    if ('code' in error) {
+      if (typeof error.code === 'number') {
+        agentError.code = error.code
+      } else if (error.code === 'string') {
+        agentError.code = parseInt(error.code, 10)
+      }
+    }
+
+    if ('details' in error) {
+      agentError.details = error.details
+    }
+
+    return agentError
+  }
+
+  if (typeof error === 'string') {
+    return {
+      name: AGENT_SERVER_ERROR,
+      message: error,
+    }
+  }
+
+  return {
+    name: AGENT_SERVER_ERROR,
+    message: 'Unknown agent server error',
+  }
+}
+
+export class FunctionNotFoundError extends Error {
+  constructor(functionName: string) {
+    super(`Function "${functionName}" not found on remote agent server.`)
+    this.name = 'FunctionNotFoundError'
+  }
+}
+
+export class InvalidArgumentsError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = 'InvalidArgumentsError'
+  }
+}

package/src/utils/helpers.ts

@@ -0,0 +1,41 @@
+export function pick<T extends object, K extends keyof T>(obj: T, keys: K[]): Pick<T, K> {
+  const result: Partial<T> = {}
+
+  for (const key of keys) {
+    if (key in obj) {
+      result[key] = obj[key]
+    }
+  }
+
+  return result as Pick<T, K>
+}
+
+export function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value)
+}
+
+export function isFalsyArgumentValue(value: unknown): boolean {
+  return value === null || value === undefined || value === ''
+}
+
+export function isBooleanString(value: unknown): value is string {
+  if (typeof value === 'string') {
+    const lowercaseValue = value.toLowerCase()
+    return lowercaseValue === 'true' || lowercaseValue === 'false'
+  }
+
+  return false
+}
+
+export function isNumberString(value: unknown): value is string {
+  if (typeof value === 'string') {
+    // Use a regular expression to check if the string is a valid number
+    return /^-?\d+(\.\d+)?$/.test(value)
+  }
+
+  return false
+}
+
+export function isClientError(status: number): boolean {
+  return status >= 400 && status < 500
+}

package/src/utils/logger.ts

@@ -0,0 +1,49 @@
+const LOG_LEVELS = ['debug', 'info', 'warn', 'error'] as const
+type LogLevel = typeof LOG_LEVELS[number]
+
+const LOG_LEVEL_RANKINGS: Record<LogLevel, number> = {
+  debug: 0,
+  info: 1,
+  warn: 2,
+  error: 3,
+}
+
+type LoggerOptions = {
+  logLevel?: LogLevel
+}
+
+export class Logger {
+  private currentLogLevel: LogLevel
+
+  constructor(options: LoggerOptions) {
+    this.currentLogLevel = options.logLevel || 'info' // Default to 'info' if logLevel is not specified
+  }
+
+  private shouldLog(level: LogLevel) {
+    return LOG_LEVEL_RANKINGS[level] >= LOG_LEVEL_RANKINGS[this.currentLogLevel] && process.env.NODE_ENV !== 'test'
+  }
+
+  debug(...messages: any[]) {
+    if (this.shouldLog('debug')) {
+      console.log(...messages)
+    }
+  }
+
+  info(...messages: any[]) {
+    if (this.shouldLog('info')) {
+      console.log(...messages)
+    }
+  }
+
+  warn(...messages: any[]) {
+    if (this.shouldLog('warn')) {
+      console.log(...messages)
+    }
+  }
+
+  error(...messages: any[]) {
+    if (this.shouldLog('error')) {
+      console.log(...messages)
+    }
+  }
+}

package/src/utils/polling.ts

@@ -0,0 +1,43 @@
+import { AgentServerStatus } from '../types'
+import { Logger } from './logger'
+
+const CONNECTION_ERROR_INITIAL_TIMEOUT_MS = 50
+const CONNECTION_ERROR_RETRY_MAX_MS = 1000 * 60 * 10 // 10 minutes
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms)
+  })
+}
+
+export async function loopWithBackoff(
+  pollingIntervalMs: number,
+  logger: Logger,
+  callback: () => Promise<AgentServerStatus>,
+): Promise<AgentServerStatus> {
+  let delayTimeMs = CONNECTION_ERROR_INITIAL_TIMEOUT_MS
+  let lastLoopTimestamp = Date.now()
+
+  while (true) {
+    try {
+      const result = await callback()
+
+      const currentTimestamp = Date.now()
+      const loopDurationMs = currentTimestamp - lastLoopTimestamp
+      lastLoopTimestamp = currentTimestamp
+      logger.debug(
+        `Loop time: ${loopDurationMs}ms, delay time: ${delayTimeMs}ms, polling interval: ${pollingIntervalMs}ms`,
+      )
+
+      if (result !== 'continue') {
+        return result
+      }
+      await sleep(pollingIntervalMs)
+      delayTimeMs = Math.max(delayTimeMs / 2, CONNECTION_ERROR_INITIAL_TIMEOUT_MS)
+    } catch (err: unknown) {
+      logger.error('Error running RPC agent', err)
+      await sleep(delayTimeMs)
+      delayTimeMs = Math.min(delayTimeMs * 2, CONNECTION_ERROR_RETRY_MAX_MS)
+    }
+  }
+}