serverless-offline 8.8.0 → 9.1.0

package/src/lambda/handler-runner/docker-runner/DockerContainer.js

@@ -0,0 +1,417 @@
+import { createHash } from 'node:crypto'
+import { createWriteStream } from 'node:fs'
+import { readFile, unlink, writeFile } from 'node:fs/promises'
+import { platform } from 'node:os'
+import { dirname, join, sep } from 'node:path'
+import { log, progress } from '@serverless/utils/log.js'
+import aws from 'aws-sdk'
+import { execa } from 'execa'
+import { ensureDir, pathExists } from 'fs-extra'
+import jszip from 'jszip'
+import pRetry from 'p-retry'
+import DockerImage from './DockerImage.js'
+
+const { stringify } = JSON
+const { entries, hasOwn } = Object
+
+export default class DockerContainer {
+  #containerId = null
+
+  #dockerOptions = null
+
+  #env = null
+
+  #functionKey = null
+
+  #handler = null
+
+  #image = null
+
+  #imageNameTag = null
+
+  #lambda = null
+
+  #layers = null
+
+  #port = null
+
+  #provider = null
+
+  #runtime = null
+
+  #servicePath = null
+
+  constructor(
+    env,
+    functionKey,
+    handler,
+    runtime,
+    layers,
+    provider,
+    servicePath,
+    dockerOptions,
+  ) {
+    this.#dockerOptions = dockerOptions
+    this.#env = env
+    this.#functionKey = functionKey
+    this.#handler = handler
+    this.#imageNameTag = this.#baseImage(runtime)
+    this.#image = new DockerImage(this.#imageNameTag)
+    this.#layers = layers
+    this.#provider = provider
+    this.#runtime = runtime
+    this.#servicePath = servicePath
+  }
+
+  #baseImage(runtime) {
+    return `lambci/lambda:${runtime}`
+  }
+
+  async start(codeDir) {
+    await this.#image.pull()
+
+    log.debug('Run Docker container...')
+
+    let permissions = 'ro'
+
+    if (!this.#dockerOptions.readOnly) {
+      permissions = 'rw'
+    }
+    // https://github.com/serverless/serverless/blob/v1.57.0/lib/plugins/aws/invokeLocal/index.js#L291-L293
+    const dockerArgs = [
+      '-v',
+      `${codeDir}:/var/task:${permissions},delegated`,
+      '-p',
+      9001,
+      '-e',
+      'DOCKER_LAMBDA_STAY_OPEN=1', // API mode
+      '-e',
+      'DOCKER_LAMBDA_WATCH=1', // Watch mode
+    ]
+
+    if (this.#layers.length > 0) {
+      log.verbose(`Found layers, checking provider type`)
+
+      if (this.#provider.name.toLowerCase() !== 'aws') {
+        log.warning(
+          `Provider ${
+            this.#provider.name
+          } is Unsupported. Layers are only supported on aws.`,
+        )
+      } else {
+        let layerDir = this.#dockerOptions.layersDir
+
+        if (!layerDir) {
+          layerDir = join(this.#servicePath, '.serverless-offline', 'layers')
+        }
+
+        layerDir = join(layerDir, this.#getLayersSha256())
+
+        if (await pathExists(layerDir)) {
+          log.verbose(
+            `Layers already exist for this function. Skipping download.`,
+          )
+        } else {
+          const layers = []
+
+          log.verbose(`Storing layers at ${layerDir}`)
+
+          // Only initialise if we have layers, we're using AWS, and they don't already exist
+          this.#lambda = new aws.Lambda({
+            apiVersion: '2015-03-31',
+            region: this.#provider.region,
+          })
+
+          log.verbose(`Getting layers`)
+
+          for (const layerArn of this.#layers) {
+            layers.push(this.#downloadLayer(layerArn, layerDir))
+          }
+
+          await Promise.all(layers)
+        }
+
+        if (
+          this.#dockerOptions.hostServicePath &&
+          layerDir.startsWith(this.#servicePath)
+        ) {
+          layerDir = layerDir.replace(
+            this.#servicePath,
+            this.#dockerOptions.hostServicePath,
+          )
+        }
+        dockerArgs.push('-v', `${layerDir}:/opt:ro,delegated`)
+      }
+    }
+
+    entries(this.#env).forEach(([key, value]) => {
+      dockerArgs.push('-e', `${key}=${value}`)
+    })
+
+    if (platform() === 'linux') {
+      // Add `host.docker.internal` DNS name to access host from inside the container
+      // https://github.com/docker/for-linux/issues/264
+      const gatewayIp = await this.#getBridgeGatewayIp()
+      if (gatewayIp) {
+        dockerArgs.push('--add-host', `host.docker.internal:${gatewayIp}`)
+      }
+    }
+
+    if (this.#dockerOptions.network) {
+      dockerArgs.push('--network', this.#dockerOptions.network)
+    }
+
+    const { stdout: containerId } = await execa('docker', [
+      'create',
+      ...dockerArgs,
+      this.#imageNameTag,
+      this.#handler,
+    ])
+
+    const dockerStart = execa('docker', ['start', '-a', containerId], {
+      all: true,
+    })
+
+    await new Promise((resolve, reject) => {
+      dockerStart.all.on('data', (data) => {
+        const str = String(data)
+        log.error(str)
+
+        if (str.includes('Lambda API listening on port')) {
+          resolve()
+        }
+      })
+
+      dockerStart.on('error', (err) => {
+        reject(err)
+      })
+    })
+
+    // parse `docker port` output and get the container port
+    let containerPort
+    const { stdout: dockerPortOutput } = await execa('docker', [
+      'port',
+      containerId,
+    ])
+    // NOTE: `docker port` may output multiple lines.
+    //
+    // e.g.:
+    // 9001/tcp -> 0.0.0.0:49153
+    // 9001/tcp -> :::49153
+    //
+    // Parse each line until it finds the mapped port.
+    for (const line of dockerPortOutput.split('\n')) {
+      const result = line.match(/^9001\/tcp -> (.*):(\d+)$/)
+      if (result && result.length > 2) {
+        ;[, , containerPort] = result
+        break
+      }
+    }
+    if (!containerPort) {
+      throw new Error('Failed to get container port')
+    }
+
+    this.#containerId = containerId
+    this.#port = containerPort
+
+    await pRetry(() => this.#ping(), {
+      // default,
+      factor: 2,
+      // milliseconds
+      minTimeout: 10,
+      // default
+      retries: 10,
+    })
+  }
+
+  async #downloadLayer(layerArn, layerDir) {
+    const layerName = layerArn.split(':layer:')[1]
+    const layerZipFile = `${layerDir}/${layerName}.zip`
+    const layerProgress = progress.get(`layer-${layerName}`)
+
+    log.verbose(`[${layerName}] ARN: ${layerArn}`)
+
+    const params = {
+      Arn: layerArn,
+    }
+
+    log.verbose(`[${layerName}] Getting Info`)
+    layerProgress.notice(`Retrieving "${layerName}": Getting info`)
+
+    try {
+      let layer = null
+
+      try {
+        layer = await this.#lambda.getLayerVersionByArn(params).promise()
+      } catch (err) {
+        log.warning(`[${layerName}] ${err.code}: ${err.message}`)
+
+        return
+      }
+
+      if (
+        hasOwn(layer, 'CompatibleRuntimes') &&
+        !layer.CompatibleRuntimes.includes(this.#runtime)
+      ) {
+        log.warning(
+          `[${layerName}] Layer is not compatible with ${
+            this.#runtime
+          } runtime`,
+        )
+
+        return
+      }
+
+      const layerUrl = layer.Content.Location
+      // const layerSha = layer.Content.CodeSha256
+
+      const layerSize = layer.Content.CodeSize
+
+      await ensureDir(layerDir)
+
+      log.verbose(
+        `Retrieving "${layerName}": Downloading ${this.#formatBytes(
+          layerSize,
+        )}...`,
+      )
+      layerProgress.notice(
+        `Retrieving "${layerName}": Downloading ${this.#formatBytes(
+          layerSize,
+        )}`,
+      )
+
+      const res = await fetch(layerUrl, {
+        method: 'get',
+      })
+
+      if (!res.ok) {
+        log.warning(
+          `[${layerName}] Failed to fetch from ${layerUrl} with ${res.statusText}`,
+        )
+
+        return
+      }
+
+      const fileStream = createWriteStream(layerZipFile)
+
+      await new Promise((resolve, reject) => {
+        res.body.pipe(fileStream)
+        res.body.on('error', (err) => {
+          reject(err)
+        })
+        fileStream.on('finish', () => {
+          resolve()
+        })
+      })
+
+      log.verbose(`Retrieving "${layerName}": Unzipping to .layers directory`)
+      layerProgress.notice(
+        `Retrieving "${layerName}": Unzipping to .layers directory`,
+      )
+
+      const data = await readFile(layerZipFile)
+      const zip = await jszip.loadAsync(data)
+
+      await Promise.all(
+        entries(zip.files).map(async ([filename, jsZipObj]) => {
+          const fileData = await jsZipObj.async('nodebuffer')
+          if (filename.endsWith(sep)) {
+            return Promise.resolve()
+          }
+          await ensureDir(join(layerDir, dirname(filename)))
+          return writeFile(join(layerDir, filename), fileData, {
+            mode: zip.files[filename].unixPermissions,
+          })
+        }),
+      )
+
+      log.verbose(`[${layerName}] Removing zip file`)
+
+      await unlink(layerZipFile)
+    } finally {
+      layerProgress.remove()
+    }
+  }
+
+  async #getBridgeGatewayIp() {
+    let gateway
+    try {
+      ;({ stdout: gateway } = await execa('docker', [
+        'network',
+        'inspect',
+        'bridge',
+        '--format',
+        '{{(index .IPAM.Config 0).Gateway}}',
+      ]))
+    } catch (err) {
+      log.error(err.stderr)
+
+      throw err
+    }
+    return gateway.split('/')[0]
+  }
+
+  async #ping() {
+    const url = `http://${this.#dockerOptions.host}:${
+      this.#port
+    }/2018-06-01/ping`
+    const res = await fetch(url)
+
+    if (!res.ok) {
+      throw new Error(`Failed to fetch from ${url} with ${res.statusText}`)
+    }
+
+    return res.text()
+  }
+
+  async request(event) {
+    const url = `http://${this.#dockerOptions.host}:${
+      this.#port
+    }/2015-03-31/functions/${this.#functionKey}/invocations`
+
+    const res = await fetch(url, {
+      body: stringify(event),
+      headers: { 'Content-Type': 'application/json' },
+      method: 'post',
+    })
+
+    if (!res.ok) {
+      throw new Error(`Failed to fetch from ${url} with ${res.statusText}`)
+    }
+
+    return res.json()
+  }
+
+  async stop() {
+    if (this.#containerId) {
+      try {
+        await execa('docker', ['stop', this.#containerId])
+        await execa('docker', ['rm', this.#containerId])
+      } catch (err) {
+        log.error(err.stderr)
+
+        throw err
+      }
+    }
+  }
+
+  #formatBytes(bytes, decimals = 2) {
+    if (bytes === 0) return '0 Bytes'
+
+    const k = 1024
+    const dm = decimals < 0 ? 0 : decimals
+    const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
+
+    const i = Math.floor(Math.log(bytes) / Math.log(k))
+
+    return `${parseFloat((bytes / k ** i).toFixed(dm))} ${sizes[i]}`
+  }
+
+  #getLayersSha256() {
+    return createHash('sha256').update(stringify(this.#layers)).digest('hex')
+  }
+
+  get isRunning() {
+    return this.#containerId !== null && this.#port !== null
+  }
+}

package/src/lambda/handler-runner/docker-runner/DockerImage.js

@@ -0,0 +1,35 @@
+import { log } from '@serverless/utils/log.js'
+import { execa } from 'execa'
+// TODO FIXME eslint plugin import bug, or not supporting package.json "expprts" field?
+// eslint-disable-next-line import/no-unresolved
+import promiseMemoize from 'p-memoize'
+
+export default class DockerImage {
+  #imageNameTag = null
+
+  static #memoizedPull = promiseMemoize(DockerImage.#pullImage)
+
+  constructor(imageNameTag) {
+    this.#imageNameTag = imageNameTag
+  }
+
+  static async #pullImage(imageNameTag) {
+    log.debug(`Downloading base Docker image... (${imageNameTag})`)
+
+    try {
+      await execa('docker', [
+        'pull',
+        '--disable-content-trust=false',
+        imageNameTag,
+      ])
+    } catch (err) {
+      log.error(err.stderr)
+
+      throw err
+    }
+  }
+
+  async pull() {
+    return DockerImage.#memoizedPull(this.#imageNameTag)
+  }
+}

package/src/lambda/handler-runner/docker-runner/DockerRunner.js

@@ -0,0 +1,63 @@
+import DockerContainer from './DockerContainer.js'
+import { checkDockerDaemon } from '../../../utils/index.js'
+
+export default class DockerRunner {
+  #codeDir = null
+
+  #container = null
+
+  constructor(funOptions, env, dockerOptions) {
+    const {
+      codeDir,
+      functionKey,
+      handler,
+      runtime,
+      layers,
+      provider,
+      servicePath,
+    } = funOptions
+
+    this.#codeDir = codeDir
+
+    if (
+      dockerOptions.hostServicePath &&
+      this.#codeDir.startsWith(servicePath)
+    ) {
+      this.#codeDir = this.#codeDir.replace(
+        servicePath,
+        dockerOptions.hostServicePath,
+      )
+    }
+
+    this.#container = new DockerContainer(
+      env,
+      functionKey,
+      handler,
+      runtime,
+      layers,
+      provider,
+      servicePath,
+      dockerOptions,
+    )
+  }
+
+  cleanup() {
+    if (this.#container) {
+      return this.#container.stop()
+    }
+
+    return undefined
+  }
+
+  // context will be generated in container
+  async run(event) {
+    // FIXME TODO this should run only once -> static private
+    await checkDockerDaemon()
+
+    if (!this.#container.isRunning) {
+      await this.#container.start(this.#codeDir)
+    }
+
+    return this.#container.request(event)
+  }
+}

package/src/lambda/handler-runner/docker-runner/index.js

@@ -0,0 +1 @@
+export { default } from './DockerRunner.js'

package/src/lambda/handler-runner/go-runner/GoRunner.js

@@ -0,0 +1,167 @@
+import { mkdir, readFile, rm, rmdir, writeFile } from 'node:fs/promises'
+import { EOL } from 'node:os'
+import { sep, resolve, parse as pathParse } from 'node:path'
+import process, { chdir, cwd } from 'node:process'
+import { log } from '@serverless/utils/log.js'
+import { execa, execaSync } from 'execa'
+
+const { parse, stringify } = JSON
+
+const PAYLOAD_IDENTIFIER = 'offline_payload'
+
+export default class GoRunner {
+  #codeDir = null
+
+  #env = null
+
+  #goEnv = null
+
+  #handlerPath = null
+
+  #tmpPath = null
+
+  #tmpFile = null
+
+  constructor(funOptions, env) {
+    const { handlerPath, codeDir } = funOptions
+
+    this.#codeDir = codeDir
+    this.#env = env
+    this.#handlerPath = handlerPath
+  }
+
+  async cleanup() {
+    try {
+      // refresh go.mod
+      await rm(this.#tmpFile)
+      execaSync('go', ['mod', 'tidy'])
+      await rmdir(this.#tmpPath, { recursive: true })
+    } catch {
+      // @ignore
+    }
+
+    this.#tmpFile = null
+    this.#tmpPath = null
+  }
+
+  #parsePayload(value) {
+    const logs = []
+    let payload
+
+    for (const item of value.split(EOL)) {
+      if (item.indexOf(PAYLOAD_IDENTIFIER) === -1) {
+        logs.push(item)
+      } else if (item.indexOf(PAYLOAD_IDENTIFIER) !== -1) {
+        try {
+          const {
+            offline_payload: { success, error },
+          } = parse(item)
+
+          if (success) {
+            payload = success
+          } else if (error) {
+            payload = error
+          }
+        } catch {
+          // @ignore
+        }
+      }
+    }
+
+    // Log to console in case engineers want to see the rest of the info
+    log(logs.join(EOL))
+
+    return payload
+  }
+
+  async run(event, context) {
+    const { dir } = pathParse(this.#handlerPath)
+    const handlerCodeRoot = dir.split(sep).slice(0, -1).join(sep)
+    const handlerCode = await readFile(`${this.#handlerPath}.go`, 'utf8')
+
+    this.#tmpPath = resolve(handlerCodeRoot, 'tmp')
+    this.#tmpFile = resolve(this.#tmpPath, 'main.go')
+
+    const out = handlerCode.replace(
+      '"github.com/aws/aws-lambda-go/lambda"',
+      'lambda "github.com/icarus-sullivan/mock-lambda"',
+    )
+
+    try {
+      await mkdir(this.#tmpPath, { recursive: true })
+    } catch {
+      // @ignore
+    }
+
+    try {
+      await writeFile(this.#tmpFile, out, 'utf8')
+    } catch {
+      // @ignore
+    }
+
+    // Get go env to run this locally
+    if (!this.#goEnv) {
+      const goEnvResponse = await execa('go', ['env'], {
+        encoding: 'utf-8',
+        stdio: 'pipe',
+      })
+
+      const goEnvString = goEnvResponse.stdout || goEnvResponse.stderr
+      this.#goEnv = goEnvString.split(EOL).reduce((a, b) => {
+        const [k, v] = b.split('="')
+        // eslint-disable-next-line no-param-reassign
+        a[k] = v ? v.slice(0, -1) : ''
+        return a
+      }, {})
+    }
+
+    // Remove our root, since we want to invoke go relatively
+    const cwdPath = `${this.#tmpFile}`.replace(`${cwd()}${sep}`, '')
+
+    try {
+      chdir(cwdPath.substring(0, cwdPath.indexOf('main.go')))
+
+      // Make sure we have the mock-lambda runner
+      execaSync('go', ['get', 'github.com/icarus-sullivan/mock-lambda@e065469'])
+      execaSync('go', ['build'])
+    } catch {
+      // @ignore
+    }
+
+    const { stdout, stderr } = await execa(`./tmp`, {
+      encoding: 'utf-8',
+      env: {
+        ...this.#env,
+        ...this.#goEnv,
+        AWS_LAMBDA_FUNCTION_MEMORY_SIZE: context.memoryLimitInMB,
+        AWS_LAMBDA_FUNCTION_NAME: context.functionName,
+        AWS_LAMBDA_FUNCTION_VERSION: context.functionVersion,
+        AWS_LAMBDA_LOG_GROUP_NAME: context.logGroupName,
+        AWS_LAMBDA_LOG_STREAM_NAME: context.logStreamName,
+        IS_LAMBDA_AUTHORIZER:
+          event.type === 'REQUEST' || event.type === 'TOKEN',
+        IS_LAMBDA_REQUEST_AUTHORIZER: event.type === 'REQUEST',
+        IS_LAMBDA_TOKEN_AUTHORIZER: event.type === 'TOKEN',
+        LAMBDA_CONTEXT: stringify(context),
+        LAMBDA_EVENT: stringify(event),
+        LAMBDA_TEST_EVENT: `${event}`,
+        PATH: process.env.PATH,
+      },
+      stdio: 'pipe',
+    })
+
+    await this.cleanup()
+
+    if (stderr) {
+      return stderr
+    }
+
+    try {
+      chdir(this.#codeDir)
+    } catch {
+      // @ignore
+    }
+
+    return this.#parsePayload(stdout)
+  }
+}

package/src/lambda/handler-runner/go-runner/index.js

@@ -0,0 +1 @@
+export { default } from './GoRunner.js'