serverless-offline 8.8.0 → 9.1.0

package/src/lambda/LambdaFunction.js

@@ -0,0 +1,309 @@
+import { readFile, writeFile } from 'node:fs/promises'
+import { dirname, join, resolve } from 'node:path'
+import process from 'node:process'
+import { performance } from 'node:perf_hooks'
+import { log } from '@serverless/utils/log.js'
+import { emptyDir, ensureDir, remove } from 'fs-extra'
+import jszip from 'jszip'
+import HandlerRunner from './handler-runner/index.js'
+import LambdaContext from './LambdaContext.js'
+import {
+  DEFAULT_LAMBDA_MEMORY_SIZE,
+  DEFAULT_LAMBDA_RUNTIME,
+  DEFAULT_LAMBDA_TIMEOUT,
+  supportedRuntimes,
+} from '../config/index.js'
+import { createUniqueId, splitHandlerPathAndName } from '../utils/index.js'
+
+const { entries, fromEntries } = Object
+const { ceil } = Math
+
+export default class LambdaFunction {
+  #artifact = null
+
+  #clientContext = null
+
+  #codeDir = null
+
+  #event = null
+
+  #executionTimeEnded = null
+
+  #executionTimeStarted = null
+
+  #functionKey = null
+
+  #functionName = null
+
+  #handler = null
+
+  #handlerRunner = null
+
+  #idleTimeStarted = null
+
+  #initialized = false
+
+  #lambdaContext = null
+
+  #lambdaDir = null
+
+  #memorySize = null
+
+  #region = null
+
+  #runtime = null
+
+  #timeout = null
+
+  status = 'IDLE' // can be 'BUSY' or 'IDLE'
+
+  constructor(functionKey, functionDefinition, serverless, options) {
+    const {
+      service,
+      config: { serverlessPath, servicePath },
+      service: { provider, package: servicePackage = {} },
+    } = serverless
+
+    // TEMP options.location, for compatibility with serverless-webpack:
+    // https://github.com/dherault/serverless-offline/issues/787
+    // TODO FIXME look into better way to work with serverless-webpack
+    const servicepath = resolve(servicePath, options.location || '')
+
+    const { handler, name, package: functionPackage = {} } = functionDefinition
+    const [handlerPath, handlerName] = splitHandlerPathAndName(handler)
+
+    const memorySize =
+      functionDefinition.memorySize ||
+      provider.memorySize ||
+      DEFAULT_LAMBDA_MEMORY_SIZE
+
+    const runtime =
+      functionDefinition.runtime || provider.runtime || DEFAULT_LAMBDA_RUNTIME
+
+    const timeout =
+      (functionDefinition.timeout ||
+        provider.timeout ||
+        DEFAULT_LAMBDA_TIMEOUT) * 1000
+
+    // this._executionTimeout = null
+    this.#functionKey = functionKey
+    this.#functionName = name
+    this.#handler = handler
+    this.#memorySize = memorySize
+    this.#region = provider.region
+    this.#runtime = runtime
+    this.#timeout = timeout
+
+    this.#verifySupportedRuntime()
+
+    const env = {
+      ...(options.localEnvironment
+        ? process.env
+        : // we always copy all AWS_xxxx environment variables over from local env
+          fromEntries(
+            entries(process.env).filter(([key]) => key.startsWith('AWS_')),
+          )),
+      ...this.#getAwsEnvVars(),
+      ...provider.environment,
+      ...functionDefinition.environment,
+      IS_OFFLINE: 'true',
+    }
+
+    this.#artifact = functionDefinition.package?.artifact
+
+    if (!this.#artifact) {
+      this.#artifact = service.package?.artifact
+    }
+
+    if (this.#artifact) {
+      // lambda directory contains code and layers
+      this.#lambdaDir = join(
+        servicepath,
+        '.serverless-offline',
+        'services',
+        service.service,
+        functionKey,
+        createUniqueId(),
+      )
+    }
+
+    this.#codeDir = this.#lambdaDir
+      ? resolve(this.#lambdaDir, 'code')
+      : servicepath
+
+    // TEMP
+    const funOptions = {
+      codeDir: this.#codeDir,
+      functionKey,
+      functionName: name,
+      functionPackage: functionPackage.artifact
+        ? resolve(servicepath, functionPackage.artifact)
+        : undefined,
+      handler,
+      handlerName,
+      handlerPath: resolve(this.#codeDir, handlerPath),
+      layers: functionDefinition.layers || [],
+      provider,
+      runtime,
+      serverlessPath,
+      servicePackage: servicePackage.artifact
+        ? resolve(servicepath, servicePackage.artifact)
+        : undefined,
+      servicePath: servicepath,
+      timeout,
+    }
+
+    this.#handlerRunner = new HandlerRunner(funOptions, options, env)
+    this.#lambdaContext = new LambdaContext(name, memorySize)
+  }
+
+  #startExecutionTimer() {
+    this.#executionTimeStarted = performance.now()
+    // this._executionTimeout = this.#executionTimeStarted + this.#timeout * 1000
+  }
+
+  #stopExecutionTimer() {
+    this.#executionTimeEnded = performance.now()
+  }
+
+  #startIdleTimer() {
+    this.#idleTimeStarted = performance.now()
+  }
+
+  #verifySupportedRuntime() {
+    // print message but keep working (don't error out or exit process)
+    if (!supportedRuntimes.has(this.#runtime)) {
+      log.warning()
+      log.warning(
+        `Warning: found unsupported runtime '${this.#runtime}' for function '${
+          this.#functionKey
+        }'`,
+      )
+    }
+  }
+
+  // based on:
+  // https://github.com/serverless/serverless/blob/v1.50.0/lib/plugins/aws/invokeLocal/index.js#L108
+  #getAwsEnvVars() {
+    return {
+      _HANDLER: this.#handler,
+      AWS_DEFAULT_REGION: this.#region,
+      AWS_LAMBDA_FUNCTION_MEMORY_SIZE: this.#memorySize,
+      AWS_LAMBDA_FUNCTION_NAME: this.#functionName,
+      AWS_LAMBDA_FUNCTION_VERSION: '$LATEST',
+      // https://github.com/serverless/serverless/blob/v1.50.0/lib/plugins/aws/lib/naming.js#L123
+      AWS_LAMBDA_LOG_GROUP_NAME: `/aws/lambda/${this.#functionName}`,
+      AWS_LAMBDA_LOG_STREAM_NAME:
+        '2016/12/02/[$LATEST]f77ff5e4026c45bda9a9ebcec6bc9cad',
+      AWS_REGION: this.#region,
+      LAMBDA_RUNTIME_DIR: '/var/runtime',
+      LAMBDA_TASK_ROOT: '/var/task',
+      LANG: 'en_US.UTF-8',
+      LD_LIBRARY_PATH:
+        '/usr/local/lib64/node-v4.3.x/lib:/lib64:/usr/lib64:/var/runtime:/var/runtime/lib:/var/task:/var/task/lib:/opt/lib',
+      NODE_PATH: '/var/runtime:/var/task:/var/runtime/node_modules',
+    }
+  }
+
+  setClientContext(clientContext) {
+    this.#clientContext = clientContext
+  }
+
+  setEvent(event) {
+    this.#event = event
+  }
+
+  // () => Promise<void>
+  async cleanup() {
+    // TODO console.log('lambda cleanup')
+    await this.#handlerRunner.cleanup()
+    if (this.#lambdaDir) {
+      await remove(this.#lambdaDir)
+    }
+  }
+
+  #executionTimeInMillis() {
+    return this.#executionTimeEnded - this.#executionTimeStarted
+  }
+
+  // round up to the nearest ms
+  #billedExecutionTimeInMillis() {
+    return ceil(this.#executionTimeEnded - this.#executionTimeStarted)
+  }
+
+  // extractArtifact, loosely based on:
+  // https://github.com/serverless/serverless/blob/v1.57.0/lib/plugins/aws/invokeLocal/index.js#L312
+  async #extractArtifact() {
+    if (!this.#artifact) {
+      return
+    }
+
+    await emptyDir(this.#codeDir)
+
+    const data = await readFile(this.#artifact)
+    const zip = await jszip.loadAsync(data)
+
+    await Promise.all(
+      entries(zip.files).map(async ([filename, jsZipObj]) => {
+        const fileData = await jsZipObj.async('nodebuffer')
+        if (filename.endsWith('/')) {
+          return Promise.resolve()
+        }
+        await ensureDir(join(this.#codeDir, dirname(filename)))
+        return writeFile(join(this.#codeDir, filename), fileData, {
+          mode: jsZipObj.unixPermissions,
+        })
+      }),
+    )
+  }
+
+  async #initialize() {
+    await this.#extractArtifact()
+    this.#initialized = true
+  }
+
+  get idleTimeInMinutes() {
+    return (performance.now() - this.#idleTimeStarted) / 1000 / 60
+  }
+
+  get functionName() {
+    return this.#functionName
+  }
+
+  async runHandler() {
+    this.status = 'BUSY'
+
+    if (!this.#initialized) {
+      await this.#initialize()
+    }
+
+    const requestId = createUniqueId()
+
+    this.#lambdaContext.setRequestId(requestId)
+    this.#lambdaContext.setClientContext(this.#clientContext)
+
+    const context = this.#lambdaContext.create()
+
+    this.#startExecutionTimer()
+
+    const result = await this.#handlerRunner.run(this.#event, context)
+
+    this.#stopExecutionTimer()
+
+    // TEMP TODO FIXME find better solution
+    if (!this.#handlerRunner.isDockerRunner()) {
+      log.notice(
+        `(λ: ${
+          this.#functionKey
+        }) RequestId: ${requestId}  Duration: ${this.#executionTimeInMillis().toFixed(
+          2,
+        )} ms  Billed Duration: ${this.#billedExecutionTimeInMillis()} ms`,
+      )
+    }
+
+    this.status = 'IDLE'
+    this.#startIdleTimer()
+
+    return result
+  }
+}

package/src/lambda/LambdaFunctionPool.js

@@ -0,0 +1,109 @@
+import LambdaFunction from './LambdaFunction.js'
+
+export default class LambdaFunctionPool {
+  #options = null
+
+  #pool = new Map()
+
+  #serverless = null
+
+  #timerRef = null
+
+  constructor(serverless, options) {
+    this.#options = options
+    this.#serverless = serverless
+  }
+
+  start() {
+    // start cleaner
+    this.#startCleanTimer()
+  }
+
+  #startCleanTimer() {
+    // NOTE: don't use setInterval, as it would schedule always a new run,
+    // regardless of function processing time and e.g. user action (debugging)
+    this.#timerRef = setTimeout(() => {
+      // console.log('run cleanup')
+      this.#pool.forEach((lambdaFunctions) => {
+        lambdaFunctions.forEach((lambdaFunction) => {
+          const { idleTimeInMinutes, status } = lambdaFunction
+          // console.log(idleTimeInMinutes, status)
+
+          if (
+            status === 'IDLE' &&
+            idleTimeInMinutes >=
+              this.#options.functionCleanupIdleTimeSeconds / 60
+          ) {
+            // console.log(`removed Lambda Function ${lambdaFunction.functionName}`)
+            lambdaFunction.cleanup()
+            lambdaFunctions.delete(lambdaFunction)
+          }
+        })
+      })
+
+      // schedule new timer
+      this.#startCleanTimer()
+    }, (this.#options.functionCleanupIdleTimeSeconds * 1000) / 2)
+  }
+
+  #cleanupPool() {
+    const wait = []
+
+    this.#pool.forEach((lambdaFunctions) => {
+      lambdaFunctions.forEach((lambdaFunction) => {
+        // collect promises
+        wait.push(lambdaFunction.cleanup())
+        lambdaFunctions.delete(lambdaFunction)
+      })
+    })
+
+    return Promise.all(wait)
+  }
+
+  // TODO make sure to call this
+  async cleanup() {
+    clearTimeout(this.#timerRef)
+
+    return this.#cleanupPool()
+  }
+
+  get(functionKey, functionDefinition) {
+    const lambdaFunctions = this.#pool.get(functionKey)
+    let lambdaFunction
+
+    // we don't have any instances
+    if (lambdaFunctions == null) {
+      lambdaFunction = new LambdaFunction(
+        functionKey,
+        functionDefinition,
+        this.#serverless,
+        this.#options,
+      )
+      this.#pool.set(functionKey, new Set([lambdaFunction]))
+
+      return lambdaFunction
+    }
+
+    if (!this.#options.reloadHandler) {
+      // find any IDLE
+      lambdaFunction = Array.from(lambdaFunctions).find(
+        ({ status }) => status === 'IDLE',
+      )
+
+      if (lambdaFunction != null) {
+        return lambdaFunction
+      }
+    }
+
+    // we don't have any IDLE instances
+    lambdaFunction = new LambdaFunction(
+      functionKey,
+      functionDefinition,
+      this.#serverless,
+      this.#options,
+    )
+    lambdaFunctions.add(lambdaFunction)
+
+    return lambdaFunction
+  }
+}

package/src/lambda/__tests__/LambdaContext.test.js

@@ -0,0 +1,30 @@
+import assert from 'node:assert'
+import LambdaContext from '../LambdaContext.js'
+
+describe('LambdaContext', () => {
+  it('should create LambdaContext with correct values', () => {
+    const functionName = 'foo'
+    const memorySize = 512
+    const requestId = 'abc123'
+
+    const lambdaContext = new LambdaContext(functionName, memorySize)
+    lambdaContext.setRequestId(requestId)
+    const context = lambdaContext.create()
+
+    const expected = {
+      awsRequestId: 'abc123',
+      callbackWaitsForEmptyEventLoop: true,
+      clientContext: undefined,
+      functionName: 'foo',
+      functionVersion: '$LATEST',
+      identity: undefined,
+      invokedFunctionArn: 'offline_invokedFunctionArn_for_foo',
+      logGroupName: 'offline_logGroupName_for_foo',
+      logStreamName: 'offline_logStreamName_for_foo',
+      memoryLimitInMB: '512',
+    }
+
+    // expect(context).toEqual(expected)
+    assert.deepEqual(context, expected)
+  })
+})

package/src/lambda/__tests__/LambdaFunction.test.js

@@ -0,0 +1,196 @@
+import assert from 'node:assert'
+import { dirname, resolve } from 'node:path'
+// import { performance } from 'node:perf_hooks'
+import { fileURLToPath } from 'node:url'
+import LambdaFunction from '../LambdaFunction.js'
+import { DEFAULT_LAMBDA_TIMEOUT } from '../../config/index.js'
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
+
+describe('LambdaFunction', () => {
+  const functionName = 'foo'
+
+  const serverless = {
+    config: {
+      serverlessPath: '',
+      servicePath: resolve(__dirname),
+    },
+    service: {
+      provider: {
+        runtime: 'nodejs12.x',
+      },
+    },
+  }
+
+  describe('Handler tests', () => {
+    ;[
+      {
+        description: 'should return result when handler is context.done',
+        expected: 'foo',
+        handler: 'fixtures/lambdaFunction.fixture.contextDoneHandler',
+      },
+      {
+        description:
+          'should return result when handler is context.done which is deferred',
+        expected: 'foo',
+        handler: 'fixtures/lambdaFunction.fixture.contextDoneHandlerDeferred',
+      },
+      {
+        description: 'should return result when handler is context.succeed',
+        expected: 'foo',
+        handler: 'fixtures/lambdaFunction.fixture.contextSucceedHandler',
+      },
+      {
+        description:
+          'should return result when handler is context.succeed which is deferred',
+        expected: 'foo',
+        handler:
+          'fixtures/lambdaFunction.fixture.contextSucceedHandlerDeferred',
+      },
+      {
+        description: 'should return result when handler is a callback',
+        expected: 'foo',
+        handler: 'fixtures/lambdaFunction.fixture.callbackHandler',
+      },
+      {
+        description:
+          'should return result when handler is a callback which is deferred',
+        expected: 'foo',
+        handler: 'fixtures/lambdaFunction.fixture.callbackHandlerDeferred',
+      },
+      {
+        description: 'should return result when handler returns a promise',
+        expected: 'foo',
+        handler: 'fixtures/lambdaFunction.fixture.promiseHandler',
+      },
+      {
+        description:
+          'should return result when handler returns a promise which is deferred',
+        expected: 'foo',
+        handler: 'fixtures/lambdaFunction.fixture.promiseHandlerDeferred',
+      },
+      {
+        description: 'should return result when handler is an async function',
+        expected: 'foo',
+        handler: 'fixtures/lambdaFunction.fixture.asyncFunctionHandler',
+      },
+      // NOTE: mix and matching of callbacks and promises is not recommended,
+      // nonetheless, we test some of the behaviour to match AWS execution precedence
+      {
+        description:
+          'should return result when handler returns a callback but defines a callback parameter',
+        expected: 'Hello Promise!',
+        handler:
+          'fixtures/lambdaFunction.fixture.promiseWithDefinedCallbackHandler',
+      },
+      {
+        description:
+          'should return result when handler calls context.succeed and context.done',
+        expected: 'Hello Context.succeed!',
+        handler:
+          'fixtures/lambdaFunction.fixture.contextSucceedWithContextDoneHandler',
+      },
+      {
+        description:
+          'should return result when handler calls callback and context.done',
+        expected: 'Hello Callback!',
+        handler:
+          'fixtures/lambdaFunction.fixture.callbackWithContextDoneHandler',
+      },
+      {
+        description:
+          'should return result when handler calls callback and returns Promise',
+        expected: 'Hello Callback!',
+        handler: 'fixtures/lambdaFunction.fixture.callbackWithPromiseHandler',
+      },
+      {
+        description:
+          'should return result when handler calls callback inside returned Promise',
+        expected: 'Hello Callback!',
+        handler: 'fixtures/lambdaFunction.fixture.callbackInsidePromiseHandler',
+      },
+    ].forEach(({ description, expected, handler }) => {
+      it(description, async () => {
+        const functionDefinition = {
+          handler,
+        }
+        const options = {}
+        const lambdaFunction = new LambdaFunction(
+          functionName,
+          functionDefinition,
+          serverless,
+          options,
+        )
+        const result = await lambdaFunction.runHandler()
+
+        await lambdaFunction.cleanup()
+
+        assert.equal(result, expected)
+      })
+    })
+  })
+
+  it('should pass remaining time to LambdaContext', async () => {
+    const functionDefinition = {
+      handler: 'fixtures/lambdaFunction.fixture.remainingExecutionTimeHandler',
+    }
+    const options = {}
+    const lambdaFunction = new LambdaFunction(
+      functionName,
+      functionDefinition,
+      serverless,
+      options,
+    )
+    const [first, second, third] = await lambdaFunction.runHandler()
+
+    await lambdaFunction.cleanup()
+
+    // handler "pauses" for 100 ms
+    assert.ok(first > second - 100)
+    assert.ok(second > third - 200)
+  })
+
+  it.skip('should use default lambda timeout when timeout is not provided', async () => {
+    const functionDefinition = {
+      handler: 'fixtures/lambdaFunction.fixture.defaultTimeoutHandler',
+    }
+    const options = {}
+    const lambdaFunction = new LambdaFunction(
+      functionName,
+      functionDefinition,
+      serverless,
+      options,
+    )
+    const remainingTime = await lambdaFunction.runHandler()
+
+    await lambdaFunction.cleanup()
+
+    assert.ok(remainingTime < DEFAULT_LAMBDA_TIMEOUT * 1000)
+
+    // result might be flaky/unreliable:
+    // (assmuning handler runs no longer than 1 s)
+    assert.ok(remainingTime + 1000 > DEFAULT_LAMBDA_TIMEOUT * 1000)
+  })
+
+  // // might run flaky (unreliable)
+  // test('executionTimeInMillis should return execution time', async () => {
+  //   const functionDefinition = {
+  //     handler: 'fixtures/lambdaFunction.fixture.executionTimeInMillisHandler',
+  //   }
+  //   const options = {}
+  //   const lambdaFunction = new LambdaFunction(
+  //     functionName,
+  //     functionDefinition,
+  //     provider,
+  //     config,
+  //     options,
+  //   )
+  //   const timerStart = performance.now()
+  //   await lambdaFunction.runHandler()
+  //   const timerEnd = performance.now()
+  //
+  //   expect(lambdaFunction.executionTimeInMillis).toBeLessThanOrEqual(
+  //     timerEnd - timerStart + 10,
+  //   )
+  // })
+})

package/src/lambda/__tests__/fixtures/Lambda/LambdaFunctionThatReturnsJSONObject.fixture.js

@@ -0,0 +1,47 @@
+import { dirname, resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import LambdaFunction from '../../../LambdaFunction.js'
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
+
+export default class LambdaFunctionThatReturnsJSONObject {
+  #lambdaFunction
+
+  options = {}
+
+  serverless = {
+    config: {
+      serverlessPath: '',
+      servicePath: resolve(__dirname),
+    },
+    service: {
+      provider: {
+        runtime: 'nodejs12.x',
+      },
+    },
+  }
+
+  listFunctionNames() {
+    return ['foo']
+  }
+
+  getByFunctionName(functionName) {
+    const functionDefinition = {
+      handler:
+        '../../fixtures/lambdaFunction.fixture.asyncFunctionHandlerObject',
+    }
+
+    this.#lambdaFunction = new LambdaFunction(
+      functionName,
+      functionDefinition,
+      this.serverless,
+      this.options,
+    )
+
+    return this.#lambdaFunction
+  }
+
+  async cleanup() {
+    await this.#lambdaFunction.cleanup()
+  }
+}