skuba 4.2.1-beta.0 → 4.3.0-beta.0

package/template/express-rest-api/package.json

@@ -8,7 +8,7 @@
     "@types/express": "^4.17.13",
     "@types/node": "^16.0.0",
     "@types/supertest": "^2.0.11",
-    "pino-pretty": "^7.1.0",
+    "pino-pretty": "^8.0.0",
     "skuba": "*",
     "supertest": "^6.1.6"
   },

package/template/greeter/.buildkite/pipeline.yml

@@ -9,7 +9,7 @@ configs:
           NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
 
     - &docker-ecr-cache
-      seek-oss/docker-ecr-cache#v1.11.0:
+      seek-oss/docker-ecr-cache#v2.0.0:
         cache-on:
           - package.json
           - yarn.lock
@@ -34,3 +34,4 @@ steps:
       - *docker-ecr-cache
       - docker-compose#v3.9.0:
           run: app
+    timeout_in_minutes: 10

package/template/koa-rest-api/.buildkite/pipeline.yml

@@ -9,7 +9,7 @@ configs:
           NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
 
     - &docker-ecr-cache
-      seek-oss/docker-ecr-cache#v1.11.0:
+      seek-oss/docker-ecr-cache#v2.0.0:
         cache-on:
           - package.json
           - yarn.lock
@@ -53,6 +53,7 @@ steps:
       - *docker-ecr-cache
       - docker-compose#v3.9.0:
           run: app
+    timeout_in_minutes: 10
 
   - label: 📦 Build & Package
     depends_on: warm-prod
@@ -60,7 +61,7 @@ steps:
       - *aws-sm
       - *private-npm
       - *docker-ecr-cache
-      - seek-jobs/gantry#v1.7.0:
+      - seek-jobs/gantry#v1.8.1:
           command: build
           file: gantry.build.yml
           region: <%- region %>
@@ -77,7 +78,7 @@ steps:
     concurrency_group: <%- teamName %>/deploy/gantry/<%- devGantryEnvironmentName %>
     key: deploy-dev
     plugins:
-      - seek-jobs/gantry#v1.7.0:
+      - seek-jobs/gantry#v1.8.1:
           command: apply
           environment: <%- devGantryEnvironmentName %>
           file: gantry.apply.yml
@@ -92,7 +93,7 @@ steps:
     concurrency_group: <%- teamName %>/deploy/gantry/<%- prodGantryEnvironmentName %>
     depends_on: deploy-dev
     plugins:
-      - seek-jobs/gantry#v1.7.0:
+      - seek-jobs/gantry#v1.8.1:
           command: apply
           environment: <%- prodGantryEnvironmentName %>
           file: gantry.apply.yml

package/template/koa-rest-api/package.json

@@ -1,11 +1,11 @@
 {
   "dependencies": {
     "@koa/router": "^10.1.1",
-    "@opentelemetry/api": "^1.0.4",
+    "@opentelemetry/api": "^1.1.0",
     "@opentelemetry/exporter-collector-grpc": "^0.25.0",
-    "@opentelemetry/instrumentation-aws-sdk": "^0.6.0",
-    "@opentelemetry/instrumentation-http": "^0.27.0",
-    "@opentelemetry/sdk-node": "^0.27.0",
+    "@opentelemetry/instrumentation-aws-sdk": "^0.7.0",
+    "@opentelemetry/instrumentation-http": "^0.28.0",
+    "@opentelemetry/sdk-node": "^0.28.0",
     "@seek/logger": "^5.0.1",
     "aws-sdk": "^2.1039.0",
     "hot-shots": "^9.0.0",
@@ -15,7 +15,7 @@
     "runtypes": "^6.4.1",
     "runtypes-filter": "^0.6.0",
     "seek-datadog-custom-metrics": "^4.0.0",
-    "seek-koala": "^5.1.0",
+    "seek-koala": "^6.0.0",
     "skuba-dive": "^2.0.0",
     "uuid": "^8.3.2"
   },
@@ -28,7 +28,7 @@
     "@types/supertest": "^2.0.11",
     "@types/uuid": "^8.3.1",
     "chance": "^1.1.8",
-    "pino-pretty": "^7.1.0",
+    "pino-pretty": "^8.0.0",
     "skuba": "*",
     "supertest": "^6.1.6"
   },

package/template/koa-rest-api/src/api/jobs/getJobs.ts

@@ -1,4 +1,4 @@
-import { contextLogger } from 'src/framework/logging';
+import { logger } from 'src/framework/logging';
 import { metricsClient } from 'src/framework/metrics';
 import * as storage from 'src/storage/jobs';
 import { Middleware } from 'src/types/koa';
@@ -7,7 +7,7 @@ export const getJobsHandler: Middleware = async (ctx) => {
   const jobs = await storage.readJobs();
 
   // no PII in these jobs
-  contextLogger(ctx).debug({ jobs }, 'read jobs');
+  logger.debug({ jobs }, 'read jobs');
 
   metricsClient.increment('job.reads', jobs.length);
 

package/template/koa-rest-api/src/api/jobs/postJob.ts

@@ -1,4 +1,4 @@
-import { contextLogger } from 'src/framework/logging';
+import { logger } from 'src/framework/logging';
 import { metricsClient } from 'src/framework/metrics';
 import { validateRequestBody } from 'src/framework/validation';
 import * as storage from 'src/storage/jobs';
@@ -11,7 +11,7 @@ export const postJobHandler: Middleware = async (ctx) => {
   const job = await storage.createJob(jobInput);
 
   // no PII in these jobs
-  contextLogger(ctx).debug({ job }, 'created job');
+  logger.debug({ job }, 'created job');
 
   metricsClient.increment('job.creations');
 

package/template/koa-rest-api/src/framework/logging.ts

@@ -2,14 +2,20 @@ import createLogger from '@seek/logger';
 import { RequestLogging } from 'seek-koala';
 
 import { config } from 'src/config';
-import { Context } from 'src/types/koa';
 
-export const rootLogger = createLogger({
+const { createContextMiddleware, mixin } =
+  RequestLogging.createContextStorage();
+
+export const contextMiddleware = createContextMiddleware();
+
+export const logger = createLogger({
   base: {
     environment: config.environment,
     version: config.version,
   },
 
+  mixin,
+
   level: config.logLevel,
 
   name: config.name,
@@ -17,6 +23,3 @@ export const rootLogger = createLogger({
   transport:
     config.environment === 'local' ? { target: 'pino-pretty' } : undefined,
 });
-
-export const contextLogger = (ctx: Context) =>
-  rootLogger.child(RequestLogging.contextFields(ctx));

package/template/koa-rest-api/src/framework/metrics.ts

@@ -3,9 +3,9 @@ import { createStatsDClient } from 'seek-datadog-custom-metrics';
 
 import { config } from 'src/config';
 
-import { rootLogger } from './logging';
+import { logger } from './logging';
 
 /* istanbul ignore next: StatsD client is not our responsibility */
 export const metricsClient = createStatsDClient(StatsD, config, (err) =>
-  rootLogger.error({ err }, 'StatsD error'),
+  logger.error({ err }, 'StatsD error'),
 );

package/template/koa-rest-api/src/framework/server.test.ts

@@ -1,6 +1,6 @@
 import Router from '@koa/router';
 
-import { rootLogger } from 'src/testing/logging';
+import { logger } from 'src/testing/logging';
 import { metricsClient } from 'src/testing/metrics';
 import { agentFromRouter } from 'src/testing/server';
 import { chance } from 'src/testing/types';
@@ -15,10 +15,10 @@ const router = new Router()
 const agent = agentFromRouter(router);
 
 describe('createApp', () => {
-  beforeAll(rootLogger.spy);
+  beforeAll(logger.spy);
 
   afterEach(metricsClient.clear);
-  afterEach(rootLogger.clear);
+  afterEach(logger.clear);
 
   it('handles root route', async () => {
     middleware.mockImplementation((ctx) => (ctx.body = ''));
@@ -29,9 +29,9 @@ describe('createApp', () => {
       .expect('server', /.+/)
       .expect('x-api-version', /.+/);
 
-    expect(rootLogger.error).not.toBeCalled();
+    expect(logger.error).not.toBeCalled();
 
-    expect(rootLogger.info).not.toBeCalled();
+    expect(logger.info).not.toBeCalled();
 
     metricsClient.expectTagSubset(['env:test', 'version:test']);
     metricsClient.expectTagSubset([
@@ -51,9 +51,9 @@ describe('createApp', () => {
       .expect('server', /.+/)
       .expect('x-api-version', /.+/);
 
-    expect(rootLogger.error).not.toBeCalled();
+    expect(logger.error).not.toBeCalled();
 
-    expect(rootLogger.info).not.toBeCalled();
+    expect(logger.info).not.toBeCalled();
 
     metricsClient.expectTagSubset([
       'http_method:put',
@@ -72,9 +72,9 @@ describe('createApp', () => {
       .expect('server', /.+/)
       .expect('x-api-version', /.+/);
 
-    expect(rootLogger.error).not.toBeCalled();
+    expect(logger.error).not.toBeCalled();
 
-    expect(rootLogger.info).nthCalledWith(
+    expect(logger.info).nthCalledWith(
       1,
       expect.objectContaining({ status: 404 }),
       'Client error',
@@ -102,9 +102,9 @@ describe('createApp', () => {
       .expect('server', /.+/)
       .expect('x-api-version', /.+/);
 
-    expect(rootLogger.error).not.toBeCalled();
+    expect(logger.error).not.toBeCalled();
 
-    expect(rootLogger.info).nthCalledWith(
+    expect(logger.info).nthCalledWith(
       1,
       expect.objectContaining({ status: 400 }),
       'Client error',
@@ -129,9 +129,9 @@ describe('createApp', () => {
       .expect('server', /.+/)
       .expect('x-api-version', /.+/);
 
-    expect(rootLogger.error).not.toBeCalled();
+    expect(logger.error).not.toBeCalled();
 
-    expect(rootLogger.info).nthCalledWith(
+    expect(logger.info).nthCalledWith(
       1,
       expect.objectContaining({ err: expect.any(Error), status: 400 }),
       'Client error',
@@ -156,13 +156,13 @@ describe('createApp', () => {
       .expect('server', /.+/)
       .expect('x-api-version', /.+/);
 
-    expect(rootLogger.error).nthCalledWith(
+    expect(logger.error).nthCalledWith(
       1,
       expect.objectContaining({ err: expect.any(Error), status: 500 }),
       'Server error',
     );
 
-    expect(rootLogger.info).not.toBeCalled();
+    expect(logger.info).not.toBeCalled();
 
     metricsClient.expectTagSubset([
       'http_method:get',
@@ -185,13 +185,13 @@ describe('createApp', () => {
       .expect('server', /.+/)
       .expect('x-api-version', /.+/);
 
-    expect(rootLogger.error).nthCalledWith(
+    expect(logger.error).nthCalledWith(
       1,
       expect.objectContaining({ err, status: 500 }),
       'Server error',
     );
 
-    expect(rootLogger.info).not.toBeCalled();
+    expect(logger.info).not.toBeCalled();
 
     metricsClient.expectTagSubset([
       'http_method:get',
@@ -213,13 +213,13 @@ describe('createApp', () => {
       .expect('server', /.+/)
       .expect('x-api-version', /.+/);
 
-    expect(rootLogger.error).nthCalledWith(
+    expect(logger.error).nthCalledWith(
       1,
       expect.objectContaining({ err: null, status: 500 }),
       'Server error',
     );
 
-    expect(rootLogger.info).not.toBeCalled();
+    expect(logger.info).not.toBeCalled();
 
     metricsClient.expectTagSubset([
       'http_method:get',
@@ -242,13 +242,13 @@ describe('createApp', () => {
       .expect('server', /.+/)
       .expect('x-api-version', /.+/);
 
-    expect(rootLogger.error).nthCalledWith(
+    expect(logger.error).nthCalledWith(
       1,
       expect.objectContaining({ err, status: 500 }),
       'Server error',
     );
 
-    expect(rootLogger.info).not.toBeCalled();
+    expect(logger.info).not.toBeCalled();
 
     metricsClient.expectTagSubset([
       'http_method:get',

package/template/koa-rest-api/src/framework/server.ts

@@ -9,7 +9,7 @@ import {
 } from 'seek-koala';
 
 import { config } from 'src/config';
-import { rootLogger } from 'src/framework/logging';
+import { contextMiddleware, logger } from 'src/framework/logging';
 import { metricsClient } from 'src/framework/metrics';
 
 const metrics = MetricsMiddleware.create(
@@ -26,8 +26,8 @@ const requestLogging = RequestLogging.createMiddleware((ctx, fields, err) => {
   }
 
   return ctx.status < 500
-    ? rootLogger.info(fields, 'Client error')
-    : rootLogger.error(fields, 'Server error');
+    ? logger.info(fields, 'Client error')
+    : logger.error(fields, 'Server error');
 });
 
 const version = VersionMiddleware.create({
@@ -43,6 +43,7 @@ export const createApp = <State, Context>(
     // https://github.com/seek-oss/koala/tree/master/src/secureHeaders
     // https://github.com/venables/koa-helmet
     // .use(SecureHeaders.middleware)
+    .use(contextMiddleware)
     .use(requestLogging)
     .use(metrics)
     .use(ErrorMiddleware.handle)

package/template/koa-rest-api/src/listen.ts

@@ -2,7 +2,7 @@ import './register';
 
 import app from './app';
 import { config } from './config';
-import { rootLogger } from './framework/logging';
+import { logger } from './framework/logging';
 
 // This implements a minimal version of `koa-cluster`'s interface
 // If your application is deployed with more than 1 vCPU you can delete this
@@ -12,6 +12,6 @@ const listener = app.listen(config.port, () => {
   const address = listener.address();
 
   if (typeof address === 'object' && address) {
-    rootLogger.debug(`listening on port ${address.port}`);
+    logger.debug(`listening on port ${address.port}`);
   }
 });

package/template/koa-rest-api/src/testing/logging.ts

@@ -1,31 +1,16 @@
 import * as logging from 'src/framework/logging';
 
-export const contextLogger = {
+export const logger = {
   error: jest.fn(),
   info: jest.fn(),
 
   clear: () => {
-    contextLogger.error.mockClear();
-    contextLogger.info.mockClear();
-  },
-
-  spy: () =>
-    jest.spyOn(logging, 'contextLogger').mockReturnValue(contextLogger as any),
-};
-
-export const rootLogger = {
-  error: jest.fn(),
-  info: jest.fn(),
-
-  clear: () => {
-    rootLogger.error.mockClear();
-    rootLogger.info.mockClear();
+    logger.error.mockClear();
+    logger.info.mockClear();
   },
 
   spy: () => {
-    jest
-      .spyOn(logging.rootLogger, 'error')
-      .mockImplementation(rootLogger.error);
-    jest.spyOn(logging.rootLogger, 'info').mockImplementation(rootLogger.info);
+    jest.spyOn(logging.logger, 'error').mockImplementation(logger.error);
+    jest.spyOn(logging.logger, 'info').mockImplementation(logger.info);
   },
 };

package/template/lambda-sqs-worker/.buildkite/pipeline.yml

@@ -9,7 +9,7 @@ configs:
           NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
 
     - &docker-ecr-cache
-      seek-oss/docker-ecr-cache#v1.11.0:
+      seek-oss/docker-ecr-cache#v2.0.0:
         cache-on:
           - package.json
           - yarn.lock
@@ -63,6 +63,7 @@ steps:
       - *docker-ecr-cache
       - docker-compose#v3.9.0:
           run: app
+    timeout_in_minutes: 10
 
   - agents:
       queue: <%- devBuildkiteQueueName %>

package/template/lambda-sqs-worker/.nvmrc

@@ -1 +1 @@
-14
+16

package/template/lambda-sqs-worker/Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.2
 
-FROM node:14-alpine AS dev-deps
+FROM node:16-alpine AS dev-deps
 
 WORKDIR /workdir
 

package/template/lambda-sqs-worker/package.json

@@ -10,16 +10,16 @@
   "devDependencies": {
     "@types/aws-lambda": "^8.10.84",
     "@types/chance": "^1.1.3",
-    "@types/node": "^14.0.0",
+    "@types/node": "^16.0.0",
     "chance": "^1.1.8",
-    "pino-pretty": "^7.1.0",
-    "serverless": "^3.0.0",
-    "serverless-plugin-canary-deployments": "^0.7.0",
+    "pino-pretty": "^8.0.0",
+    "serverless": "^3.17.0",
+    "serverless-plugin-canary-deployments": "^0.8.0",
     "serverless-prune-plugin": "^2.0.0",
     "skuba": "*"
   },
   "engines": {
-    "node": ">=14"
+    "node": ">=16"
   },
   "license": "UNLICENSED",
   "private": true,

package/template/lambda-sqs-worker/serverless.yml

@@ -25,11 +25,11 @@ provider:
   logRetentionInDays: 30
   name: aws
   region: ap-southeast-2
-  runtime: nodejs14.x
+  runtime: nodejs16.x
   architecture: arm64
+  deploymentMethod: direct
   stackName: ${self:service}
   stage: ${env:ENVIRONMENT}
-  versionFunctions: true
   deploymentBucket:
     # Use a shared account-level bucket for Lambda bundles and other artefacts.
     # This is easier to manage in terms of access, deployment, and tagging.

package/template/lambda-sqs-worker/src/app.test.ts

@@ -1,6 +1,6 @@
 import { metricsClient } from 'src/framework/metrics';
 import { createCtx, createSqsEvent } from 'src/testing/handler';
-import { contextLogger } from 'src/testing/logging';
+import { logger } from 'src/testing/logging';
 import { scoringService, sns } from 'src/testing/services';
 import { chance, mockJobPublishedEvent } from 'src/testing/types';
 
@@ -19,7 +19,7 @@ describe('handler', () => {
 
   const increment = jest.spyOn(metricsClient, 'increment').mockReturnValue();
 
-  beforeAll(contextLogger.spy);
+  beforeAll(logger.spy);
   beforeAll(scoringService.spy);
   beforeAll(sns.spy);
 
@@ -31,7 +31,7 @@ describe('handler', () => {
   });
 
   afterEach(() => {
-    contextLogger.clear();
+    logger.clear();
     increment.mockClear();
     scoringService.clear();
     sns.clear();
@@ -44,9 +44,9 @@ describe('handler', () => {
 
     expect(scoringService.request).toBeCalledTimes(1);
 
-    expect(contextLogger.error).not.toBeCalled();
+    expect(logger.error).not.toBeCalled();
 
-    expect(contextLogger.info.mock.calls).toEqual([
+    expect(logger.info.mock.calls).toEqual([
       [{ count: 1 }, 'received jobs'],
       [{ snsMessageId: expect.any(String) }, 'scored job'],
       ['request'],
@@ -75,7 +75,7 @@ describe('handler', () => {
 
     await expect(app.handler(event, ctx)).rejects.toThrow('invoke error');
 
-    expect(contextLogger.error).toBeCalledWith({ err }, 'request');
+    expect(logger.error).toBeCalledWith({ err }, 'request');
   });
 
   it('bubbles up SNS error', async () => {
@@ -87,7 +87,7 @@ describe('handler', () => {
 
     await expect(app.handler(event, ctx)).rejects.toThrow('invoke error');
 
-    expect(contextLogger.error).toBeCalledWith({ err }, 'request');
+    expect(logger.error).toBeCalledWith({ err }, 'request');
   });
 
   it('throws on zero records', async () => {
@@ -97,7 +97,7 @@ describe('handler', () => {
 
     await expect(app.handler(event, ctx)).rejects.toThrow('invoke error');
 
-    expect(contextLogger.error).toBeCalledWith({ err }, 'request');
+    expect(logger.error).toBeCalledWith({ err }, 'request');
   });
 
   it('throws on multiple records', async () => {
@@ -110,6 +110,6 @@ describe('handler', () => {
 
     await expect(app.handler(event, ctx)).rejects.toThrow('invoke error');
 
-    expect(contextLogger.error).toBeCalledWith({ err }, 'request');
+    expect(logger.error).toBeCalledWith({ err }, 'request');
   });
 });

package/template/lambda-sqs-worker/src/app.ts

@@ -3,6 +3,7 @@ import 'skuba-dive/register';
 import { SQSEvent } from 'aws-lambda';
 
 import { createHandler } from 'src/framework/handler';
+import { logger } from 'src/framework/logging';
 import { metricsClient } from 'src/framework/metrics';
 import { validateJson } from 'src/framework/validation';
 import { scoreJobPublishedEvent, scoringService } from 'src/services/jobScorer';
@@ -16,7 +17,7 @@ const smokeTest = async () => {
   await Promise.all([scoringService.smokeTest(), sendPipelineEvent({}, true)]);
 };
 
-export const handler = createHandler<SQSEvent>(async (event, { logger }) => {
+export const handler = createHandler<SQSEvent>(async (event) => {
   // Treat an empty object as our smoke test event.
   if (!Object.keys(event).length) {
     logger.info('received smoke test request');

package/template/lambda-sqs-worker/src/framework/handler.test.ts

@@ -1,5 +1,5 @@
 import { createCtx } from 'src/testing/handler';
-import { contextLogger } from 'src/testing/logging';
+import { logger } from 'src/testing/logging';
 import { chance } from 'src/testing/types';
 
 import { createHandler } from './handler';
@@ -8,14 +8,14 @@ describe('createHandler', () => {
   const ctx = createCtx();
   const input = chance.paragraph();
 
-  beforeAll(contextLogger.spy);
+  beforeAll(logger.spy);
 
-  afterEach(contextLogger.clear);
+  afterEach(logger.clear);
 
   it('handles happy path', async () => {
     const output = chance.paragraph();
 
-    const handler = createHandler((event, { logger }) => {
+    const handler = createHandler((event) => {
       expect(event).toBe(input);
 
       logger.info('hello from handler');
@@ -25,9 +25,9 @@ describe('createHandler', () => {
 
     await expect(handler(input, ctx)).resolves.toBe(output);
 
-    expect(contextLogger.error).not.toBeCalled();
+    expect(logger.error).not.toBeCalled();
 
-    expect(contextLogger.info.mock.calls).toEqual([
+    expect(logger.info.mock.calls).toEqual([
       ['hello from handler'],
       ['request'],
     ]);
@@ -40,9 +40,9 @@ describe('createHandler', () => {
 
     await expect(handler(input, ctx)).rejects.toThrow('invoke error');
 
-    expect(contextLogger.error.mock.calls).toEqual([[{ err }, 'request']]);
+    expect(logger.error.mock.calls).toEqual([[{ err }, 'request']]);
 
-    expect(contextLogger.info).not.toBeCalled();
+    expect(logger.info).not.toBeCalled();
   });
 
   it('handles sync error', async () => {
@@ -54,8 +54,8 @@ describe('createHandler', () => {
 
     await expect(handler(input, ctx)).rejects.toThrow('invoke error');
 
-    expect(contextLogger.error.mock.calls).toEqual([[{ err }, 'request']]);
+    expect(logger.error.mock.calls).toEqual([[{ err }, 'request']]);
 
-    expect(contextLogger.info).not.toBeCalled();
+    expect(logger.info).not.toBeCalled();
   });
 });

package/template/lambda-sqs-worker/src/framework/handler.ts

@@ -1,23 +1,20 @@
-import { Logger } from '@seek/logger';
 import { Context } from 'aws-lambda';
 
-import { contextLogger } from 'src/framework/logging';
+import { logger, loggerContext } from 'src/framework/logging';
 
-export const createHandler = <Event, Output = unknown>(
-  fn: (event: Event, ctx: { logger: Logger }) => Promise<Output>,
-) =>
-  async function lambdaHandler(event: Event, ctx: Context) {
-    const logger = contextLogger(ctx);
+export const createHandler =
+  <Event, Output = unknown>(fn: (event: Event) => Promise<Output>) =>
+  (event: Event, { awsRequestId }: Context) =>
+    loggerContext.run({ awsRequestId }, async () => {
+      try {
+        const output = await fn(event);
 
-    try {
-      const output = await fn(event, { logger });
+        logger.info('request');
 
-      logger.info('request');
+        return output;
+      } catch (err) {
+        logger.error({ err }, 'request');
 
-      return output;
-    } catch (err) {
-      logger.error({ err }, 'request');
-
-      throw new Error('invoke error');
-    }
-  };
+        throw new Error('invoke error');
+      }
+    });