skuba 4.4.0 → 5.0.0-beta.0

package/template/koa-rest-api/src/framework/validation.ts

@@ -1,23 +1,64 @@
+import { ErrorMiddleware } from 'seek-koala';
+import { z } from 'zod';
+
 import { Context } from 'src/types/koa';
 
+/**
+ * Converts a `ZodError` into an `invalidFields` object
+ *
+ * For example, the `ZodError`:
+ *
+ * ```json
+ * {
+ *   "issues": [
+ *     {
+ *       "code": "invalid_type",
+ *       "expected": "string",
+ *       "received": "undefined",
+ *       "path": ["advertiserId"],
+ *       "message": "advertiserId is required in the URL"
+ *     }
+ *   ],
+ *   "name": "ZodError"
+ * }
+ * ```
+ *
+ * Returns:
+ *
+ * ```json
+ * { "/advertiserId": "advertiserId is required in the URL" }
+ * ```
+ */
+const parseInvalidFieldsFromError = ({
+  errors,
+}: z.ZodError): Record<string, string> =>
+  Object.fromEntries(
+    errors.map((err) => [`/${err.path.join('/')}`, err.message]),
+  );
+
 export const validate = <T>({
   ctx,
   input,
-  filter,
+  schema,
 }: {
   ctx: Context;
   input: unknown;
-  filter: (data: unknown) => T;
+  schema: z.ZodSchema<T>;
 }) => {
-  try {
-    return filter(input);
-  } catch (err) {
-    // TODO: consider providing structured error messages for your consumers.
-    return ctx.throw(422, err instanceof Error ? err.message : String(err));
+  const parseResult = schema.safeParse(input);
+  if (parseResult.success === false) {
+    return ctx.throw(
+      422,
+      new ErrorMiddleware.JsonResponse('Input validation failed', {
+        message: 'Input validation failed',
+        invalidFields: parseInvalidFieldsFromError(parseResult.error),
+      }),
+    );
   }
+  return parseResult.data;
 };
 
 export const validateRequestBody = <T>(
   ctx: Context,
-  filter: (input: unknown) => T,
-): T => validate({ ctx, input: ctx.request.body as unknown, filter });
+  schema: z.ZodSchema<T>,
+): T => validate<T>({ ctx, input: ctx.request.body as unknown, schema });

package/template/koa-rest-api/src/storage/jobs.ts

@@ -1,11 +1,11 @@
-import { v4 as uuidv4 } from 'uuid';
+import { randomUUID } from 'crypto';
 
 import { Job, JobInput } from 'src/types/jobs';
 
 const jobStore: Record<string, Job> = {};
 
 export const createJob = (jobInput: JobInput): Promise<Job> => {
-  const id = uuidv4();
+  const id = randomUUID();
 
   const job = { ...jobInput, id };
 

package/template/koa-rest-api/src/testing/types.ts

@@ -1,20 +1,15 @@
-/* eslint-disable new-cap */
-
 import { Chance } from 'chance';
-import * as t from 'runtypes';
-import checkFilter from 'runtypes-filter';
+import { z } from 'zod';
 
 import { JobInput } from 'src/types/jobs';
 
-export type IdDescription = t.Static<typeof IdDescription>;
+export type IdDescription = z.infer<typeof IdDescriptionSchema>;
 
-const IdDescription = t.Record({
-  id: t.String,
-  description: t.String,
+export const IdDescriptionSchema = z.object({
+  id: z.string(),
+  description: z.string(),
 });
 
-export const filterIdDescription = checkFilter(IdDescription);
-
 export const chance = new Chance();
 
 export const mockIdDescription = (): IdDescription => ({

package/template/koa-rest-api/src/types/jobs.ts

@@ -1,7 +1,4 @@
-/* eslint-disable new-cap */
-
-import * as t from 'runtypes';
-import checkFilter from 'runtypes-filter';
+import { z } from 'zod';
 
 export interface Job {
   id: string;
@@ -11,12 +8,10 @@ export interface Job {
   };
 }
 
-export type JobInput = t.Static<typeof JobInput>;
+export type JobInput = z.infer<typeof JobInputSchema>;
 
-const JobInput = t.Record({
-  hirer: t.Record({
-    id: t.String,
+export const JobInputSchema = z.object({
+  hirer: z.object({
+    id: z.string(),
   }),
 });
-
-export const filterJobInput = checkFilter(JobInput);

package/template/lambda-sqs-worker/.buildkite/pipeline.yml

@@ -26,13 +26,13 @@ configs:
         - yarn deploy
       concurrency: 1
       plugins:
-        - artifacts#v1.5.0:
+        - artifacts#v1.7.0:
             build: ${BUILDKITE_BUILD_ID}
             download: lib/*
         - *aws-sm
         - *private-npm
         - *docker-ecr-cache
-        - docker-compose#v3.10.0:
+        - docker-compose#v4.5.0:
             dependencies: false
             run: app
       retry:
@@ -60,7 +60,7 @@ steps:
       - *aws-sm
       - *private-npm
       - *docker-ecr-cache
-      - docker-compose#v3.10.0:
+      - docker-compose#v4.5.0:
           run: app
     timeout_in_minutes: 10
 

package/template/lambda-sqs-worker/README.md

@@ -15,7 +15,7 @@ Next steps:
 3. [ ] Create a new repository in the appropriate GitHub organisation.
 4. [ ] Add the repository to BuildAgency;
        see [Builds at SEEK] for more information.
-5. [ ] Add deployment bucket configuration and data classification tags to [serverless.yml](serverless.yml).
+5. [ ] Add Datadog extension, deployment bucket configuration and data classification tags to [serverless.yml](serverless.yml).
 6. [ ] Push local commits to the upstream GitHub branch.
 7. [ ] Configure [GitHub repository settings].
 8. [ ] Delete this checklist 😌.

package/template/lambda-sqs-worker/package.json

@@ -2,19 +2,19 @@
   "dependencies": {
     "@seek/logger": "^5.0.1",
     "aws-sdk": "^2.1011.0",
-    "seek-datadog-custom-metrics": "^4.0.0",
+    "datadog-lambda-js": "^6.83.0",
     "skuba-dive": "^2.0.0",
-    "runtypes": "^6.4.1",
-    "runtypes-filter": "^0.6.0"
+    "zod": "^3.19.1"
   },
   "devDependencies": {
     "@types/aws-lambda": "^8.10.84",
     "@types/chance": "^1.1.3",
-    "@types/node": "^16.0.0",
+    "@types/node": "16.11.64",
     "chance": "^1.1.8",
     "pino-pretty": "^9.0.0",
     "serverless": "^3.17.0",
     "serverless-plugin-canary-deployments": "^0.8.0",
+    "serverless-plugin-datadog": "^5.7.0",
     "serverless-prune-plugin": "^2.0.0",
     "skuba": "*"
   },

package/template/lambda-sqs-worker/serverless.yml

@@ -4,21 +4,33 @@ configValidationMode: error
 
 params:
   default:
+    datadogApiKeySecretArn: 'TODO: arn:aws:secretsmanager:${aws:region}:${aws:accountId}:secret:SECRET-NAME'
     description: <%- description %>
   dev:
-    deploymentBucket: 'TODO: deploymentBucketName'
+    deploymentBucket: 'TODO: deployment-bucket-name'
     isProduction: false
   prod:
-    deploymentBucket: 'TODO: deploymentBucketName'
+    deploymentBucket: 'TODO: deployment-bucket-name'
     isProduction: true
 
 custom:
+  datadog:
+    addLayers: false
+    apiKeySecretArn: ${param:datadogApiKeySecretArn}
+    enableDDLogs: false
+    # TODO: enable Datadog extension
+    enabled: false
+    exclude:
+      - WorkerPreHook
+    injectLogContext: false
+    version: ${env:VERSION}
   prune:
     automatic: true
     number: 3
 
 plugins:
   - serverless-plugin-canary-deployments
+  - serverless-plugin-datadog
   - serverless-prune-plugin
 
 provider:
@@ -51,6 +63,9 @@ provider:
         - Action: lambda:InvokeFunction
           Effect: Allow
           Resource: !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${self:functions.Worker.name}
+        - Action: secretsmanager:GetSecretValue
+          Effect: Allow
+          Resource: ${param:datadogApiKeySecretArn}-??????
         - Action: sns:Publish
           Effect: Allow
           Resource: !Ref DestinationTopic

package/template/lambda-sqs-worker/src/app.test.ts

@@ -17,7 +17,9 @@ describe('handler', () => {
 
   const score = chance.floating({ max: 1, min: 0 });
 
-  const increment = jest.spyOn(metricsClient, 'increment').mockReturnValue();
+  const distribution = jest
+    .spyOn(metricsClient, 'distribution')
+    .mockReturnValue();
 
   beforeAll(logger.spy);
   beforeAll(scoringService.spy);
@@ -32,7 +34,7 @@ describe('handler', () => {
 
   afterEach(() => {
     logger.clear();
-    increment.mockClear();
+    distribution.mockClear();
     scoringService.clear();
     sns.clear();
   });
@@ -47,12 +49,12 @@ describe('handler', () => {
     expect(logger.error).not.toHaveBeenCalled();
 
     expect(logger.info.mock.calls).toEqual([
-      [{ count: 1 }, 'received jobs'],
-      [{ snsMessageId: expect.any(String) }, 'scored job'],
-      ['request'],
+      [{ count: 1 }, 'Received jobs'],
+      [{ snsMessageId: expect.any(String) }, 'Scored job'],
+      ['Function succeeded'],
     ]);
 
-    expect(increment.mock.calls).toEqual([
+    expect(distribution.mock.calls).toEqual([
       ['job.received', 1],
       ['job.scored', 1],
     ]);
@@ -63,7 +65,7 @@ describe('handler', () => {
   it('throws on invalid input', () => {
     const event = createSqsEvent(['}']);
 
-    return expect(app.handler(event, ctx)).rejects.toThrow('invoke error');
+    return expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
   });
 
   it('bubbles up scoring service error', async () => {
@@ -73,9 +75,9 @@ describe('handler', () => {
 
     const event = createSqsEvent([JSON.stringify(jobPublished)]);
 
-    await expect(app.handler(event, ctx)).rejects.toThrow('invoke error');
+    await expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
 
-    expect(logger.error).toHaveBeenCalledWith({ err }, 'request');
+    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
   });
 
   it('bubbles up SNS error', async () => {
@@ -85,31 +87,31 @@ describe('handler', () => {
 
     const event = createSqsEvent([JSON.stringify(jobPublished)]);
 
-    await expect(app.handler(event, ctx)).rejects.toThrow('invoke error');
+    await expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
 
-    expect(logger.error).toHaveBeenCalledWith({ err }, 'request');
+    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
   });
 
   it('throws on zero records', async () => {
-    const err = new Error('received 0 records');
+    const err = new Error('Received 0 records');
 
     const event = createSqsEvent([]);
 
-    await expect(app.handler(event, ctx)).rejects.toThrow('invoke error');
+    await expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
 
-    expect(logger.error).toHaveBeenCalledWith({ err }, 'request');
+    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
   });
 
   it('throws on multiple records', async () => {
-    const err = new Error('received 2 records');
+    const err = new Error('Received 2 records');
 
     const event = createSqsEvent([
       JSON.stringify(jobPublished),
       JSON.stringify(jobPublished),
     ]);
 
-    await expect(app.handler(event, ctx)).rejects.toThrow('invoke error');
+    await expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
 
-    expect(logger.error).toHaveBeenCalledWith({ err }, 'request');
+    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
   });
 });

package/template/lambda-sqs-worker/src/app.ts

@@ -8,7 +8,7 @@ import { metricsClient } from 'src/framework/metrics';
 import { validateJson } from 'src/framework/validation';
 import { scoreJobPublishedEvent, scoringService } from 'src/services/jobScorer';
 import { sendPipelineEvent } from 'src/services/pipelineEventSender';
-import { filterJobPublishedEvent } from 'src/types/pipelineEvents';
+import { JobPublishedEventSchema } from 'src/types/pipelineEvents';
 
 /**
  * Tests connectivity to ensure appropriate access and network configuration.
@@ -20,19 +20,19 @@ const smokeTest = async () => {
 export const handler = createHandler<SQSEvent>(async (event) => {
   // Treat an empty object as our smoke test event.
   if (!Object.keys(event).length) {
-    logger.info('received smoke test request');
+    logger.info('Received smoke test request');
     return smokeTest();
   }
 
   const count = event.Records.length;
 
   if (count !== 1) {
-    throw Error(`received ${count} records`);
+    throw Error(`Received ${count} records`);
   }
 
-  logger.info({ count }, 'received jobs');
+  logger.info({ count }, 'Received jobs');
 
-  metricsClient.increment('job.received', event.Records.length);
+  metricsClient.distribution('job.received', event.Records.length);
 
   const record = event.Records[0];
 
@@ -40,13 +40,13 @@ export const handler = createHandler<SQSEvent>(async (event) => {
   // the event and eventually send it to your dead-letter queue. If you don't
   // trust your source to provide consistently well-formed input, consider
   // catching and handling this error in code.
-  const publishedJob = validateJson(record.body, filterJobPublishedEvent);
+  const publishedJob = validateJson(record.body, JobPublishedEventSchema);
 
   const scoredJob = await scoreJobPublishedEvent(publishedJob);
 
   const snsMessageId = await sendPipelineEvent(scoredJob);
 
-  logger.info({ snsMessageId }, 'scored job');
+  logger.info({ snsMessageId }, 'Scored job');
 
-  metricsClient.increment('job.scored', 1);
+  metricsClient.distribution('job.scored', 1);
 });

package/template/lambda-sqs-worker/src/config.ts

@@ -4,6 +4,7 @@ interface Config {
   environment: Environment;
 
   logLevel: string;
+  metrics: boolean;
   name: string;
   version: string;
 
@@ -20,6 +21,7 @@ const environment = Env.oneOf(environments)('ENVIRONMENT');
 const configs: Record<Environment, () => Omit<Config, 'environment'>> = {
   local: () => ({
     logLevel: 'debug',
+    metrics: false,
     name: '<%- serviceName %>',
     version: 'local',
 
@@ -41,6 +43,7 @@ const configs: Record<Environment, () => Omit<Config, 'environment'>> = {
 
   prod: () => ({
     logLevel: 'info',
+    metrics: true,
     name: Env.string('SERVICE'),
     version: Env.string('VERSION'),
 

package/template/lambda-sqs-worker/src/framework/handler.test.ts

@@ -18,7 +18,7 @@ describe('createHandler', () => {
     const handler = createHandler((event) => {
       expect(event).toBe(input);
 
-      logger.info('hello from handler');
+      logger.info('Handler invoked');
 
       return Promise.resolve(output);
     });
@@ -28,8 +28,8 @@ describe('createHandler', () => {
     expect(logger.error).not.toHaveBeenCalled();
 
     expect(logger.info.mock.calls).toEqual([
-      ['hello from handler'],
-      ['request'],
+      ['Handler invoked'],
+      ['Function succeeded'],
     ]);
   });
 
@@ -38,9 +38,9 @@ describe('createHandler', () => {
 
     const handler = createHandler(() => Promise.reject(err));
 
-    await expect(handler(input, ctx)).rejects.toThrow('invoke error');
+    await expect(handler(input, ctx)).rejects.toThrow('Function failed');
 
-    expect(logger.error.mock.calls).toEqual([[{ err }, 'request']]);
+    expect(logger.error.mock.calls).toEqual([[{ err }, 'Function failed']]);
 
     expect(logger.info).not.toHaveBeenCalled();
   });
@@ -52,9 +52,9 @@ describe('createHandler', () => {
       throw err;
     });
 
-    await expect(handler(input, ctx)).rejects.toThrow('invoke error');
+    await expect(handler(input, ctx)).rejects.toThrow('Function failed');
 
-    expect(logger.error.mock.calls).toEqual([[{ err }, 'request']]);
+    expect(logger.error.mock.calls).toEqual([[{ err }, 'Function failed']]);
 
     expect(logger.info).not.toHaveBeenCalled();
   });

package/template/lambda-sqs-worker/src/framework/handler.ts

@@ -1,20 +1,43 @@
-import { Context } from 'aws-lambda';
+import { datadog } from 'datadog-lambda-js';
 
+import { config } from 'src/config';
 import { logger, loggerContext } from 'src/framework/logging';
 
-export const createHandler =
-  <Event, Output = unknown>(fn: (event: Event) => Promise<Output>) =>
-  (event: Event, { awsRequestId }: Context) =>
+interface LambdaContext {
+  awsRequestId: string;
+}
+
+type Handler<Event, Output> = (
+  event: Event,
+  ctx: LambdaContext,
+) => Promise<Output>;
+
+/**
+ * Conditionally applies the Datadog wrapper to a Lambda handler.
+ *
+ * This also "fixes" its broken type definitions.
+ */
+const withDatadog = <Event, Output = unknown>(
+  fn: Handler<Event, Output>,
+): Handler<Event, Output> =>
+  // istanbul ignore next
+  config.metrics ? (datadog(fn) as Handler<Event, Output>) : fn;
+
+export const createHandler = <Event, Output = unknown>(
+  fn: (event: Event) => Promise<Output>,
+) =>
+  withDatadog<Event>((event, { awsRequestId }) =>
     loggerContext.run({ awsRequestId }, async () => {
       try {
         const output = await fn(event);
 
-        logger.info('request');
+        logger.info('Function succeeded');
 
         return output;
       } catch (err) {
-        logger.error({ err }, 'request');
+        logger.error({ err }, 'Function failed');
 
-        throw new Error('invoke error');
+        throw new Error('Function failed');
       }
-    });
+    }),
+  );

package/template/lambda-sqs-worker/src/framework/metrics.ts

@@ -1,10 +1,14 @@
-import {
-  createCloudWatchClient,
-  createTimedSpan,
-} from 'seek-datadog-custom-metrics';
+import { sendDistributionMetric } from 'datadog-lambda-js';
 
 import { config } from 'src/config';
 
-export const metricsClient = createCloudWatchClient(config);
+const prefix = `${config.name}.`;
 
-export const timedSpan = createTimedSpan(metricsClient);
+export const metricsClient = {
+  distribution: (
+    ...[name, ...rest]: Parameters<typeof sendDistributionMetric>
+  ) =>
+    config.metrics
+      ? sendDistributionMetric(`${prefix}${name}`, ...rest)
+      : undefined,
+};

package/template/lambda-sqs-worker/src/framework/validation.test.ts

@@ -1,6 +1,6 @@
 import {
+  IdDescriptionSchema,
   chance,
-  filterIdDescription,
   mockIdDescription,
 } from 'src/testing/types';
 
@@ -12,7 +12,7 @@ describe('validateJson', () => {
   it('permits valid input', () => {
     const input = JSON.stringify(idDescription);
 
-    expect(validateJson(input, filterIdDescription)).toStrictEqual(
+    expect(validateJson(input, IdDescriptionSchema)).toStrictEqual(
       idDescription,
     );
   });
@@ -20,7 +20,7 @@ describe('validateJson', () => {
   it('filters additional properties', () => {
     const input = JSON.stringify({ ...idDescription, hacker: chance.name() });
 
-    expect(validateJson(input, filterIdDescription)).toStrictEqual(
+    expect(validateJson(input, IdDescriptionSchema)).toStrictEqual(
       idDescription,
     );
   });
@@ -28,27 +28,47 @@ describe('validateJson', () => {
   it('blocks mistyped prop', () => {
     const input = JSON.stringify({ ...idDescription, id: null });
 
-    expect(() => validateJson(input, filterIdDescription))
+    expect(() => validateJson(input, IdDescriptionSchema))
       .toThrowErrorMatchingInlineSnapshot(`
-      "Validation failed:
-      {
-        "id": "Expected string, but was null"
-      }.
-      Object should match { id: string; description: string; }"
+      "[
+        {
+          "code": "invalid_type",
+          "expected": "string",
+          "received": "null",
+          "path": [
+            "id"
+          ],
+          "message": "Expected string, received null"
+        }
+      ]"
     `);
   });
 
   it('blocks missing prop', () => {
     const input = '{}';
 
-    expect(() => validateJson(input, filterIdDescription))
+    expect(() => validateJson(input, IdDescriptionSchema))
       .toThrowErrorMatchingInlineSnapshot(`
-      "Validation failed:
-      {
-        "id": "Expected string, but was missing",
-        "description": "Expected string, but was missing"
-      }.
-      Object should match { id: string; description: string; }"
+      "[
+        {
+          "code": "invalid_type",
+          "expected": "string",
+          "received": "undefined",
+          "path": [
+            "id"
+          ],
+          "message": "Required"
+        },
+        {
+          "code": "invalid_type",
+          "expected": "string",
+          "received": "undefined",
+          "path": [
+            "description"
+          ],
+          "message": "Required"
+        }
+      ]"
     `);
   });
 
@@ -56,7 +76,7 @@ describe('validateJson', () => {
     const input = '}';
 
     expect(() =>
-      validateJson(input, filterIdDescription),
+      validateJson(input, IdDescriptionSchema),
     ).toThrowErrorMatchingInlineSnapshot(
       `"Unexpected token } in JSON at position 0"`,
     );

package/template/lambda-sqs-worker/src/framework/validation.ts

@@ -1,2 +1,4 @@
-export const validateJson = <T>(input: string, filter: (data: unknown) => T) =>
-  filter(JSON.parse(input));
+import { z } from 'zod';
+
+export const validateJson = <T>(input: string, schema: z.ZodSchema<T>) =>
+  schema.parse(JSON.parse(input));

package/template/lambda-sqs-worker/src/services/jobScorer.ts

@@ -5,7 +5,7 @@ import {
 import {
   JobScorerInput,
   JobScorerOutput,
-  filterJobScorerOutput,
+  JobScorerOutputSchema,
 } from 'src/types/jobScorer';
 import { JobPublishedEvent, JobScoredEvent } from 'src/types/pipelineEvents';
 
@@ -39,7 +39,7 @@ const scoreJob = async ({
 }: JobScorerInput): Promise<JobScorerOutput> => {
   const score = await scoringService.request(details);
 
-  return filterJobScorerOutput({
+  return JobScorerOutputSchema.parse({
     id,
     score,
   });