skuba 0.0.0-beta-20231002015947

package/template/lambda-sqs-worker/src/framework/validation.ts

@@ -0,0 +1,10 @@
+import type { z } from 'zod';
+
+export const validateJson = <
+  Output,
+  Def extends z.ZodTypeDef = z.ZodTypeDef,
+  Input = Output,
+>(
+  input: string,
+  schema: z.ZodSchema<Output, Def, Input>,
+): Output => schema.parse(JSON.parse(input));

package/template/lambda-sqs-worker/src/hooks.ts

@@ -0,0 +1,96 @@
+/* eslint-disable no-console */
+/* istanbul ignore file */
+
+// Use minimal dependencies to reduce the chance of crashes on module load.
+import {
+  CodeDeployClient,
+  PutLifecycleEventHookExecutionStatusCommand,
+} from '@aws-sdk/client-codedeploy';
+import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda';
+import { toUtf8 } from '@aws-sdk/util-utf8-node';
+
+const codeDeploy = new CodeDeployClient({
+  apiVersion: '2014-10-06',
+  maxAttempts: 5,
+});
+
+const lambda = new LambdaClient({
+  apiVersion: '2015-03-31',
+  maxAttempts: 5,
+});
+
+type Status = 'Succeeded' | 'Failed';
+
+/**
+ * Synchronously invokes a Lambda function with a smoke test event.
+ *
+ * Any non-error response is treated as a success.
+ */
+const smokeTestLambdaFunction = async (): Promise<Status> => {
+  const functionName = process.env.FUNCTION_NAME_TO_INVOKE;
+
+  if (!functionName) {
+    console.error('Missing process.env.FUNCTION_NAME_TO_INVOKE');
+    return 'Failed';
+  }
+
+  console.info('Function:', functionName);
+
+  const response = await lambda.send(
+    new InvokeCommand({
+      FunctionName: functionName,
+      InvocationType: 'RequestResponse',
+      // Treat an empty object as our smoke test event.
+      Payload: Buffer.from('{}'),
+    }),
+  );
+
+  console.info('Version:', response.ExecutedVersion ?? '?');
+  console.info('Status', response.StatusCode ?? '?');
+
+  if (response.FunctionError) {
+    console.error('Error:', response.FunctionError);
+    if (response.Payload) {
+      console.error(toUtf8(response.Payload));
+    }
+    return 'Failed';
+  }
+
+  return response.StatusCode === 200 ? 'Succeeded' : 'Failed';
+};
+
+/**
+ * The event supplied to a CodeDeploy lifecycle hook Lambda function.
+ *
+ * {@link https://docs.aws.amazon.com/codedeploy/latest/userguide/tutorial-ecs-with-hooks-create-hooks.html}
+ */
+interface CodeDeployLifecycleHookEvent {
+  DeploymentId: string;
+  LifecycleEventHookExecutionId: string;
+}
+
+/**
+ * A handler to smoke test a new Lambda function version before it goes live.
+ *
+ * This tries to be exception safe so that a status reaches CodeDeploy. If we
+ * crash or otherwise fail to report back, the deployment will hang for an hour.
+ */
+export const pre = async (
+  event: CodeDeployLifecycleHookEvent,
+): Promise<void> => {
+  let status: Status;
+  try {
+    status = await smokeTestLambdaFunction();
+  } catch (err) {
+    console.error('Exception:', err);
+    status = 'Failed';
+  }
+
+  await codeDeploy.send(
+    new PutLifecycleEventHookExecutionStatusCommand({
+      deploymentId: event.DeploymentId,
+      lifecycleEventHookExecutionId: event.LifecycleEventHookExecutionId,
+      status,
+    }),
+  );
+};

package/template/lambda-sqs-worker/src/mapping/jobScorer.ts

@@ -0,0 +1,22 @@
+import type { JobScorerInput, JobScorerOutput } from 'src/types/jobScorer';
+import type {
+  JobPublishedEvent,
+  JobScoredEvent,
+} from 'src/types/pipelineEvents';
+
+export const jobPublishedEventToScorerInput = (
+  record: JobPublishedEvent,
+): JobScorerInput => ({
+  details: record.data.details,
+  id: record.entityId,
+});
+
+export const jobScorerOutputToScoredEvent = (
+  output: JobScorerOutput,
+): JobScoredEvent => ({
+  data: {
+    score: output.score,
+  },
+  entityId: output.id,
+  eventType: 'JobScored',
+});

package/template/lambda-sqs-worker/src/services/aws.ts

@@ -0,0 +1,5 @@
+import { SNSClient } from '@aws-sdk/client-sns';
+
+export const sns = new SNSClient({
+  apiVersion: '2010-03-31',
+});

package/template/lambda-sqs-worker/src/services/jobScorer.test.ts

@@ -0,0 +1,44 @@
+import { scoringService } from 'src/testing/services';
+import { chance, mockJobPublishedEvent } from 'src/testing/types';
+
+import * as jobScorer from './jobScorer';
+
+describe('scoreJobPublishedEvent', () => {
+  beforeAll(scoringService.spy);
+
+  afterEach(scoringService.clear);
+
+  it('scores an event', async () => {
+    const score = chance.floating({ max: 1, min: 0 });
+
+    scoringService.request.mockResolvedValue(score);
+
+    await expect(
+      jobScorer.scoreJobPublishedEvent(
+        mockJobPublishedEvent({ entityId: '1' }),
+      ),
+    ).resolves.toStrictEqual({
+      data: {
+        score,
+      },
+      entityId: '1',
+      eventType: 'JobScored',
+    });
+
+    expect(scoringService.request).toHaveBeenCalledTimes(1);
+  });
+
+  it('bubbles up scoring service error', async () => {
+    const err = Error(chance.sentence());
+
+    scoringService.request.mockRejectedValue(err);
+
+    await expect(
+      jobScorer.scoreJobPublishedEvent(
+        mockJobPublishedEvent({ entityId: '1' }),
+      ),
+    ).rejects.toThrow(err);
+
+    expect(scoringService.request).toHaveBeenCalledTimes(1);
+  });
+});

package/template/lambda-sqs-worker/src/services/jobScorer.ts

@@ -0,0 +1,59 @@
+import {
+  jobPublishedEventToScorerInput,
+  jobScorerOutputToScoredEvent,
+} from 'src/mapping/jobScorer';
+import {
+  type JobScorerInput,
+  type JobScorerOutput,
+  JobScorerOutputSchema,
+} from 'src/types/jobScorer';
+import type {
+  JobPublishedEvent,
+  JobScoredEvent,
+} from 'src/types/pipelineEvents';
+
+/* istanbul ignore next: simulation of an external service */
+export const scoringService = {
+  request: (details: string): Promise<unknown> => {
+    // Networking woes
+    if (Math.random() < 0.05) {
+      const err = Error('could not reach scoring service');
+
+      return Promise.reject(err);
+    }
+
+    // Unexpected behaviour on certain inputs
+    if (details.length % 100 === 0) {
+      return Promise.resolve(null);
+    }
+
+    return Promise.resolve(Math.random());
+  },
+
+  smokeTest: async (): Promise<void> => {
+    // A connectivity test
+    await Promise.resolve();
+  },
+};
+
+const scoreJob = async ({
+  details,
+  id,
+}: JobScorerInput): Promise<JobScorerOutput> => {
+  const score = await scoringService.request(details);
+
+  return JobScorerOutputSchema.parse({
+    id,
+    score,
+  });
+};
+
+export const scoreJobPublishedEvent = async (
+  publishedJob: JobPublishedEvent,
+): Promise<JobScoredEvent> => {
+  const scorerInput = jobPublishedEventToScorerInput(publishedJob);
+
+  const scorerOutput = await scoreJob(scorerInput);
+
+  return jobScorerOutputToScoredEvent(scorerOutput);
+};

package/template/lambda-sqs-worker/src/services/pipelineEventSender.test.ts

@@ -0,0 +1,40 @@
+import { PublishCommand } from '@aws-sdk/client-sns';
+
+import { sns } from 'src/testing/services';
+import { chance } from 'src/testing/types';
+
+import { sendPipelineEvent } from './pipelineEventSender';
+
+describe('sendPipelineEvent', () => {
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('handles happy path', async () => {
+    const messageId = chance.guid({ version: 4 });
+
+    sns.publish.resolves({ MessageId: messageId });
+
+    await expect(sendPipelineEvent({})).resolves.toBe(messageId);
+
+    expect(sns.client).toReceiveCommandTimes(PublishCommand, 1);
+  });
+
+  it('bubbles up SNS error', () => {
+    const err = Error(chance.sentence());
+
+    sns.publish.rejects(err);
+
+    return expect(sendPipelineEvent({})).rejects.toThrow(err);
+  });
+
+  it('throws on missing message ID', () => {
+    sns.publish.resolves({});
+
+    return expect(
+      sendPipelineEvent({}),
+    ).rejects.toThrowErrorMatchingInlineSnapshot(
+      `"SNS did not return a message ID"`,
+    );
+  });
+});

package/template/lambda-sqs-worker/src/services/pipelineEventSender.ts

@@ -0,0 +1,33 @@
+import { PublishCommand } from '@aws-sdk/client-sns';
+
+import { config } from 'src/config';
+
+import { sns } from './aws';
+
+export const sendPipelineEvent = async (
+  event: unknown,
+  smokeTest: boolean = false,
+): Promise<string> => {
+  const snsResponse = await sns.send(
+    new PublishCommand({
+      Message: JSON.stringify(event),
+      ...(smokeTest && {
+        MessageAttributes: {
+          // Used for connectivity tests.
+          // Subscribers should filter out messages containing this attribute.
+          SmokeTest: {
+            DataType: 'String',
+            StringValue: 'true',
+          },
+        },
+      }),
+      TopicArn: config.destinationSnsTopicArn,
+    }),
+  );
+
+  if (snsResponse.MessageId === undefined) {
+    throw Error('SNS did not return a message ID');
+  }
+
+  return snsResponse.MessageId;
+};

package/template/lambda-sqs-worker/src/testing/handler.ts

@@ -0,0 +1,13 @@
+import type { Context, SQSEvent } from 'aws-lambda';
+
+import { chance } from './types';
+
+export const createCtx = () =>
+  ({
+    awsRequestId: chance.guid({ version: 4 }),
+  }) as Context;
+
+export const createSqsEvent = (bodies: string[]) =>
+  ({
+    Records: bodies.map((body) => ({ body })),
+  }) as SQSEvent;

package/template/lambda-sqs-worker/src/testing/logging.ts

@@ -0,0 +1,19 @@
+import * as logging from 'src/framework/logging';
+
+export const logger = {
+  error: jest.fn(),
+  info: jest.fn(),
+  debug: jest.fn(),
+
+  clear: () => {
+    logger.error.mockClear();
+    logger.info.mockClear();
+    logger.debug.mockClear();
+  },
+
+  spy: () => {
+    jest.spyOn(logging.logger, 'error').mockImplementation(logger.error);
+    jest.spyOn(logging.logger, 'info').mockImplementation(logger.info);
+    jest.spyOn(logging.logger, 'debug').mockImplementation(logger.debug);
+  },
+};

package/template/lambda-sqs-worker/src/testing/services.ts

@@ -0,0 +1,28 @@
+import 'aws-sdk-client-mock-jest';
+
+import { PublishCommand } from '@aws-sdk/client-sns';
+import { mockClient } from 'aws-sdk-client-mock';
+
+import { sns as snsClient } from 'src/services/aws';
+import * as jobScorer from 'src/services/jobScorer';
+
+export const scoringService = {
+  request: jest.fn(),
+
+  clear: () => scoringService.request.mockClear(),
+
+  spy: () =>
+    jest
+      .spyOn(jobScorer.scoringService, 'request')
+      .mockImplementation(scoringService.request),
+};
+
+const snsMock = mockClient(snsClient);
+
+export const sns = {
+  publish: snsMock.on(PublishCommand),
+
+  clear: () => snsMock.resetHistory(),
+
+  client: snsMock,
+};

package/template/lambda-sqs-worker/src/testing/types.ts

@@ -0,0 +1,33 @@
+import { Chance } from 'chance';
+import { z } from 'zod';
+
+import type { JobPublishedEvent } from 'src/types/pipelineEvents';
+
+export type IdDescription = z.infer<typeof IdDescriptionSchema>;
+
+export const IdDescriptionSchema = z.object({
+  id: z.string(),
+  description: z.string(),
+});
+
+export const chance = new Chance();
+
+export const mockIdDescription = (): IdDescription => ({
+  id: chance.guid({ version: 4 }),
+  description: chance.sentence(),
+});
+
+export const mockIdDescriptionJson = (): string =>
+  JSON.stringify(mockIdDescription());
+
+export const mockJobPublishedEvent = ({
+  entityId,
+}: {
+  entityId: string;
+}): JobPublishedEvent => ({
+  data: {
+    details: chance.paragraph(),
+  },
+  entityId,
+  eventType: 'JobPublished',
+});

package/template/lambda-sqs-worker/src/types/jobScorer.ts

@@ -0,0 +1,15 @@
+import { z } from 'zod';
+
+export type JobScorerInput = z.infer<typeof JobScorerInputSchema>;
+
+const JobScorerInputSchema = z.object({
+  id: z.string(),
+  details: z.string(),
+});
+
+export type JobScorerOutput = z.infer<typeof JobScorerOutputSchema>;
+
+export const JobScorerOutputSchema = z.object({
+  id: z.string(),
+  score: z.number(),
+});

package/template/lambda-sqs-worker/src/types/pipelineEvents.ts

@@ -0,0 +1,21 @@
+import { z } from 'zod';
+
+export type JobPublishedEvent = z.infer<typeof JobPublishedEventSchema>;
+
+export const JobPublishedEventSchema = z.object({
+  data: z.object({
+    details: z.string(),
+  }),
+  entityId: z.string(),
+  eventType: z.literal('JobPublished'),
+});
+
+export type JobScoredEvent = z.infer<typeof JobScoredEventSchema>;
+
+export const JobScoredEventSchema = z.object({
+  data: z.object({
+    score: z.number(),
+  }),
+  entityId: z.string(),
+  eventType: z.literal('JobScored'),
+});

package/template/lambda-sqs-worker/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "baseUrl": ".",
+    "lib": ["ES2022"],
+    "outDir": "lib",
+    "paths": {
+      "src": ["src"]
+    },
+    "target": "ES2022"
+  },
+  "exclude": ["lib*/**/*"],
+  "extends": "skuba/config/tsconfig.json"
+}

package/template/lambda-sqs-worker-cdk/.buildkite/pipeline.yml

@@ -0,0 +1,95 @@
+agents:
+  queue: <%- prodBuildkiteQueueName %>
+
+configs:
+  plugins:
+    - &aws-sm
+      seek-oss/aws-sm#v2.3.1:
+        env:
+          NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
+
+    - &docker-ecr-cache
+      seek-oss/docker-ecr-cache#v2.1.0: &docker-ecr-cache-defaults
+        cache-on:
+          - package.json
+          - yarn.lock
+        secrets: id=npm,src=.npmrc
+
+    - &private-npm
+      seek-oss/private-npm#v1.2.0:
+        env: NPM_READ_TOKEN
+
+  base-steps:
+    - &deploy
+      commands:
+        - echo '+++ yarn deploy'
+        - yarn deploy
+      concurrency: 1
+      plugins:
+        - artifacts#v1.9.2:
+            build: ${BUILDKITE_BUILD_ID}
+            download: lib/*
+        - *aws-sm
+        - *private-npm
+        - *docker-ecr-cache
+        - docker-compose#v4.14.0:
+            dependencies: false
+            run: app
+      retry:
+        manual:
+          # Only use this if you need to roll back a deployment ASAP.
+          # Always follow up with a proper revert or fix in Git history.
+          permit_on_passed: true
+
+steps:
+  - label: 🧪 Test, Lint & Build
+    artifact_paths: lib/**/*
+    commands:
+      - echo '+++ yarn test:ci'
+      - yarn test
+      - echo '--- yarn lint'
+      - yarn lint
+      - echo '--- yarn build'
+      - yarn build
+    env:
+      GET_GITHUB_TOKEN: please
+    plugins:
+      - *aws-sm
+      - *private-npm
+      - *docker-ecr-cache
+      - docker-compose#v4.14.0:
+          run: app
+    timeout_in_minutes: 10
+
+  - agents:
+      queue: <%- devBuildkiteQueueName %>
+    branches: '!renovate-*'
+    label: 🧖‍♀️ Warm Dev
+    command: ':'
+    plugins:
+      - *aws-sm
+      - *private-npm
+      - seek-oss/docker-ecr-cache#v2.1.0:
+          <<: *docker-ecr-cache-defaults
+          skip-pull-from-cache: true
+
+  - wait
+  - block: 🙋🏻‍♀️ Deploy Dev
+    branches: '!${BUILDKITE_PIPELINE_DEFAULT_BRANCH}'
+
+  - <<: *deploy
+    agents:
+      queue: <%- devBuildkiteQueueName %>
+    env:
+      ENVIRONMENT: dev
+    label: 🤞 Deploy Dev
+    concurrency_group: '<%- repoName %>/deploy/dev'
+    key: deploy-dev
+
+  - <<: *deploy
+    env:
+      ENVIRONMENT: prod
+    label: 🚀 Deploy Prod
+    branches: ${BUILDKITE_PIPELINE_DEFAULT_BRANCH}
+    concurrency_group: '<%- repoName %>/deploy/prod'
+    depends_on: deploy-dev

package/template/lambda-sqs-worker-cdk/.nvmrc

@@ -0,0 +1 @@
+18

package/template/lambda-sqs-worker-cdk/Dockerfile

@@ -0,0 +1,13 @@
+# syntax=docker/dockerfile:1.2
+
+FROM --platform=${BUILDPLATFORM:-<%- platformName %>} node:18-alpine AS dev-deps
+
+WORKDIR /workdir
+
+COPY package.json yarn.lock ./
+
+RUN \
+  --mount=type=secret,id=npm,dst=/workdir/.npmrc \
+  yarn install --frozen-lockfile --ignore-optional --non-interactive && \
+  yarn package && \
+  yarn cache clean

package/template/lambda-sqs-worker-cdk/cdk.json

@@ -0,0 +1,25 @@
+{
+  "app": "npx ts-node infra/index.ts",
+  "context": {
+    "global": {
+      "appName": "<%- serviceName %>"
+    },
+    "dev": {
+      "workerLambda": {
+        "reservedConcurrency": 1,
+        "environment": {
+          "SOMETHING": "dev"
+        }
+      }
+    },
+    "prod": {
+      "workerLambda": {
+        "reservedConcurrency": 2,
+        "environment": {
+          "SOMETHING": "prod"
+        }
+      }
+    }
+  },
+  "progress": "events"
+}

package/template/lambda-sqs-worker-cdk/docker-compose.yml

@@ -0,0 +1,28 @@
+version: '3.7'
+
+services:
+  app:
+    environment:
+      # Enable Buildkite + GitHub integrations.
+      - BUILDKITE
+      - BUILDKITE_AGENT_ACCESS_TOKEN
+      - BUILDKITE_BRANCH
+      - BUILDKITE_BUILD_NUMBER
+      - BUILDKITE_JOB_ID
+      - BUILDKITE_PIPELINE_DEFAULT_BRANCH
+      - BUILDKITE_STEP_ID
+      - GITHUB_API_TOKEN
+      # Tag AWS resources with the commit hash.
+      - BUILDKITE_COMMIT
+      # Pass through application configuration.
+      - ENVIRONMENT
+      - VERSION
+    image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''}
+    init: true
+    volumes:
+      - ./:/workdir
+      # Mount agent for Buildkite annotations.
+      - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent
+      # Mount cached dependencies.
+      - /workdir/node_modules
+      - /workdir/lib/node_modules