skuba 0.0.0-beta-20231002015947

package/template/lambda-sqs-worker/serverless.yml

@@ -0,0 +1,202 @@
+service: <%- serviceName %>
+
+configValidationMode: error
+
+params:
+  default:
+    datadogApiKeySecretArn: 'TODO: arn:aws:secretsmanager:${aws:region}:${aws:accountId}:secret:SECRET-NAME'
+    description: <%- description %>
+  dev:
+    deploymentBucket: 'TODO: deployment-bucket-name'
+    isProduction: false
+  prod:
+    deploymentBucket: 'TODO: deployment-bucket-name'
+    isProduction: true
+
+custom:
+  datadog:
+    addLayers: false
+    apiKeySecretArn: ${param:datadogApiKeySecretArn}
+    enableDDLogs: false
+    # TODO: enable Datadog extension
+    enabled: false
+    exclude:
+      - WorkerPreHook
+    injectLogContext: false
+    version: ${env:VERSION}
+  prune:
+    automatic: true
+    number: 3
+
+plugins:
+  - serverless-plugin-canary-deployments
+  - serverless-plugin-datadog
+  - serverless-prune-plugin
+
+provider:
+  logRetentionInDays: 30
+  name: aws
+  region: ap-southeast-2
+  runtime: nodejs18.x
+  architecture: <%- lambdaServerlessArchitecture %>
+  deploymentMethod: direct
+  stackName: ${self:service}
+  stage: ${env:ENVIRONMENT}
+  deploymentBucket:
+    # Use a shared account-level bucket for Lambda bundles and other artefacts.
+    # This is easier to manage in terms of access, deployment, and tagging.
+    name: ${param:deploymentBucket}
+  environment:
+    NODE_ENV: production
+    # https://nodejs.org/api/cli.html#cli_node_options_options
+    NODE_OPTIONS: --enable-source-maps
+  iam:
+    role:
+      statements:
+        - Action:
+            - kms:Decrypt
+            - kms:GenerateDataKey*
+          Effect: Allow
+          Resource: !GetAtt EncryptionKey.Arn
+        - Action: lambda:InvokeFunction
+          Effect: Allow
+          Resource: !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${self:functions.Worker.name}
+        - Action: secretsmanager:GetSecretValue
+          Effect: Allow
+          Resource: ${param:datadogApiKeySecretArn}-??????
+        - Action: sns:Publish
+          Effect: Allow
+          Resource: !Ref DestinationTopic
+  stackTags:
+    # TODO: add data classification tags
+    # https://rfc.skinfra.xyz/RFC019-AWS-Tagging-Standard.html#seekdataconsumers
+    # seek:data:consumers: internal
+    # https://rfc.skinfra.xyz/RFC019-AWS-Tagging-Standard.html#seekdatatypes
+    # seek:data:types:restricted: job-ads
+    seek:env:label: ${env:ENVIRONMENT}
+    seek:env:production: ${param:isProduction}
+    seek:owner:team: '<%- teamName %>'
+    seek:source:sha: ${env:BUILDKITE_COMMIT, 'na'}
+    seek:source:url: 'https://github.com/SEEK-Jobs/<%- repoName %>'
+
+package:
+  patterns:
+    - '!**'
+    - lib/**
+    - node_modules/**
+
+functions:
+  Worker:
+    name: ${self:service}
+    handler: lib/app.handler
+    description: ${param:description}
+    memorySize: 128
+    reservedConcurrency: 20
+    timeout: 30
+    deploymentSettings:
+      type: AllAtOnce
+      alias: Live
+      preTrafficHook: WorkerPreHook
+    environment:
+      ENVIRONMENT: ${env:ENVIRONMENT}
+      SERVICE: ${self:service}
+      VERSION: ${env:VERSION, 'local'}
+
+      DESTINATION_SNS_TOPIC_ARN: !Ref DestinationTopic
+    events:
+      - sqs:
+          arn: !GetAtt MessageQueue.Arn
+          batchSize: 1
+  WorkerPreHook:
+    name: ${self:functions.Worker.name}-pre-hook
+    handler: lib/hooks.pre
+    memorySize: 128
+    # This is generous because a timeout will hang the deployment
+    timeout: 300
+    environment:
+      FUNCTION_NAME_TO_INVOKE: ${self:functions.Worker.name}
+
+resources:
+  # This becomes the Lambda application's description
+  Description: ${param:description}
+
+  Resources:
+    DeadLetterQueue:
+      Type: AWS::SQS::Queue
+      Properties:
+        MessageRetentionPeriod: 1209600
+        KmsDataKeyReusePeriodSeconds: 300
+        KmsMasterKeyId: !Ref EncryptionKey
+        QueueName: ${self:service}-dead-letters
+
+    MessageQueue:
+      Type: AWS::SQS::Queue
+      Properties:
+        MessageRetentionPeriod: 1209600
+        KmsDataKeyReusePeriodSeconds: 300
+        KmsMasterKeyId: !Ref EncryptionKey
+        QueueName: ${self:service}-messages
+        RedrivePolicy:
+          deadLetterTargetArn: !GetAtt DeadLetterQueue.Arn
+          maxReceiveCount: 3
+        VisibilityTimeout: 180
+
+    # MessageQueuePolicy:
+    #   Type: AWS::SQS::QueuePolicy
+    #   Properties:
+    #     PolicyDocument:
+    #       Version: '2012-10-17'
+    #       Statement:
+    #         - Action: sqs:SendMessage
+    #           Condition:
+    #             ArnEquals:
+    #               aws:SourceArn: 'TODO: sourceSnsTopicArn'
+    #           Effect: 'Allow'
+    #           Principal:
+    #             AWS: '*'
+    #           Resource: '*'
+    #     Queues:
+    #       - !Ref MessageQueue
+
+    # TopicSubscription:
+    #   Type: AWS::SNS::Subscription
+    #   Properties:
+    #     Endpoint: !GetAtt MessageQueue.Arn
+    #     Protocol: sqs
+    #     RawMessageDelivery: true
+    #     TopicArn: 'TODO: sourceSnsTopicArn'
+
+    DestinationTopic:
+      Type: AWS::SNS::Topic
+      Properties:
+        KmsMasterKeyId: alias/aws/sns
+        TopicName: ${self:service}
+
+    EncryptionKey:
+      Type: AWS::KMS::Key
+      Properties:
+        Description: ${self:service}
+        EnableKeyRotation: true
+        KeyPolicy:
+          Version: '2012-10-17'
+          Statement:
+            - Sid: Admin access
+              Effect: Allow
+              Action: kms:*
+              Principal:
+                AWS: !Ref AWS::AccountId
+              Resource: '*'
+            - Sid: SNS compatibility
+              Effect: Allow
+              Action:
+                - kms:Decrypt
+                - kms:GenerateDataKey*
+              Principal:
+                Service: sns.amazonaws.com
+              Resource: '*'
+
+    EncryptionKeyAlias:
+      Type: AWS::KMS::Alias
+      Properties:
+        AliasName: alias/seek/${self:service}
+        TargetKeyId: !Ref EncryptionKey

package/template/lambda-sqs-worker/skuba.template.js

@@ -0,0 +1,32 @@
+/**
+ * Run `skuba configure` to finish templating and remove this file.
+ */
+
+module.exports = {
+  entryPoint: 'src/app.ts#handler',
+  fields: [
+    {
+      name: 'serviceName',
+      message: 'Service slug',
+      initial: 'my-project',
+    },
+    {
+      name: 'description',
+      message: 'Description',
+      initial: 'A project just for me',
+    },
+    {
+      name: 'devBuildkiteQueueName',
+      message: 'Dev Buildkite queue',
+      initial: 'my-team-aws-account-dev:cicd',
+      validate: (value) => /^.+:.+$/.test(value),
+    },
+    {
+      name: 'prodBuildkiteQueueName',
+      message: 'Prod Buildkite queue',
+      initial: 'my-team-aws-account-prod:cicd',
+      validate: (value) => /^.+:.+$/.test(value),
+    },
+  ],
+  type: 'application',
+};

package/template/lambda-sqs-worker/src/app.test.ts

@@ -0,0 +1,116 @@
+import { PublishCommand } from '@aws-sdk/client-sns';
+
+import { metricsClient } from 'src/framework/metrics';
+import { createCtx, createSqsEvent } from 'src/testing/handler';
+import { logger } from 'src/testing/logging';
+import { scoringService, sns } from 'src/testing/services';
+import { chance, mockJobPublishedEvent } from 'src/testing/types';
+
+import * as app from './app';
+
+describe('app', () => {
+  it('exports a handler', () => expect(app).toHaveProperty('handler'));
+});
+
+describe('handler', () => {
+  const ctx = createCtx();
+
+  const jobPublished = mockJobPublishedEvent({ entityId: chance.name() });
+
+  const score = chance.floating({ max: 1, min: 0 });
+
+  const distribution = jest
+    .spyOn(metricsClient, 'distribution')
+    .mockReturnValue();
+
+  beforeAll(logger.spy);
+  beforeAll(scoringService.spy);
+
+  beforeEach(() => {
+    scoringService.request.mockResolvedValue(score);
+    sns.publish.resolves({ MessageId: chance.guid({ version: 4 }) });
+  });
+
+  afterEach(() => {
+    logger.clear();
+    distribution.mockClear();
+    scoringService.clear();
+    sns.clear();
+  });
+
+  it('handles one record', async () => {
+    const event = createSqsEvent([JSON.stringify(jobPublished)]);
+
+    await expect(app.handler(event, ctx)).resolves.toBeUndefined();
+
+    expect(scoringService.request).toHaveBeenCalledTimes(1);
+
+    expect(logger.error).not.toHaveBeenCalled();
+
+    expect(logger.debug.mock.calls).toEqual([
+      [{ count: 1 }, 'Received jobs'],
+      [{ snsMessageId: expect.any(String) }, 'Scored job'],
+      ['Function succeeded'],
+    ]);
+
+    expect(distribution.mock.calls).toEqual([
+      ['job.received', 1],
+      ['job.scored', 1],
+    ]);
+
+    expect(sns.client).toReceiveCommandTimes(PublishCommand, 1);
+  });
+
+  it('throws on invalid input', () => {
+    const event = createSqsEvent(['}']);
+
+    return expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
+  });
+
+  it('bubbles up scoring service error', async () => {
+    const err = Error(chance.sentence());
+
+    scoringService.request.mockRejectedValue(err);
+
+    const event = createSqsEvent([JSON.stringify(jobPublished)]);
+
+    await expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
+
+    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
+  });
+
+  it('bubbles up SNS error', async () => {
+    const err = Error(chance.sentence());
+
+    sns.publish.rejects(err);
+
+    const event = createSqsEvent([JSON.stringify(jobPublished)]);
+
+    await expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
+
+    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
+  });
+
+  it('throws on zero records', async () => {
+    const err = new Error('Received 0 records');
+
+    const event = createSqsEvent([]);
+
+    await expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
+
+    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
+  });
+
+  it('throws on multiple records', async () => {
+    const err = new Error('Received 2 records');
+
+    const event = createSqsEvent([
+      JSON.stringify(jobPublished),
+      JSON.stringify(jobPublished),
+    ]);
+
+    await expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
+
+    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
+  });
+});

package/template/lambda-sqs-worker/src/app.ts

@@ -0,0 +1,57 @@
+import 'skuba-dive/register';
+
+import type { SQSEvent } from 'aws-lambda';
+
+import { createHandler } from 'src/framework/handler';
+import { logger } from 'src/framework/logging';
+import { metricsClient } from 'src/framework/metrics';
+import { validateJson } from 'src/framework/validation';
+import { scoreJobPublishedEvent, scoringService } from 'src/services/jobScorer';
+import { sendPipelineEvent } from 'src/services/pipelineEventSender';
+import { JobPublishedEventSchema } from 'src/types/pipelineEvents';
+
+/**
+ * Tests connectivity to ensure appropriate access and network configuration.
+ */
+const smokeTest = async () => {
+  await Promise.all([scoringService.smokeTest(), sendPipelineEvent({}, true)]);
+};
+
+export const handler = createHandler<SQSEvent>(async (event) => {
+  // Treat an empty object as our smoke test event.
+  if (!Object.keys(event).length) {
+    logger.debug('Received smoke test request');
+    return smokeTest();
+  }
+
+  const count = event.Records.length;
+
+  if (count !== 1) {
+    throw Error(`Received ${count} records`);
+  }
+
+  logger.debug({ count }, 'Received jobs');
+
+  metricsClient.distribution('job.received', event.Records.length);
+
+  const record = event.Records[0];
+  if (!record) {
+    throw new Error('Malformed SQS event with no records');
+  }
+
+  const { body } = record;
+
+  // TODO: this throws an error, which will cause the Lambda function to retry
+  // the event and eventually send it to your dead-letter queue. If you don't
+  // trust your source to provide consistently well-formed input, consider
+  // catching and handling this error in code.
+  const publishedJob = validateJson(body, JobPublishedEventSchema);
+
+  const scoredJob = await scoreJobPublishedEvent(publishedJob);
+
+  const snsMessageId = await sendPipelineEvent(scoredJob);
+
+  logger.debug({ snsMessageId }, 'Scored job');
+
+  metricsClient.distribution('job.scored', 1);
+});

package/template/lambda-sqs-worker/src/config.ts

@@ -0,0 +1,57 @@
+import { Env } from 'skuba-dive';
+
+interface Config {
+  environment: Environment;
+
+  logLevel: string;
+  metrics: boolean;
+  name: string;
+  version: string;
+
+  destinationSnsTopicArn: string;
+}
+
+type Environment = (typeof environments)[number];
+
+const environments = ['local', 'test', 'dev', 'prod'] as const;
+
+const environment = Env.oneOf(environments)('ENVIRONMENT');
+
+/* istanbul ignore next: config verification makes more sense in a smoke test */
+const configs: Record<Environment, () => Omit<Config, 'environment'>> = {
+  local: () => ({
+    logLevel: 'debug',
+    metrics: false,
+    name: '<%- serviceName %>',
+    version: 'local',
+
+    destinationSnsTopicArn: 'arn:aws:sns:us-east-2:123456789012:destination',
+  }),
+
+  test: () => ({
+    ...configs.local(),
+
+    logLevel: Env.string('LOG_LEVEL', { default: 'silent' }),
+    version: 'test',
+  }),
+
+  dev: () => ({
+    ...configs.prod(),
+
+    logLevel: 'debug',
+  }),
+
+  prod: () => ({
+    logLevel: 'info',
+    metrics: true,
+    name: Env.string('SERVICE'),
+    version: Env.string('VERSION'),
+
+    destinationSnsTopicArn: Env.string('DESTINATION_SNS_TOPIC_ARN'),
+  }),
+};
+
+export const config: Config = {
+  ...configs[environment](),
+  environment,
+};

package/template/lambda-sqs-worker/src/framework/handler.test.ts

@@ -0,0 +1,61 @@
+import { createCtx } from 'src/testing/handler';
+import { logger } from 'src/testing/logging';
+import { chance } from 'src/testing/types';
+
+import { createHandler } from './handler';
+
+describe('createHandler', () => {
+  const ctx = createCtx();
+  const input = chance.paragraph();
+
+  beforeAll(logger.spy);
+
+  afterEach(logger.clear);
+
+  it('handles happy path', async () => {
+    const output = chance.paragraph();
+
+    const handler = createHandler((event) => {
+      expect(event).toBe(input);
+
+      logger.debug('Handler invoked');
+
+      return Promise.resolve(output);
+    });
+
+    await expect(handler(input, ctx)).resolves.toBe(output);
+
+    expect(logger.error).not.toHaveBeenCalled();
+
+    expect(logger.debug.mock.calls).toEqual([
+      ['Handler invoked'],
+      ['Function succeeded'],
+    ]);
+  });
+
+  it('handles async error', async () => {
+    const err = Error(chance.sentence());
+
+    const handler = createHandler(() => Promise.reject(err));
+
+    await expect(handler(input, ctx)).rejects.toThrow('Function failed');
+
+    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
+
+    expect(logger.debug).not.toHaveBeenCalled();
+  });
+
+  it('handles sync error', async () => {
+    const err = Error(chance.sentence());
+
+    const handler = createHandler(() => {
+      throw err;
+    });
+
+    await expect(handler(input, ctx)).rejects.toThrow('Function failed');
+
+    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
+
+    expect(logger.debug).not.toHaveBeenCalled();
+  });
+});

package/template/lambda-sqs-worker/src/framework/handler.ts

@@ -0,0 +1,43 @@
+import { datadog } from 'datadog-lambda-js';
+
+import { config } from 'src/config';
+import { logger, loggerContext } from 'src/framework/logging';
+
+interface LambdaContext {
+  awsRequestId: string;
+}
+
+type Handler<Event, Output> = (
+  event: Event,
+  ctx: LambdaContext,
+) => Promise<Output>;
+
+/**
+ * Conditionally applies the Datadog wrapper to a Lambda handler.
+ *
+ * This also "fixes" its broken type definitions.
+ */
+const withDatadog = <Event, Output = unknown>(
+  fn: Handler<Event, Output>,
+): Handler<Event, Output> =>
+  // istanbul ignore next
+  config.metrics ? (datadog(fn) as Handler<Event, Output>) : fn;
+
+export const createHandler = <Event, Output = unknown>(
+  fn: (event: Event) => Promise<Output>,
+) =>
+  withDatadog<Event>((event, { awsRequestId }) =>
+    loggerContext.run({ awsRequestId }, async () => {
+      try {
+        const output = await fn(event);
+
+        logger.debug('Function succeeded');
+
+        return output;
+      } catch (err) {
+        logger.error({ err }, 'Function failed');
+
+        throw new Error('Function failed');
+      }
+    }),
+  );

package/template/lambda-sqs-worker/src/framework/logging.ts

@@ -0,0 +1,27 @@
+import { AsyncLocalStorage } from 'async_hooks';
+
+import createLogger from '@seek/logger';
+
+import { config } from 'src/config';
+
+interface LoggerContext {
+  awsRequestId: string;
+}
+
+export const loggerContext = new AsyncLocalStorage<LoggerContext>();
+
+export const logger = createLogger({
+  base: {
+    environment: config.environment,
+    version: config.version,
+  },
+
+  level: config.logLevel,
+
+  mixin: () => ({ ...loggerContext.getStore() }),
+
+  name: config.name,
+
+  transport:
+    config.environment === 'local' ? { target: 'pino-pretty' } : undefined,
+});

package/template/lambda-sqs-worker/src/framework/metrics.ts

@@ -0,0 +1,14 @@
+import { sendDistributionMetric } from 'datadog-lambda-js';
+
+import { config } from 'src/config';
+
+const prefix = `${config.name}.`;
+
+export const metricsClient = {
+  distribution: (
+    ...[name, ...rest]: Parameters<typeof sendDistributionMetric>
+  ) =>
+    config.metrics
+      ? sendDistributionMetric(`${prefix}${name}`, ...rest)
+      : undefined,
+};

package/template/lambda-sqs-worker/src/framework/validation.test.ts

@@ -0,0 +1,84 @@
+import {
+  IdDescriptionSchema,
+  chance,
+  mockIdDescription,
+} from 'src/testing/types';
+
+import { validateJson } from './validation';
+
+describe('validateJson', () => {
+  const idDescription = mockIdDescription();
+
+  it('permits valid input', () => {
+    const input = JSON.stringify(idDescription);
+
+    expect(validateJson(input, IdDescriptionSchema)).toStrictEqual(
+      idDescription,
+    );
+  });
+
+  it('filters additional properties', () => {
+    const input = JSON.stringify({ ...idDescription, hacker: chance.name() });
+
+    expect(validateJson(input, IdDescriptionSchema)).toStrictEqual(
+      idDescription,
+    );
+  });
+
+  it('blocks mistyped prop', () => {
+    const input = JSON.stringify({ ...idDescription, id: null });
+
+    expect(() => validateJson(input, IdDescriptionSchema))
+      .toThrowErrorMatchingInlineSnapshot(`
+      "[
+        {
+          "code": "invalid_type",
+          "expected": "string",
+          "received": "null",
+          "path": [
+            "id"
+          ],
+          "message": "Expected string, received null"
+        }
+      ]"
+    `);
+  });
+
+  it('blocks missing prop', () => {
+    const input = '{}';
+
+    expect(() => validateJson(input, IdDescriptionSchema))
+      .toThrowErrorMatchingInlineSnapshot(`
+      "[
+        {
+          "code": "invalid_type",
+          "expected": "string",
+          "received": "undefined",
+          "path": [
+            "id"
+          ],
+          "message": "Required"
+        },
+        {
+          "code": "invalid_type",
+          "expected": "string",
+          "received": "undefined",
+          "path": [
+            "description"
+          ],
+          "message": "Required"
+        }
+      ]"
+    `);
+  });
+
+  it('blocks invalid JSON', () => {
+    const input = '}';
+
+    expect(() =>
+      validateJson(input, IdDescriptionSchema),
+    ).toThrowErrorMatchingInlineSnapshot(
+      `"Unexpected token } in JSON at position 0"`,
+    );
+  });
+});