skuba 10.0.0-node-22-20250115230358 → 10.0.0-node-22-20250213035014

package/template/lambda-sqs-worker/serverless.yml

@@ -1,213 +0,0 @@
-service: <%- serviceName %>
-
-configValidationMode: error
-
-params:
-  default:
-    datadogApiKeySecretArn: 'TODO: arn:aws:secretsmanager:${aws:region}:${aws:accountId}:secret:SECRET-NAME'
-    dataStoreTags:
-      - Key: seek:auto:backup:frequency
-        Value: none
-      # TODO: https://rfc.skinfra.xyz/RFC019-AWS-Tagging-Standard.html#data-services
-      # - Key: seek:data:derived
-      #   Value: copy
-      # - Key: seek:data:domain
-      #   Value: unassigned
-      # - Key: seek:data:jurisdiction-source
-      #   Value: australia hong-kong indonesia malaysia new-zealand philippines singapore thailand
-      # - Key: seek:data:types:restricted
-      #   Value: job-ads
-    description: <%- description %>
-  dev:
-    deploymentBucket: 'TODO: deployment-bucket-name'
-    concurrency: 2
-  prod:
-    deploymentBucket: 'TODO: deployment-bucket-name'
-    concurrency: 20
-
-custom:
-  datadog:
-    addLayers: false
-    apiKeySecretArn: ${param:datadogApiKeySecretArn}
-    enableDDLogs: false
-    # TODO: enable Datadog extension
-    enabled: false
-    exclude:
-      - WorkerPreHook
-    injectLogContext: false
-    version: ${env:VERSION}
-  prune:
-    automatic: true
-    number: 3
-
-plugins:
-  - serverless-plugin-canary-deployments
-  - serverless-plugin-datadog
-  - serverless-prune-plugin
-
-provider:
-  logRetentionInDays: 30
-  name: aws
-  region: ap-southeast-2
-  runtime: nodejs20.x
-  architecture: <%- lambdaServerlessArchitecture %>
-  deploymentMethod: direct
-  stackName: ${self:service}
-  stage: ${env:ENVIRONMENT}
-  deploymentBucket:
-    # Use a shared account-level bucket for Lambda bundles and other artefacts.
-    # This is easier to manage in terms of access, deployment, and tagging.
-    name: ${param:deploymentBucket}
-  environment:
-    NODE_ENV: production
-    # https://nodejs.org/api/cli.html#cli_node_options_options
-    NODE_OPTIONS: --enable-source-maps
-  iam:
-    role:
-      statements:
-        - Action:
-            - kms:Decrypt
-            - kms:GenerateDataKey*
-          Effect: Allow
-          Resource: !GetAtt EncryptionKey.Arn
-        - Action: lambda:InvokeFunction
-          Effect: Allow
-          Resource: !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${self:functions.Worker.name}
-        - Action: secretsmanager:GetSecretValue
-          Effect: Allow
-          Resource: ${param:datadogApiKeySecretArn}-??????
-        - Action: sns:Publish
-          Effect: Allow
-          Resource: !Ref DestinationTopic
-  stackTags:
-    seek:env:label: ${env:ENVIRONMENT}
-    seek:source:sha: ${env:BUILDKITE_COMMIT, 'na'}
-    seek:source:url: 'https://github.com/SEEK-Jobs/<%- repoName %>'
-    seek:system:deployed-by: CFN
-    # seek:system:name: 'TODO: https://rfc.skinfra.xyz/RFC019-AWS-Tagging-Standard.html#required-tags'
-
-package:
-  patterns:
-    - '!**'
-    - lib/**
-    - node_modules/**
-
-functions:
-  Worker:
-    name: ${self:service}
-    handler: lib/app.handler
-    description: ${param:description}
-    memorySize: 128
-    reservedConcurrency: ${param:concurrency}
-    timeout: 30
-    deploymentSettings:
-      type: AllAtOnce
-      alias: Live
-      preTrafficHook: WorkerPreHook
-    environment:
-      ENVIRONMENT: ${env:ENVIRONMENT}
-      SERVICE: ${self:service}
-      VERSION: ${env:VERSION, 'local'}
-
-      DESTINATION_SNS_TOPIC_ARN: !Ref DestinationTopic
-    events:
-      - sqs:
-          arn: !GetAtt MessageQueue.Arn
-          batchSize: 1
-          maximumConcurrency: ${param:concurrency}
-  WorkerPreHook:
-    name: ${self:functions.Worker.name}-pre-hook
-    handler: lib/hooks.pre
-    memorySize: 128
-    # This is generous because a timeout will hang the deployment
-    timeout: 300
-    environment:
-      FUNCTION_NAME_TO_INVOKE: ${self:functions.Worker.name}
-
-resources:
-  # This becomes the Lambda application's description
-  Description: ${param:description}
-
-  Resources:
-    DeadLetterQueue:
-      Type: AWS::SQS::Queue
-      Properties:
-        MessageRetentionPeriod: 1209600
-        KmsDataKeyReusePeriodSeconds: 300
-        KmsMasterKeyId: !Ref EncryptionKey
-        QueueName: ${self:service}-dead-letters
-        Tags: ${param:dataStoreTags}
-
-    MessageQueue:
-      Type: AWS::SQS::Queue
-      Properties:
-        MessageRetentionPeriod: 1209600
-        KmsDataKeyReusePeriodSeconds: 300
-        KmsMasterKeyId: !Ref EncryptionKey
-        QueueName: ${self:service}-messages
-        RedrivePolicy:
-          deadLetterTargetArn: !GetAtt DeadLetterQueue.Arn
-          maxReceiveCount: 3
-        Tags: ${param:dataStoreTags}
-        VisibilityTimeout: 180
-
-    # MessageQueuePolicy:
-    #   Type: AWS::SQS::QueuePolicy
-    #   Properties:
-    #     PolicyDocument:
-    #       Version: '2012-10-17'
-    #       Statement:
-    #         - Action: sqs:SendMessage
-    #           Condition:
-    #             ArnEquals:
-    #               aws:SourceArn: 'TODO: sourceSnsTopicArn'
-    #           Effect: 'Allow'
-    #           Principal:
-    #             AWS: '*'
-    #           Resource: '*'
-    #     Queues:
-    #       - !Ref MessageQueue
-
-    # TopicSubscription:
-    #   Type: AWS::SNS::Subscription
-    #   Properties:
-    #     Endpoint: !GetAtt MessageQueue.Arn
-    #     Protocol: sqs
-    #     RawMessageDelivery: true # Remove this property if you require end to end datadog tracing
-    #     TopicArn: 'TODO: sourceSnsTopicArn'
-
-    DestinationTopic:
-      Type: AWS::SNS::Topic
-      Properties:
-        KmsMasterKeyId: alias/aws/sns
-        Tags: ${param:dataStoreTags}
-        TopicName: ${self:service}
-
-    EncryptionKey:
-      Type: AWS::KMS::Key
-      Properties:
-        Description: ${self:service}
-        EnableKeyRotation: true
-        KeyPolicy:
-          Version: '2012-10-17'
-          Statement:
-            - Sid: Admin access
-              Effect: Allow
-              Action: kms:*
-              Principal:
-                AWS: !Ref AWS::AccountId
-              Resource: '*'
-            - Sid: SNS compatibility
-              Effect: Allow
-              Action:
-                - kms:Decrypt
-                - kms:GenerateDataKey*
-              Principal:
-                Service: sns.amazonaws.com
-              Resource: '*'
-
-    EncryptionKeyAlias:
-      Type: AWS::KMS::Alias
-      Properties:
-        AliasName: alias/seek/${self:service}
-        TargetKeyId: !Ref EncryptionKey

package/template/lambda-sqs-worker/skuba.template.js

@@ -1,33 +0,0 @@
-/**
- * Run `skuba configure` to finish templating and remove this file.
- */
-
-module.exports = {
-  entryPoint: 'src/app.ts#handler',
-  fields: [
-    {
-      name: 'serviceName',
-      message: 'Service slug',
-      initial: 'my-project',
-    },
-    {
-      name: 'description',
-      message: 'Description',
-      initial: 'A project just for me',
-    },
-    {
-      name: 'devBuildkiteQueueName',
-      message: 'Dev Buildkite queue',
-      initial: 'my-team-aws-account-dev:cicd',
-      validate: (value) => /^.+:.+$/.test(value),
-    },
-    {
-      name: 'prodBuildkiteQueueName',
-      message: 'Prod Buildkite queue',
-      initial: 'my-team-aws-account-prod:cicd',
-      validate: (value) => /^.+:.+$/.test(value),
-    },
-  ],
-  packageManager: 'pnpm',
-  type: 'application',
-};

package/template/lambda-sqs-worker/src/app.test.ts

@@ -1,116 +0,0 @@
-import { PublishCommand } from '@aws-sdk/client-sns';
-
-import { metricsClient } from 'src/framework/metrics';
-import { createCtx, createSqsEvent } from 'src/testing/handler';
-import { logger } from 'src/testing/logging';
-import { scoringService, sns } from 'src/testing/services';
-import { chance, mockJobPublishedEvent } from 'src/testing/types';
-
-import * as app from './app';
-
-describe('app', () => {
-  it('exports a handler', () => expect(app).toHaveProperty('handler'));
-});
-
-describe('handler', () => {
-  const ctx = createCtx();
-
-  const jobPublished = mockJobPublishedEvent({ entityId: chance.name() });
-
-  const score = chance.floating({ max: 1, min: 0 });
-
-  const distribution = jest
-    .spyOn(metricsClient, 'distribution')
-    .mockReturnValue();
-
-  beforeAll(logger.spy);
-  beforeAll(scoringService.spy);
-
-  beforeEach(() => {
-    scoringService.request.mockResolvedValue(score);
-    sns.publish.resolves({ MessageId: chance.guid({ version: 4 }) });
-  });
-
-  afterEach(() => {
-    logger.clear();
-    distribution.mockClear();
-    scoringService.clear();
-    sns.clear();
-  });
-
-  it('handles one record', async () => {
-    const event = createSqsEvent([JSON.stringify(jobPublished)]);
-
-    await expect(app.handler(event, ctx)).resolves.toBeUndefined();
-
-    expect(scoringService.request).toHaveBeenCalledTimes(1);
-
-    expect(logger.error).not.toHaveBeenCalled();
-
-    expect(logger.debug.mock.calls).toEqual([
-      [{ count: 1 }, 'Received jobs'],
-      [{ snsMessageId: expect.any(String) }, 'Scored job'],
-      ['Function succeeded'],
-    ]);
-
-    expect(distribution.mock.calls).toEqual([
-      ['job.received', 1],
-      ['job.scored', 1],
-    ]);
-
-    expect(sns.client).toReceiveCommandTimes(PublishCommand, 1);
-  });
-
-  it('throws on invalid input', () => {
-    const event = createSqsEvent(['}']);
-
-    return expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
-  });
-
-  it('bubbles up scoring service error', async () => {
-    const err = Error(chance.sentence());
-
-    scoringService.request.mockRejectedValue(err);
-
-    const event = createSqsEvent([JSON.stringify(jobPublished)]);
-
-    await expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
-
-    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
-  });
-
-  it('bubbles up SNS error', async () => {
-    const err = Error(chance.sentence());
-
-    sns.publish.rejects(err);
-
-    const event = createSqsEvent([JSON.stringify(jobPublished)]);
-
-    await expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
-
-    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
-  });
-
-  it('throws on zero records', async () => {
-    const err = new Error('Received 0 records');
-
-    const event = createSqsEvent([]);
-
-    await expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
-
-    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
-  });
-
-  it('throws on multiple records', async () => {
-    const err = new Error('Received 2 records');
-
-    const event = createSqsEvent([
-      JSON.stringify(jobPublished),
-      JSON.stringify(jobPublished),
-    ]);
-
-    await expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
-
-    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
-  });
-});

package/template/lambda-sqs-worker/src/app.ts

@@ -1,57 +0,0 @@
-import 'skuba-dive/register';
-
-import type { SQSEvent } from 'aws-lambda';
-
-import { createHandler } from 'src/framework/handler';
-import { logger } from 'src/framework/logging';
-import { metricsClient } from 'src/framework/metrics';
-import { validateJson } from 'src/framework/validation';
-import { scoreJobPublishedEvent, scoringService } from 'src/services/jobScorer';
-import { sendPipelineEvent } from 'src/services/pipelineEventSender';
-import { JobPublishedEventSchema } from 'src/types/pipelineEvents';
-
-/**
- * Tests connectivity to ensure appropriate access and network configuration.
- */
-const smokeTest = async () => {
-  await Promise.all([scoringService.smokeTest(), sendPipelineEvent({}, true)]);
-};
-
-export const handler = createHandler<SQSEvent>(async (event) => {
-  // Treat an empty object as our smoke test event.
-  if (!Object.keys(event).length) {
-    logger.debug('Received smoke test request');
-    return smokeTest();
-  }
-
-  const count = event.Records.length;
-
-  if (count !== 1) {
-    throw Error(`Received ${count} records`);
-  }
-
-  logger.debug({ count }, 'Received jobs');
-
-  metricsClient.distribution('job.received', event.Records.length);
-
-  const record = event.Records[0];
-  if (!record) {
-    throw new Error('Malformed SQS event with no records');
-  }
-
-  const { body } = record;
-
-  // TODO: this throws an error, which will cause the Lambda function to retry
-  // the event and eventually send it to your dead-letter queue. If you don't
-  // trust your source to provide consistently well-formed input, consider
-  // catching and handling this error in code.
-  const publishedJob = validateJson(body, JobPublishedEventSchema);
-
-  const scoredJob = await scoreJobPublishedEvent(publishedJob);
-
-  const snsMessageId = await sendPipelineEvent(scoredJob);
-
-  logger.debug({ snsMessageId }, 'Scored job');
-
-  metricsClient.distribution('job.scored', 1);
-});

package/template/lambda-sqs-worker/src/config.ts

@@ -1,62 +0,0 @@
-import { Env } from 'skuba-dive';
-
-interface Config {
-  environment: Environment;
-
-  logLevel: string;
-  metrics: boolean;
-  name: string;
-  version: string;
-
-  destinationSnsTopicArn: string;
-}
-
-type Environment = (typeof environments)[number];
-
-const environments = ['local', 'test', 'dev', 'prod'] as const;
-
-const environment = Env.oneOf(environments)('ENVIRONMENT');
-
-/* istanbul ignore next: config verification makes more sense in a smoke test */
-const configs: Record<Environment, () => Omit<Config, 'environment'>> = {
-  local: () => ({
-    logLevel: 'debug',
-    metrics: false,
-    name: '<%- serviceName %>',
-    version: 'local',
-
-    destinationSnsTopicArn: 'arn:aws:sns:us-east-2:123456789012:destination',
-  }),
-
-  test: () => ({
-    logLevel: Env.string('LOG_LEVEL', { default: 'silent' }),
-    metrics: false,
-    name: '<%- serviceName %>',
-    version: 'test',
-
-    destinationSnsTopicArn: 'arn:aws:sns:us-east-2:123456789012:destination',
-  }),
-
-  dev: () => ({
-    logLevel: 'debug',
-    metrics: true,
-    name: Env.string('SERVICE'),
-    version: Env.string('VERSION'),
-
-    destinationSnsTopicArn: Env.string('DESTINATION_SNS_TOPIC_ARN'),
-  }),
-
-  prod: () => ({
-    logLevel: 'info',
-    metrics: true,
-    name: Env.string('SERVICE'),
-    version: Env.string('VERSION'),
-
-    destinationSnsTopicArn: Env.string('DESTINATION_SNS_TOPIC_ARN'),
-  }),
-};
-
-export const config: Config = {
-  ...configs[environment](),
-  environment,
-};

package/template/lambda-sqs-worker/src/framework/handler.test.ts

@@ -1,61 +0,0 @@
-import { createCtx } from 'src/testing/handler';
-import { logger } from 'src/testing/logging';
-import { chance } from 'src/testing/types';
-
-import { createHandler } from './handler';
-
-describe('createHandler', () => {
-  const ctx = createCtx();
-  const input = chance.paragraph();
-
-  beforeAll(logger.spy);
-
-  afterEach(logger.clear);
-
-  it('handles happy path', async () => {
-    const output = chance.paragraph();
-
-    const handler = createHandler((event) => {
-      expect(event).toBe(input);
-
-      logger.debug('Handler invoked');
-
-      return Promise.resolve(output);
-    });
-
-    await expect(handler(input, ctx)).resolves.toBe(output);
-
-    expect(logger.error).not.toHaveBeenCalled();
-
-    expect(logger.debug.mock.calls).toEqual([
-      ['Handler invoked'],
-      ['Function succeeded'],
-    ]);
-  });
-
-  it('handles async error', async () => {
-    const err = Error(chance.sentence());
-
-    const handler = createHandler(() => Promise.reject(err));
-
-    await expect(handler(input, ctx)).rejects.toThrow('Function failed');
-
-    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
-
-    expect(logger.debug).not.toHaveBeenCalled();
-  });
-
-  it('handles sync error', async () => {
-    const err = Error(chance.sentence());
-
-    const handler = createHandler(() => {
-      throw err;
-    });
-
-    await expect(handler(input, ctx)).rejects.toThrow('Function failed');
-
-    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
-
-    expect(logger.debug).not.toHaveBeenCalled();
-  });
-});

package/template/lambda-sqs-worker/src/framework/handler.ts

@@ -1,43 +0,0 @@
-import { datadog } from 'datadog-lambda-js';
-
-import { config } from 'src/config';
-import { logger, loggerContext } from 'src/framework/logging';
-
-interface LambdaContext {
-  awsRequestId: string;
-}
-
-type Handler<Event, Output> = (
-  event: Event,
-  ctx: LambdaContext,
-) => Promise<Output>;
-
-/**
- * Conditionally applies the Datadog wrapper to a Lambda handler.
- *
- * This also "fixes" its broken type definitions.
- */
-const withDatadog = <Event, Output = unknown>(
-  fn: Handler<Event, Output>,
-): Handler<Event, Output> =>
-  // istanbul ignore next
-  config.metrics ? (datadog(fn) as Handler<Event, Output>) : fn;
-
-export const createHandler = <Event, Output = unknown>(
-  fn: (event: Event) => Promise<Output>,
-) =>
-  withDatadog<Event>((event, { awsRequestId }) =>
-    loggerContext.run({ awsRequestId }, async () => {
-      try {
-        const output = await fn(event);
-
-        logger.debug('Function succeeded');
-
-        return output;
-      } catch (err) {
-        logger.error({ err }, 'Function failed');
-
-        throw new Error('Function failed');
-      }
-    }),
-  );

package/template/lambda-sqs-worker/src/framework/logging.ts

@@ -1,27 +0,0 @@
-import { AsyncLocalStorage } from 'async_hooks';
-
-import createLogger from '@seek/logger';
-
-import { config } from 'src/config';
-
-interface LoggerContext {
-  awsRequestId: string;
-}
-
-export const loggerContext = new AsyncLocalStorage<LoggerContext>();
-
-export const logger = createLogger({
-  base: {
-    environment: config.environment,
-    version: config.version,
-  },
-
-  level: config.logLevel,
-
-  mixin: () => ({ ...loggerContext.getStore() }),
-
-  name: config.name,
-
-  transport:
-    config.environment === 'local' ? { target: 'pino-pretty' } : undefined,
-});

package/template/lambda-sqs-worker/src/framework/metrics.ts

@@ -1,14 +0,0 @@
-import { sendDistributionMetric } from 'datadog-lambda-js';
-
-import { config } from 'src/config';
-
-const prefix = `${config.name}.`;
-
-export const metricsClient = {
-  distribution: (
-    ...[name, ...rest]: Parameters<typeof sendDistributionMetric>
-  ) =>
-    config.metrics
-      ? sendDistributionMetric(`${prefix}${name}`, ...rest)
-      : undefined,
-};