skuba 0.0.0-beta-20231002015947

package/template/koa-rest-api/src/listen.ts

@@ -0,0 +1,24 @@
+import './register';
+
+import app from './app';
+import { config } from './config';
+import { logger } from './framework/logging';
+
+// This implements a minimal version of `koa-cluster`'s interface
+// If your application is deployed with more than 1 vCPU you can delete this
+// file and use `koa-cluster` to run `lib/app`.
+
+const listener = app.listen(config.port, () => {
+  const address = listener.address();
+
+  if (typeof address === 'object' && address) {
+    logger.debug(`listening on port ${address.port}`);
+  }
+});
+
+// Gantry ALB default idle timeout is 30 seconds
+// https://nodejs.org/docs/latest-v18.x/api/http.html#serverkeepalivetimeout
+// Node default is 5 seconds
+// https://docs.aws.amazon.com/elasticloadbalancing/latest/application/application-load-balancers.html#connection-idle-timeout
+// AWS recommends setting an application timeout larger than the load balancer
+listener.keepAliveTimeout = 31000;

package/template/koa-rest-api/src/register.ts

@@ -0,0 +1 @@
+import 'skuba-dive/register';

package/template/koa-rest-api/src/storage/jobs.ts

@@ -0,0 +1,28 @@
+import { randomUUID } from 'crypto';
+
+import type { Job, JobInput } from 'src/types/jobs';
+
+const jobStore: Record<string, Job> = {};
+
+export const createJob = (jobInput: JobInput): Promise<Job> => {
+  const id = randomUUID();
+
+  const job = { ...jobInput, id };
+
+  jobStore[id] = job;
+
+  return Promise.resolve(job);
+};
+
+export const readJobs = (): Promise<Job[]> => {
+  const jobs = Object.values(jobStore);
+
+  return Promise.resolve(jobs);
+};
+
+/**
+ * Tests that we have access to read from and write to our storage.
+ */
+export const smokeTestJobStorage = async (): Promise<void> => {
+  await Promise.resolve();
+};

package/template/koa-rest-api/src/testing/logging.ts

@@ -0,0 +1,16 @@
+import * as logging from 'src/framework/logging';
+
+export const logger = {
+  error: jest.fn(),
+  info: jest.fn(),
+
+  clear: () => {
+    logger.error.mockClear();
+    logger.info.mockClear();
+  },
+
+  spy: () => {
+    jest.spyOn(logging.logger, 'error').mockImplementation(logger.error);
+    jest.spyOn(logging.logger, 'info').mockImplementation(logger.info);
+  },
+};

package/template/koa-rest-api/src/testing/metrics.ts

@@ -0,0 +1,29 @@
+import * as metrics from 'src/framework/metrics';
+
+function assertDefined<T>(value: T | undefined): asserts value is T {
+  expect(value).toBeDefined();
+}
+
+export const metricsClient = {
+  clear: () => (metrics.metricsClient.mockBuffer = []),
+
+  expectTagSubset: (subset: string[], index: number = 0) => {
+    const tags = metricsClient.tags(index);
+
+    subset.forEach((tag) => expect(tags).toContain(tag));
+  },
+
+  tags: (index: number = 0) => {
+    assertDefined(metrics.metricsClient.mockBuffer);
+
+    const line = metrics.metricsClient.mockBuffer[index];
+
+    assertDefined(line);
+
+    const [, tagString] = line.split('#', 2);
+
+    assertDefined(tagString);
+
+    return tagString.split(',');
+  },
+};

package/template/koa-rest-api/src/testing/server.ts

@@ -0,0 +1,31 @@
+import type Router from '@koa/router';
+import type Koa from 'koa';
+import request from 'supertest';
+
+import { createApp } from 'src/framework/server';
+
+/**
+ * Create a new SuperTest agent from a Koa application.
+ */
+export const agentFromApp = <State, Context>(app: Koa<State, Context>) =>
+  request.agent(app.callback());
+
+/**
+ * Create a new SuperTest agent from a set of Koa middleware.
+ */
+export const agentFromMiddleware = <State, Context>(
+  ...middleware: Array<Koa.Middleware<State, Context>>
+) => {
+  const app = createApp(...middleware);
+
+  return agentFromApp(app);
+};
+
+/**
+ * Create a new SuperTest agent from a Koa router.
+ */
+export const agentFromRouter = (router: Router) => {
+  const app = createApp(router.routes(), router.allowedMethods());
+
+  return agentFromApp(app);
+};

package/template/koa-rest-api/src/testing/types.ts

@@ -0,0 +1,24 @@
+import { Chance } from 'chance';
+import { z } from 'zod';
+
+import type { JobInput } from 'src/types/jobs';
+
+export type IdDescription = z.infer<typeof IdDescriptionSchema>;
+
+export const IdDescriptionSchema = z.object({
+  id: z.string(),
+  description: z.string(),
+});
+
+export const chance = new Chance();
+
+export const mockIdDescription = (): IdDescription => ({
+  id: chance.guid({ version: 4 }),
+  description: chance.sentence(),
+});
+
+export const mockJobInput = (): JobInput => ({
+  hirer: {
+    id: chance.guid({ version: 4 }),
+  },
+});

package/template/koa-rest-api/src/tracing.ts

@@ -0,0 +1,56 @@
+/**
+ * OpenTelemetry tracing initialisation. This is a standalone TS/JS module that is not
+ * referenced by application source code directly. It is required at runtime using the
+ * node command's `--require` argument, see Dockerfile for details.
+ */
+
+import { propagation } from '@opentelemetry/api';
+import { CompositePropagator } from '@opentelemetry/core';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
+import { AwsInstrumentation } from '@opentelemetry/instrumentation-aws-sdk';
+import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
+import { B3InjectEncoding, B3Propagator } from '@opentelemetry/propagator-b3';
+import { NodeSDK } from '@opentelemetry/sdk-node';
+
+const app = 'opentelemetry';
+const log = (level: string, msg: string, extra = {}) => {
+  const toLog = { msg, level, app, time: new Date().toISOString(), ...extra };
+  console.log(JSON.stringify(toLog)); // eslint-disable-line no-console
+};
+
+const main = () => {
+  // Use B3 propagation to ensure proper propagation between systems that use
+  // OpenTelemetry and native Datadog APM, such as Istio/Envoy.
+  propagation.setGlobalPropagator(
+    new CompositePropagator({
+      propagators: [
+        new B3Propagator(),
+        new B3Propagator({ injectEncoding: B3InjectEncoding.MULTI_HEADER }),
+      ],
+    }),
+  );
+
+  const sdk = new NodeSDK({
+    traceExporter: new OTLPTraceExporter(),
+    autoDetectResources: false,
+    instrumentations: [new HttpInstrumentation(), new AwsInstrumentation()],
+  });
+
+  sdk.start();
+
+  process.on('SIGTERM', () => {
+    sdk
+      .shutdown()
+      .then(() => log('info', 'OpenTelemetry successfully terminated'))
+      .catch((err: Error) =>
+        log('error', 'OpenTelemetry failed to terminate', { err }),
+      )
+      .finally(() => process.exit(0)); // eslint-disable-line no-process-exit
+  });
+};
+
+if (process.env.OPENTELEMETRY_ENABLED === 'true') {
+  main();
+} else {
+  log('info', 'OpenTelemetry not enabled');
+}

package/template/koa-rest-api/src/types/jobs.ts

@@ -0,0 +1,17 @@
+import { z } from 'zod';
+
+export interface Job {
+  id: string;
+
+  hirer: {
+    id: string;
+  };
+}
+
+export type JobInput = z.infer<typeof JobInputSchema>;
+
+export const JobInputSchema = z.object({
+  hirer: z.object({
+    id: z.string(),
+  }),
+});

package/template/koa-rest-api/src/types/koa.ts

@@ -0,0 +1,8 @@
+import type Koa from 'koa';
+import type { RequestLogging } from 'seek-koala';
+
+export type Context = Koa.ParameterizedContext<State>;
+
+export type Middleware = Koa.Middleware<State>;
+
+export type State = RequestLogging.State;

package/template/koa-rest-api/tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "baseUrl": ".",
+    "lib": [
+      // open-telemetry/opentelemetry-js#3580
+      "DOM",
+
+      "ES2022"
+    ],
+    "outDir": "lib",
+    "paths": {
+      "src": ["src"]
+    },
+    "target": "ES2022"
+  },
+  "exclude": ["lib*/**/*"],
+  "extends": "skuba/config/tsconfig.json"
+}

package/template/lambda-sqs-worker/.buildkite/pipeline.yml

@@ -0,0 +1,98 @@
+agents:
+  queue: <%- prodBuildkiteQueueName %>
+
+configs:
+  plugins:
+    - &aws-sm
+      seek-oss/aws-sm#v2.3.1:
+        env:
+          NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
+
+    - &docker-ecr-cache
+      seek-oss/docker-ecr-cache#v2.1.0: &docker-ecr-cache-defaults
+        cache-on:
+          - package.json
+          - yarn.lock
+        secrets: id=npm,src=.npmrc
+
+    - &private-npm
+      seek-oss/private-npm#v1.2.0:
+        env: NPM_READ_TOKEN
+
+  base-steps:
+    - &deploy
+      commands:
+        - echo '+++ yarn deploy'
+        - yarn deploy
+      concurrency: 1
+      plugins:
+        - artifacts#v1.9.2:
+            build: ${BUILDKITE_BUILD_ID}
+            download: lib/*
+        - *aws-sm
+        - *private-npm
+        - *docker-ecr-cache
+        - docker-compose#v4.14.0:
+            dependencies: false
+            run: app
+      retry:
+        manual:
+          # Only use this if you need to roll back a deployment ASAP.
+          # Always follow up with a proper revert or fix in Git history.
+          permit_on_passed: true
+
+env:
+  VERSION: ${BUILDKITE_COMMIT:0:7}.${BUILDKITE_BUILD_NUMBER}
+
+steps:
+  - label: 🧪 Test, Lint & Build
+    artifact_paths: lib/**/*
+    commands:
+      - echo '+++ yarn test:ci'
+      - yarn test:ci
+      - echo '--- yarn lint'
+      - yarn lint
+      - echo '--- yarn build'
+      - yarn build
+    env:
+      GET_GITHUB_TOKEN: please
+    plugins:
+      - *aws-sm
+      - *private-npm
+      - *docker-ecr-cache
+      - docker-compose#v4.14.0:
+          run: app
+    timeout_in_minutes: 10
+
+  - agents:
+      queue: <%- devBuildkiteQueueName %>
+    branches: '!renovate-*'
+    label: 🧖‍♀️ Warm Dev
+    command: ':'
+    plugins:
+      - *aws-sm
+      - *private-npm
+      - seek-oss/docker-ecr-cache#v2.1.0:
+          <<: *docker-ecr-cache-defaults
+          skip-pull-from-cache: true
+
+  - wait
+  - block: 🙋🏻‍♀️ Deploy Dev
+    branches: '!${BUILDKITE_PIPELINE_DEFAULT_BRANCH}'
+
+  - <<: *deploy
+    agents:
+      queue: <%- devBuildkiteQueueName %>
+    env:
+      ENVIRONMENT: dev
+    concurrency_group: <%- repoName %>/deploy/dev
+    label: 🤞 Deploy Dev
+
+  - wait
+
+  - <<: *deploy
+    env:
+      ENVIRONMENT: prod
+    branches: ${BUILDKITE_PIPELINE_DEFAULT_BRANCH}
+    concurrency_group: <%- repoName %>/deploy/prod
+    label: 🚀 Deploy Prod

package/template/lambda-sqs-worker/.env

@@ -0,0 +1 @@
+ENVIRONMENT=local

package/template/lambda-sqs-worker/.nvmrc

@@ -0,0 +1 @@
+18

package/template/lambda-sqs-worker/Dockerfile

@@ -0,0 +1,12 @@
+# syntax=docker/dockerfile:1.2
+
+FROM --platform=${BUILDPLATFORM:-<%- platformName %>} node:18-alpine AS dev-deps
+
+WORKDIR /workdir
+
+COPY package.json yarn.lock ./
+
+RUN \
+  --mount=type=secret,id=npm,dst=/workdir/.npmrc \
+  yarn install --frozen-lockfile --ignore-optional --non-interactive && \
+  yarn cache clean

package/template/lambda-sqs-worker/README.md

@@ -0,0 +1,132 @@
+# <%- repoName %>
+
+[![Powered by skuba](https://img.shields.io/badge/🤿%20skuba-powered-009DC4)](https://github.com/seek-oss/skuba)
+
+Next steps:
+
+1. [ ] Finish templating if this was skipped earlier:
+
+   ```shell
+   yarn skuba configure
+   ```
+
+2. [ ] Create a new repository in the appropriate GitHub organisation.
+3. [ ] Add the repository to BuildAgency;
+       see [Builds at SEEK] for more information.
+4. [ ] Add Datadog extension, deployment bucket configuration and data classification tags to [serverless.yml](serverless.yml).
+5. [ ] Push local commits to the upstream GitHub branch.
+6. [ ] Configure [GitHub repository settings].
+7. [ ] Delete this checklist 😌.
+
+[builds at seek]: https://builds-at-seek.ssod.skinfra.xyz
+[github repository settings]: https://github.com/<%-orgName%>/<%-repoName%>/settings
+
+## Design
+
+<%-repoName %> is a Node.js [Lambda] application built in line with our [technology strategy].
+It is backed by a typical SQS message + dead letter queue configuration and uses common SEEK packages.
+Workers enable fault-tolerant asynchronous processing of events.
+
+The `lambda-sqs-worker` template is modelled after a hypothetical enricher that scores job advertisements.
+It's stubbed out with in-memory [scoring service](src/services/jobScorer.ts).
+This would be replaced with internal logic or an external service in production.
+
+This project is deployed with [Serverless].
+The Lambda runtime provisions a single Node.js process per container.
+The supplied [serverless.yml](serverless.yml) starts out with a minimal `memorySize` which may require tuning based on workload.
+Under load, we autoscale horizontally in terms of container count up to `reservedConcurrency`.
+
+Serverless configures [CodeDeploy] for a blue-green deployment approach.
+A smoke test is run against the new version before traffic is switched over,
+providing an opportunity to test access and connectivity to online dependencies.
+This defaults to an invocation with an empty object `{}`, per [src/hooks.ts](src/hooks.ts).
+
+## Development
+
+### Test
+
+```shell
+# Run Jest tests locally
+yarn test
+
+# Authenticate to dev account
+awsauth
+
+# Run smoke test against deployed application
+ENVIRONMENT=dev yarn smoke
+```
+
+### Lint
+
+```shell
+# Fix issues
+yarn format
+
+# Check for issues
+yarn lint
+```
+
+### Start
+
+```shell
+# Start a local HTTP server
+yarn start
+
+# Start with Node.js Inspector enabled
+yarn start:debug
+```
+
+This serves the Lambda application over HTTP.
+For example, to invoke the handler with an empty object `{}` for smoke testing:
+
+```shell
+curl --data '[{}, {"awsRequestId": "local"}]' --include localhost:<%- port %>
+```
+
+### Deploy
+
+This project is deployed through a [Buildkite pipeline](.buildkite/pipeline.yml).
+
+- Commits to a feature branch can be deployed to the dev environment by unblocking a step in the Buildkite UI
+- Commits to the default branch are automatically deployed to the dev and prod environments in sequence
+
+To deploy locally:
+
+```shell
+# Authenticate to dev account
+awsauth
+
+ENVIRONMENT=dev yarn deploy
+```
+
+To rapidly roll back a change,
+retry an individual deployment step from the previous build in Buildkite.
+Note that this will introduce drift between the head of the default Git branch and the live environment;
+use with caution and always follow up with a proper revert or fix in Git history.
+
+## Support
+
+### Dev
+
+TODO: add support links for the dev environment.
+
+<!--
+- CloudWatch dashboard
+- Datadog dashboard
+- Splunk logs
+-->
+
+### Prod
+
+TODO: add support links for the prod environment.
+
+<!--
+- CloudWatch dashboard
+- Datadog dashboard
+- Splunk logs
+-->
+
+[codedeploy]: https://docs.aws.amazon.com/codedeploy
+[lambda]: https://docs.aws.amazon.com/lambda
+[serverless]: https://www.serverless.com/
+[technology strategy]: https://tech-strategy.ssod.skinfra.xyz

package/template/lambda-sqs-worker/docker-compose.yml

@@ -0,0 +1,27 @@
+version: '3.7'
+
+services:
+  app:
+    environment:
+      # Enable Buildkite + GitHub integrations.
+      - BUILDKITE
+      - BUILDKITE_AGENT_ACCESS_TOKEN
+      - BUILDKITE_BRANCH
+      - BUILDKITE_BUILD_NUMBER
+      - BUILDKITE_JOB_ID
+      - BUILDKITE_PIPELINE_DEFAULT_BRANCH
+      - BUILDKITE_STEP_ID
+      - GITHUB_API_TOKEN
+      # Tag AWS resources with the commit hash.
+      - BUILDKITE_COMMIT
+      # Pass through application configuration.
+      - ENVIRONMENT
+      - VERSION
+    image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''}
+    init: true
+    volumes:
+      - ./:/workdir
+      # Mount agent for Buildkite annotations.
+      - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent
+      # Mount cached dependencies.
+      - /workdir/node_modules

package/template/lambda-sqs-worker/package.json

@@ -0,0 +1,44 @@
+{
+  "private": true,
+  "license": "UNLICENSED",
+  "scripts": {
+    "build": "skuba build",
+    "deploy": "serverless deploy --force --verbose",
+    "format": "skuba format",
+    "lint": "skuba lint",
+    "smoke": "serverless invoke --data '{}' --function Worker",
+    "start": "skuba start --port <%- port %>",
+    "start:debug": "yarn start --inspect-brk",
+    "test": "skuba test",
+    "test:ci": "skuba test --coverage",
+    "test:watch": "skuba test --watch"
+  },
+  "dependencies": {
+    "@aws-sdk/client-codedeploy": "^3.363.0",
+    "@aws-sdk/client-lambda": "^3.363.0",
+    "@aws-sdk/client-sns": "^3.363.0",
+    "@aws-sdk/util-utf8-node": "^3.259.0",
+    "@seek/logger": "^6.0.0",
+    "datadog-lambda-js": "^7.0.0",
+    "dd-trace": "^4.0.0",
+    "skuba-dive": "^2.0.0",
+    "zod": "^3.19.1"
+  },
+  "devDependencies": {
+    "@types/aws-lambda": "^8.10.84",
+    "@types/chance": "^1.1.3",
+    "@types/node": "^18.11.9",
+    "aws-sdk-client-mock": "^3.0.0",
+    "aws-sdk-client-mock-jest": "^3.0.0",
+    "chance": "^1.1.8",
+    "pino-pretty": "^10.0.0",
+    "serverless": "^3.25.0",
+    "serverless-plugin-canary-deployments": "^0.8.0",
+    "serverless-plugin-datadog": "^5.12.0",
+    "serverless-prune-plugin": "^2.0.0",
+    "skuba": "*"
+  },
+  "engines": {
+    "node": ">=18.12"
+  }
+}