skuba 0.0.0-master-20230318044948

package/template/express-rest-api/.buildkite/pipeline.yml

@@ -0,0 +1,103 @@
+agents:
+  queue: <%- prodBuildkiteQueueName %>
+
+configs:
+  plugins:
+    - &aws-sm
+      seek-oss/aws-sm#v2.3.1:
+        env:
+          NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
+
+    - &docker-ecr-cache
+      seek-oss/docker-ecr-cache#v2.0.0:
+        cache-on:
+          - package.json
+          - yarn.lock
+        dockerfile: Dockerfile.dev-deps
+        secrets: id=npm,src=.npmrc
+
+    - &private-npm
+      seek-oss/private-npm#v1.2.0:
+        env: NPM_READ_TOKEN
+
+  base-steps:
+    - &deploy
+      concurrency: 1
+      retry:
+        manual:
+          # Only use this if you need to roll back a deployment ASAP.
+          # Always follow up with a proper revert or fix in Git history.
+          permit_on_passed: true
+
+steps:
+  - label: 🧖‍♀️ Warm Prod
+    command: ':'
+    key: warm-prod
+    plugins:
+      - *aws-sm
+      - *private-npm
+      - *docker-ecr-cache
+
+  - label: 🧪 Test & Lint
+    commands:
+      - echo '+++ yarn test:ci'
+      - yarn test:ci
+      - echo '--- yarn lint'
+      - yarn lint
+    depends_on: warm-prod
+    env:
+      GET_GITHUB_TOKEN: please
+    plugins:
+      - *aws-sm
+      - *private-npm
+      - *docker-ecr-cache
+      - docker-compose#v4.11.0:
+          run: app
+    timeout_in_minutes: 10
+
+  - label: 📦 Build & Package
+    depends_on: warm-prod
+    plugins:
+      - *aws-sm
+      - *private-npm
+      - *docker-ecr-cache
+      - seek-jobs/gantry#v2.0.0:
+          command: build
+          file: gantry.build.yml
+          region: <%- region %>
+          values: .gantry/common.yml
+
+  - wait
+  - block: 🙋🏻‍♀️ Deploy Dev
+    branches: '!${BUILDKITE_PIPELINE_DEFAULT_BRANCH}'
+
+  - <<: *deploy
+    agents:
+      queue: <%- devBuildkiteQueueName %>
+    label: 🤞 Deploy Dev
+    concurrency_group: <%- teamName %>/deploy/gantry/<%- devGantryEnvironmentName %>
+    key: deploy-dev
+    plugins:
+      - seek-jobs/gantry#v2.0.0:
+          command: apply
+          environment: <%- devGantryEnvironmentName %>
+          file: gantry.apply.yml
+          region: <%- region %>
+          values:
+            - .gantry/common.yml
+            - .gantry/dev.yml
+
+  - <<: *deploy
+    label: 🚀 Deploy Prod
+    branches: ${BUILDKITE_PIPELINE_DEFAULT_BRANCH}
+    concurrency_group: <%- teamName %>/deploy/gantry/<%- prodGantryEnvironmentName %>
+    depends_on: deploy-dev
+    plugins:
+      - seek-jobs/gantry#v2.0.0:
+          command: apply
+          environment: <%- prodGantryEnvironmentName %>
+          file: gantry.apply.yml
+          region: <%- region %>
+          values:
+            - .gantry/common.yml
+            - .gantry/prod.yml

package/template/express-rest-api/.env

@@ -0,0 +1 @@
+ENVIRONMENT=local

package/template/express-rest-api/.gantry/common.yml

@@ -0,0 +1,20 @@
+prodAccountId: '<%- prodAwsAccountId %>'
+
+image: '{{values "prodAccountId"}}.dkr.ecr.<%- region %>.amazonaws.com/{{values "service"}}:{{.BuildID}}'
+service: '<%- serviceName %>'
+
+# TODO: enable Datadog agent
+# https://backstage.myseek.xyz/docs/default/component/gantry/v1/reference/resources/service/#datadogSecretId
+# datadogSecretId: arn:aws:secretsmanager:<%- region %>:<aws-account-id>:secret:<secret-name>
+
+tags:
+  # TODO: add data classification tags
+  # https://rfc.skinfra.xyz/RFC019-AWS-Tagging-Standard.html#seekdataconsumers
+  # seek:data:consumers: internal
+  # https://rfc.skinfra.xyz/RFC019-AWS-Tagging-Standard.html#seekdatatypes
+  # seek:data:types:restricted: job-ads
+  seek:env:label: '{{values "environment"}}'
+  seek:env:production: '{{values "isProduction"}}'
+  seek:owner:team: '<%- teamName %>'
+  seek:source:sha: '{{.CommitSHA}}'
+  seek:source:url: 'https://github.com/SEEK-Jobs/<%- repoName %>'

package/template/express-rest-api/.gantry/dev.yml

@@ -0,0 +1,8 @@
+environment: dev
+env:
+  SOME_ENVIRONMENT_VARIABLE: dev-value
+
+isProduction: false
+
+maxInstanceCount: 1
+minInstanceCount: 1

package/template/express-rest-api/.gantry/prod.yml

@@ -0,0 +1,8 @@
+environment: prod
+env:
+  SOME_ENVIRONMENT_VARIABLE: prod-value
+
+isProduction: true
+
+maxInstanceCount: 10
+minInstanceCount: 3

package/template/express-rest-api/.nvmrc

@@ -0,0 +1 @@
+18

package/template/express-rest-api/Dockerfile

@@ -0,0 +1,37 @@
+ARG BASE_IMAGE
+ARG BASE_TAG
+
+###
+
+FROM ${BASE_IMAGE}:${BASE_TAG} AS deps
+
+RUN yarn install --ignore-optional --ignore-scripts --non-interactive --offline --production
+
+###
+
+FROM ${BASE_IMAGE}:${BASE_TAG} AS build
+
+COPY . .
+
+RUN yarn build
+
+###
+
+FROM --platform=${BUILDPLATFORM:-<%- platformName %>} gcr.io/distroless/nodejs:18 AS runtime
+
+WORKDIR /workdir
+
+COPY --from=build /workdir/lib lib
+
+COPY --from=deps /workdir/node_modules node_modules
+
+ENV NODE_ENV=production
+
+# https://nodejs.org/api/cli.html#cli_node_options_options
+ENV NODE_OPTIONS=--enable-source-maps
+
+ARG PORT=8001
+ENV PORT=${PORT}
+EXPOSE ${PORT}
+
+CMD ["lib/listen.js"]

package/template/express-rest-api/Dockerfile.dev-deps

@@ -0,0 +1,11 @@
+# syntax=docker/dockerfile:1.2
+
+FROM --platform=${BUILDPLATFORM:-<%- platformName %>} node:18-alpine AS dev-deps
+
+WORKDIR /workdir
+
+COPY package.json yarn.lock ./
+
+RUN \
+  --mount=type=secret,id=npm,dst=/workdir/.npmrc \
+  yarn install --frozen-lockfile --ignore-optional --non-interactive

package/template/express-rest-api/README.md

@@ -0,0 +1,106 @@
+# <%- repoName %>
+
+[![Powered by skuba](https://img.shields.io/badge/🤿%20skuba-powered-009DC4)](https://github.com/seek-oss/skuba)
+
+Next steps:
+
+1. [ ] Finish templating if this was skipped earlier:
+
+   ```shell
+   yarn skuba configure
+   ```
+
+2. [ ] Create a new repository in the appropriate GitHub organisation.
+3. [ ] Add the repository to BuildAgency;
+       see [Builds at SEEK] for more information.
+4. [ ] Add Datadog configuration and data classification tags to [.gantry/common.yml](.gantry/common.yml);
+       see the [Gantry] documentation for more information.
+5. [ ] Push local commits to the upstream GitHub branch.
+6. [ ] Configure [GitHub repository settings].
+7. [ ] Delete this checklist 😌.
+
+[builds at seek]: https://builds-at-seek.ssod.skinfra.xyz
+[github repository settings]: https://github.com/<%-orgName%>/<%-repoName%>/settings
+
+## Design
+
+<%-repoName %> is a Node.js HTTP server built in line with our [technology strategy].
+It uses the [Express] middleware framework and common SEEK packages.
+Resource APIs enable synchronous interactions and serve as the backbone of SEEK's general service architecture.
+
+This project is deployed as a containerised application with [Gantry].
+A typical resource API instance does not require more than 1 vCPU,
+so we eschew clustering configurations in favour of a single Node.js process per container.
+Under load, we autoscale horizontally in terms of container count up to `autoScaling.maxCount`.
+
+Gantry configures [CodeDeploy] for a blue-green deployment approach.
+A smoke test is run against the new version before traffic is switched over,
+providing an opportunity to test access and connectivity to online dependencies.
+This defaults to an HTTP request to the `GET /smoke` endpoint.
+
+## Development
+
+### Test
+
+```shell
+yarn test
+```
+
+### Lint
+
+```shell
+# fix
+yarn format
+
+# check
+yarn lint
+```
+
+### Start
+
+```shell
+# Start a local HTTP server
+yarn start
+
+# Start with Node.js Inspector enabled
+yarn start:debug
+```
+
+### Deploy
+
+This project is deployed through a [Buildkite pipeline](.buildkite/pipeline.yml).
+
+- Commits to a feature branch can be deployed to the dev environment by unblocking a step in the Buildkite UI
+- Commits to the default branch are automatically deployed to the dev and prod environments in sequence
+
+To rapidly roll back a change,
+retry an individual deployment step from the previous build in Buildkite.
+Note that this will introduce drift between the head of the default Git branch and the live environment;
+use with caution and always follow up with a proper revert or fix in Git history.
+
+## Support
+
+### Dev
+
+TODO: add support links for the dev environment.
+
+<!--
+- CloudWatch dashboard
+- Datadog dashboard
+- Splunk logs
+-->
+
+### Prod
+
+TODO: add support links for the prod environment.
+
+<!--
+- CloudWatch dashboard
+- Datadog dashboard
+- Splunk logs
+-->
+
+[codedeploy]: https://docs.aws.amazon.com/codedeploy
+[express]: https://expressjs.com
+[gantry]: https://backstage.myseek.xyz/docs/default/component/gantry/
+[technology strategy]: https://tech-strategy.ssod.skinfra.xyz

package/template/express-rest-api/docker-compose.yml

@@ -0,0 +1,22 @@
+version: '3.7'
+
+services:
+  app:
+    environment:
+      # Enable Buildkite + GitHub integrations.
+      - BUILDKITE
+      - BUILDKITE_AGENT_ACCESS_TOKEN
+      - BUILDKITE_BRANCH
+      - BUILDKITE_BUILD_NUMBER
+      - BUILDKITE_JOB_ID
+      - BUILDKITE_PIPELINE_DEFAULT_BRANCH
+      - BUILDKITE_STEP_ID
+      - GITHUB_API_TOKEN
+    image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''}
+    init: true
+    volumes:
+      - ./:/workdir
+      # Mount agent for Buildkite annotations.
+      - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent
+      # Mount cached dependencies.
+      - /workdir/node_modules

package/template/express-rest-api/gantry.apply.yml

@@ -0,0 +1,122 @@
+kind: Service
+
+schemaVersion: v0.0
+
+name: '{{values "service"}}'
+
+image: '{{values "image"}}'
+
+env:
+  # https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/node-reusing-connections.html
+  AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1'
+
+  ENVIRONMENT: '{{values "environment"}}'
+  SERVICE: '{{values "service"}}'
+
+  {{range $key, $value := .Values.env}}
+  {{$key}}: {{$value}}
+  {{end}}
+
+{{if .Values.datadogSecretId}}
+datadogSecretId: '{{values "datadogSecretId"}}'
+{{end}}
+
+{{if .Values.pagerDutyEndpoint}}
+pagerDutyEndpoint: '{{values "pagerDutyEndpoint"}}'
+{{end}}
+
+authentication:
+  ignoreRoutes:
+    - /health
+    - /smoke
+
+sslEnabled: true
+
+autoScaling:
+  maxCount: {{values "maxInstanceCount"}}
+  minCount: {{values "minInstanceCount"}}
+
+  cpuThreshold: 50
+  memoryThreshold: 50
+
+  # Disable Gantry's request-based scaling; the load a request generates can be
+  # highly variable. Instead, scale out when *either* CPU or memory are above
+  # the threshold, and scale in when *both* are 5% below the threshold.
+  #
+  # See https://docs.aws.amazon.com/autoscaling/application/userguide/application-auto-scaling-target-tracking.html
+  autoScalingPolicies:
+    CpuAutoScalingPolicy:
+      Type: AWS::ApplicationAutoScaling::ScalingPolicy
+      Properties:
+        PolicyName: CpuAutoScalingPolicy
+        PolicyType: TargetTrackingScaling
+        ScalingTargetId:
+          Ref: AutoScalingTarget
+        TargetTrackingScalingPolicyConfiguration:
+          ScaleInCooldown: 300
+          ScaleOutCooldown: 60
+          TargetValue:
+            Ref: CpuUtilisationAutoScalingThreshold
+          PredefinedMetricSpecification:
+            PredefinedMetricType: ECSServiceAverageCPUUtilization
+
+    MemoryAutoScalingPolicy:
+      Type: AWS::ApplicationAutoScaling::ScalingPolicy
+      Properties:
+        PolicyName: MemoryAutoScalingPolicy
+        PolicyType: TargetTrackingScaling
+        ScalingTargetId:
+          Ref: AutoScalingTarget
+        TargetTrackingScalingPolicyConfiguration:
+          ScaleInCooldown: 300
+          ScaleOutCooldown: 60
+          TargetValue:
+            Ref: MemoryUtilisationAutoScalingThreshold
+          PredefinedMetricSpecification:
+            PredefinedMetricType: ECSServiceAverageMemoryUtilization
+
+    RequestsScaleOutAlarm:
+      Type: AWS::CloudWatch::Alarm
+      Properties:
+        AlarmDescription: Stub alarm for Gantry to tag
+        Metrics:
+          - Id: missing
+            Expression: SUM(METRICS())
+            Period: 60
+            ReturnData: true
+        ActionsEnabled: false
+        EvaluationPeriods: 1
+        Threshold: 0
+        ComparisonOperator: LessThanThreshold
+
+    RequestsScaleInAlarm:
+      Type: AWS::CloudWatch::Alarm
+      Properties:
+        AlarmDescription: Stub alarm for Gantry to tag
+        Metrics:
+          - Id: missing
+            Expression: SUM(METRICS())
+            Period: 60
+            ReturnData: true
+        ActionsEnabled: false
+        EvaluationPeriods: 1
+        Threshold: 0
+        ComparisonOperator: LessThanThreshold
+
+
+cpu: 256
+memory: 512
+
+deployment:
+  # SEEK-Jobs/gantry#488
+  ignoreAlarms: true
+  smokeTest:
+    path: /smoke
+    useExternalDns: true
+
+tags:
+  {{range $key, $value := .Values.tags}}
+  {{$key}}: '{{$value}}'
+  {{end}}
+
+cpuArchitecture: <%- platformName %>

package/template/express-rest-api/gantry.build.yml

@@ -0,0 +1,15 @@
+kind: ContainerImage
+
+schemaVersion: v0.0
+
+image: '{{values "image"}}'
+
+buildArgs:
+  # https://github.com/seek-oss/docker-ecr-cache-buildkite-plugin#building-on-the-resulting-image
+  BASE_IMAGE: '{{.Env.BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_IMAGE}}'
+  BASE_TAG: '{{.Env.BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_TAG}}'
+
+# SEEK-Jobs/gantry#1661
+failOnScanFindings: false
+
+cpuArchitecture: <%- platformName %>

package/template/express-rest-api/package.json

@@ -0,0 +1,30 @@
+{
+  "private": true,
+  "license": "UNLICENSED",
+  "scripts": {
+    "build": "skuba build",
+    "format": "skuba format",
+    "lint": "skuba lint",
+    "start": "skuba start --port <%- port %>",
+    "start:debug": "yarn start --inspect-brk",
+    "test": "skuba test",
+    "test:ci": "skuba test --coverage",
+    "test:watch": "skuba test --watch"
+  },
+  "dependencies": {
+    "@seek/logger": "^5.0.1",
+    "express": "^4.17.1",
+    "skuba-dive": "^2.0.0"
+  },
+  "devDependencies": {
+    "@types/express": "^4.17.13",
+    "@types/node": "^18.11.5",
+    "@types/supertest": "^2.0.11",
+    "pino-pretty": "^10.0.0",
+    "skuba": "*",
+    "supertest": "^6.1.6"
+  },
+  "engines": {
+    "node": ">=18"
+  }
+}

package/template/express-rest-api/skuba.template.js

@@ -0,0 +1,49 @@
+/**
+ * Run `skuba configure` to finish templating and remove this file.
+ */
+
+module.exports = {
+  entryPoint: 'src/app.ts',
+  fields: [
+    {
+      name: 'serviceName',
+      message: 'Service slug',
+      initial: 'my-project',
+    },
+    {
+      name: 'devBuildkiteQueueName',
+      message: 'Dev Buildkite queue',
+      initial: 'my-team-aws-account-dev:cicd',
+      validate: (value) => /^.+:.+$/.test(value),
+    },
+    {
+      name: 'prodBuildkiteQueueName',
+      message: 'Prod Buildkite queue',
+      initial: 'my-team-aws-account-prod:cicd',
+      validate: (value) => /^.+:.+$/.test(value),
+    },
+    {
+      name: 'devGantryEnvironmentName',
+      message: 'Dev Gantry environment',
+      initial: 'my-gantry-dev',
+    },
+    {
+      name: 'prodGantryEnvironmentName',
+      message: 'Prod Gantry environment',
+      initial: 'my-gantry-prod',
+    },
+    {
+      name: 'prodAwsAccountId',
+      message: 'Prod AWS account',
+      initial: '123456789012',
+      validate: (value) => /^\d{12}$/.test(value),
+    },
+    {
+      name: 'region',
+      message: 'AWS region',
+      initial: 'my-region-0',
+      validate: (value) => /^[a-z]{2}-[a-z]+-\d+$/.test(value),
+    },
+  ],
+  type: 'application',
+};

package/template/express-rest-api/src/api/healthCheck.ts

@@ -0,0 +1,11 @@
+import { Handler } from 'express';
+
+/**
+ * Signifies that the API is available to serve requests.
+ *
+ * The deployment environment calls this endpoint to see if the container is
+ * unhealthy and needs to be recycled.
+ */
+export const healthCheckHandler: Handler = (_req, res) => {
+  res.send('');
+};

package/template/express-rest-api/src/api/smokeTest.ts

@@ -0,0 +1,8 @@
+import { Handler } from 'express';
+
+/**
+ * Tests connectivity to ensure appropriate access and network configuration.
+ */
+export const smokeTestHandler: Handler = (_req, res) => {
+  res.send('');
+};

package/template/express-rest-api/src/app.test.ts

@@ -0,0 +1,9 @@
+import request from 'supertest';
+
+import app from './app';
+
+const agent = request.agent(app);
+
+describe('app', () => {
+  it('has a happy health check', () => agent.get('/health').expect(200, ''));
+});

package/template/express-rest-api/src/app.ts

@@ -0,0 +1,14 @@
+import './register';
+
+import express from 'express';
+
+import { healthCheckHandler } from './api/healthCheck';
+import { smokeTestHandler } from './api/smokeTest';
+
+const app = express()
+  // TODO: consider using a middleware that adds secure HTTP headers.
+  // https://github.com/helmetjs/helmet
+  .get('/health', healthCheckHandler)
+  .get('/smoke', smokeTestHandler);
+
+export default app;

package/template/express-rest-api/src/config.ts

@@ -0,0 +1,57 @@
+import { Env } from 'skuba-dive';
+
+interface Config {
+  environment: Environment;
+
+  logLevel: string;
+  name: string;
+  version: string;
+
+  metricsServer?: string;
+  port?: number;
+}
+
+type Environment = (typeof environments)[number];
+
+const dev = '<%- devGantryEnvironmentName %>';
+const prod = '<%- prodGantryEnvironmentName %>';
+
+const environments = ['local', 'test', dev, prod] as const;
+
+const environment = Env.oneOf(environments)('ENVIRONMENT');
+
+/* istanbul ignore next: config verification makes more sense in a smoke test */
+const configs: Record<Environment, () => Omit<Config, 'environment'>> = {
+  local: () => ({
+    logLevel: 'debug',
+    name: '<%- serviceName %>',
+    version: 'local',
+  }),
+
+  test: () => ({
+    ...configs.local(),
+
+    logLevel: Env.string('LOG_LEVEL', { default: 'silent' }),
+    version: 'test',
+  }),
+
+  [dev]: () => ({
+    ...configs[prod](),
+
+    logLevel: 'debug',
+  }),
+
+  [prod]: () => ({
+    logLevel: 'info',
+    name: Env.string('SERVICE'),
+    version: Env.string('VERSION'),
+
+    metricsServer: 'localhost',
+    port: Env.nonNegativeInteger('PORT'),
+  }),
+};
+
+export const config: Config = {
+  ...configs[environment](),
+  environment,
+};

package/template/express-rest-api/src/framework/logging.ts

@@ -0,0 +1,17 @@
+import createLogger from '@seek/logger';
+
+import { config } from 'src/config';
+
+export const rootLogger = createLogger({
+  base: {
+    environment: config.environment,
+    version: config.version,
+  },
+
+  level: config.logLevel,
+
+  name: config.name,
+
+  transport:
+    config.environment === 'local' ? { target: 'pino-pretty' } : undefined,
+});