skuba 0.0.0-beta-20231002015947

package/template/greeter/package.json

@@ -0,0 +1,24 @@
+{
+  "private": true,
+  "license": "UNLICENSED",
+  "scripts": {
+    "build": "skuba build",
+    "format": "skuba format",
+    "lint": "skuba lint",
+    "start": "skuba start",
+    "start:debug": "yarn start --inspect-brk",
+    "test": "skuba test",
+    "test:ci": "skuba test --coverage",
+    "test:watch": "skuba test --watch"
+  },
+  "dependencies": {
+    "skuba-dive": "^2.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^18.11.5",
+    "skuba": "*"
+  },
+  "engines": {
+    "node": ">=18.12"
+  }
+}

package/template/greeter/skuba.template.js

@@ -0,0 +1,16 @@
+/**
+ * Run `skuba configure` to finish templating and remove this file.
+ */
+
+module.exports = {
+  entryPoint: 'src/app.ts',
+  fields: [
+    {
+      name: 'prodBuildkiteQueueName',
+      message: 'Prod Buildkite queue',
+      initial: 'my-team-aws-account-prod:cicd',
+      validate: (value) => /^.+:.+$/.test(value),
+    },
+  ],
+  type: 'application',
+};

package/template/greeter/src/app.test.ts

@@ -0,0 +1,5 @@
+import * as app from './app';
+
+describe('app', () => {
+  it('is importable', () => expect(app).toEqual({}));
+});

package/template/greeter/src/app.ts

@@ -0,0 +1,4 @@
+import 'skuba-dive/register';
+
+/* eslint-disable-next-line no-console */
+console.log('Hello world');

package/template/koa-rest-api/.buildkite/pipeline.yml

@@ -0,0 +1,105 @@
+agents:
+  queue: <%- prodBuildkiteQueueName %>
+
+configs:
+  plugins:
+    - &aws-sm
+      seek-oss/aws-sm#v2.3.1:
+        env:
+          NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
+
+    - &docker-ecr-cache
+      seek-oss/docker-ecr-cache#v2.1.0: &docker-ecr-cache-defaults
+        cache-on:
+          - package.json
+          - yarn.lock
+        dockerfile: Dockerfile.dev-deps
+        secrets: id=npm,src=.npmrc
+
+    - &private-npm
+      seek-oss/private-npm#v1.2.0:
+        env: NPM_READ_TOKEN
+
+  base-steps:
+    - &deploy
+      concurrency: 1
+      retry:
+        manual:
+          # Only use this if you need to roll back a deployment ASAP.
+          # Always follow up with a proper revert or fix in Git history.
+          permit_on_passed: true
+
+steps:
+  - label: 🧖‍♀️ Warm Prod
+    command: ':'
+    key: warm-prod
+    plugins:
+      - *aws-sm
+      - *private-npm
+      - seek-oss/docker-ecr-cache#v2.1.0:
+          <<: *docker-ecr-cache-defaults
+          skip-pull-from-cache: true
+
+  - label: 🧪 Test & Lint
+    commands:
+      - echo '+++ yarn test:ci'
+      - yarn test:ci
+      - echo '--- yarn lint'
+      - yarn lint
+    depends_on: warm-prod
+    env:
+      GET_GITHUB_TOKEN: please
+    plugins:
+      - *aws-sm
+      - *private-npm
+      - *docker-ecr-cache
+      - docker-compose#v4.14.0:
+          run: app
+    timeout_in_minutes: 10
+
+  - label: 📦 Build & Package
+    depends_on: warm-prod
+    plugins:
+      - *aws-sm
+      - *private-npm
+      - *docker-ecr-cache
+      - seek-jobs/gantry#v2.0.0:
+          command: build
+          file: gantry.build.yml
+          region: <%- region %>
+          values: .gantry/common.yml
+
+  - wait
+  - block: 🙋🏻‍♀️ Deploy Dev
+    branches: '!${BUILDKITE_PIPELINE_DEFAULT_BRANCH}'
+
+  - <<: *deploy
+    agents:
+      queue: <%- devBuildkiteQueueName %>
+    label: 🤞 Deploy Dev
+    concurrency_group: <%- teamName %>/deploy/gantry/<%- devGantryEnvironmentName %>
+    key: deploy-dev
+    plugins:
+      - seek-jobs/gantry#v2.0.0:
+          command: apply
+          environment: <%- devGantryEnvironmentName %>
+          file: gantry.apply.yml
+          region: <%- region %>
+          values:
+            - .gantry/common.yml
+            - .gantry/dev.yml
+
+  - <<: *deploy
+    label: 🚀 Deploy Prod
+    branches: ${BUILDKITE_PIPELINE_DEFAULT_BRANCH}
+    concurrency_group: <%- teamName %>/deploy/gantry/<%- prodGantryEnvironmentName %>
+    depends_on: deploy-dev
+    plugins:
+      - seek-jobs/gantry#v2.0.0:
+          command: apply
+          environment: <%- prodGantryEnvironmentName %>
+          file: gantry.apply.yml
+          region: <%- region %>
+          values:
+            - .gantry/common.yml
+            - .gantry/prod.yml

package/template/koa-rest-api/.env

@@ -0,0 +1 @@
+ENVIRONMENT=local

package/template/koa-rest-api/.gantry/common.yml

@@ -0,0 +1,20 @@
+prodAccountId: '<%- prodAwsAccountId %>'
+
+image: '{{values "prodAccountId"}}.dkr.ecr.<%- region %>.amazonaws.com/{{values "service"}}:{{.BuildID}}'
+service: '<%- serviceName %>'
+
+# TODO: enable Datadog agent
+# https://backstage.myseek.xyz/docs/default/component/gantry/v1/reference/resources/service/#datadogSecretId
+# datadogSecretId: arn:aws:secretsmanager:<%- region %>:<aws-account-id>:secret:<secret-name>
+
+tags:
+  # TODO: add data classification tags
+  # https://rfc.skinfra.xyz/RFC019-AWS-Tagging-Standard.html#seekdataconsumers
+  # seek:data:consumers: internal
+  # https://rfc.skinfra.xyz/RFC019-AWS-Tagging-Standard.html#seekdatatypes
+  # seek:data:types:restricted: job-ads
+  seek:env:label: '{{values "environment"}}'
+  seek:env:production: '{{values "isProduction"}}'
+  seek:owner:team: '<%- teamName %>'
+  seek:source:sha: '{{.CommitSHA}}'
+  seek:source:url: 'https://github.com/SEEK-Jobs/<%- repoName %>'

package/template/koa-rest-api/.gantry/dev.yml

@@ -0,0 +1,11 @@
+environment: dev
+env:
+  SOME_ENVIRONMENT_VARIABLE: dev-value
+
+isProduction: false
+
+maxInstanceCount: 1
+minInstanceCount: 1
+
+openTelemetry:
+  enabled: false

package/template/koa-rest-api/.gantry/prod.yml

@@ -0,0 +1,11 @@
+environment: prod
+env:
+  SOME_ENVIRONMENT_VARIABLE: prod-value
+
+isProduction: true
+
+maxInstanceCount: 10
+minInstanceCount: 3
+
+openTelemetry:
+  enabled: false

package/template/koa-rest-api/.nvmrc

@@ -0,0 +1 @@
+18

package/template/koa-rest-api/Dockerfile

@@ -0,0 +1,37 @@
+ARG BASE_IMAGE
+ARG BASE_TAG
+
+###
+
+FROM ${BASE_IMAGE}:${BASE_TAG} AS deps
+
+RUN yarn install --ignore-optional --ignore-scripts --non-interactive --offline --production
+
+###
+
+FROM ${BASE_IMAGE}:${BASE_TAG} AS build
+
+COPY . .
+
+RUN yarn build
+
+###
+
+FROM --platform=${BUILDPLATFORM:-<%- platformName %>} gcr.io/distroless/nodejs18-debian11 AS runtime
+
+WORKDIR /workdir
+
+COPY --from=build /workdir/lib lib
+
+COPY --from=deps /workdir/node_modules node_modules
+
+ENV NODE_ENV=production
+
+# https://nodejs.org/api/cli.html#cli_node_options_options
+ENV NODE_OPTIONS=--enable-source-maps
+
+ARG PORT=8001
+ENV PORT=${PORT}
+EXPOSE ${PORT}
+
+CMD ["--require", "./lib/tracing.js", "./lib/listen.js"]

package/template/koa-rest-api/Dockerfile.dev-deps

@@ -0,0 +1,11 @@
+# syntax=docker/dockerfile:1.2
+
+FROM --platform=${BUILDPLATFORM:-<%- platformName %>} node:18-alpine AS dev-deps
+
+WORKDIR /workdir
+
+COPY package.json yarn.lock ./
+
+RUN \
+  --mount=type=secret,id=npm,dst=/workdir/.npmrc \
+  yarn install --frozen-lockfile --ignore-optional --non-interactive

package/template/koa-rest-api/README.md

@@ -0,0 +1,111 @@
+# <%- repoName %>
+
+[![Powered by skuba](https://img.shields.io/badge/🤿%20skuba-powered-009DC4)](https://github.com/seek-oss/skuba)
+
+Next steps:
+
+1. [ ] Finish templating if this was skipped earlier:
+
+   ```shell
+   yarn skuba configure
+   ```
+
+2. [ ] Create a new repository in the appropriate GitHub organisation.
+3. [ ] Add the repository to BuildAgency;
+       see [Builds at SEEK] for more information.
+4. [ ] Add Datadog configuration and data classification tags to [.gantry/common.yml](.gantry/common.yml);
+       see the [Gantry] documentation for more information.
+5. [ ] Push local commits to the upstream GitHub branch.
+6. [ ] Configure [GitHub repository settings].
+7. [ ] Delete this checklist 😌.
+
+[builds at seek]: https://builds-at-seek.ssod.skinfra.xyz
+[github repository settings]: https://github.com/<%-orgName%>/<%-repoName%>/settings
+
+## Design
+
+<%-repoName %> is a Node.js HTTP server built in line with our [technology strategy].
+It uses the [Koa] middleware framework and common SEEK packages.
+Resource APIs enable synchronous interactions and serve as the backbone of SEEK's general service architecture.
+
+The `koa-rest-api` template is modelled after a hypothetical service for posting and retrieving job advertisements.
+It's stubbed out with in-memory [storage](src/storage) which can observed by standing up an environment with multiple instances.
+Storage is local to each instance, so load balancing across the instances may render a read inconsistent with a previous write.
+This would be replaced with an external data store in production.
+
+This project is deployed as a containerised application with [Gantry].
+A typical resource API instance does not require more than 1 vCPU,
+so we eschew clustering configurations in favour of a single Node.js process per container.
+Under load, we autoscale horizontally in terms of container count up to `autoScaling.maxCount`.
+
+Gantry configures [CodeDeploy] for a blue-green deployment approach.
+A smoke test is run against the new version before traffic is switched over,
+providing an opportunity to test access and connectivity to online dependencies.
+This defaults to an HTTP request to the `GET /smoke` endpoint.
+
+## Development
+
+### Test
+
+```shell
+yarn test
+```
+
+### Lint
+
+```shell
+# Fix issues
+yarn format
+
+# Check for issues
+yarn lint
+```
+
+### Start
+
+```shell
+# Start a local HTTP server
+yarn start
+
+# Start with Node.js Inspector enabled
+yarn start:debug
+```
+
+### Deploy
+
+This project is deployed through a [Buildkite pipeline](.buildkite/pipeline.yml).
+
+- Commits to a feature branch can be deployed to the dev environment by unblocking a step in the Buildkite UI
+- Commits to the default branch are automatically deployed to the dev and prod environments in sequence
+
+To rapidly roll back a change,
+retry an individual deployment step from the previous build in Buildkite.
+Note that this will introduce drift between the head of the default Git branch and the live environment;
+use with caution and always follow up with a proper revert or fix in Git history.
+
+## Support
+
+### Dev
+
+TODO: add support links for the dev environment.
+
+<!--
+- CloudWatch dashboard
+- Datadog dashboard
+- Splunk logs
+-->
+
+### Prod
+
+TODO: add support links for the prod environment.
+
+<!--
+- CloudWatch dashboard
+- Datadog dashboard
+- Splunk logs
+-->
+
+[codedeploy]: https://docs.aws.amazon.com/codedeploy
+[gantry]: https://backstage.myseek.xyz/docs/default/component/gantry/
+[koa]: https://koajs.com
+[technology strategy]: https://tech-strategy.ssod.skinfra.xyz

package/template/koa-rest-api/docker-compose.yml

@@ -0,0 +1,22 @@
+version: '3.7'
+
+services:
+  app:
+    environment:
+      # Enable Buildkite + GitHub integrations.
+      - BUILDKITE
+      - BUILDKITE_AGENT_ACCESS_TOKEN
+      - BUILDKITE_BRANCH
+      - BUILDKITE_BUILD_NUMBER
+      - BUILDKITE_JOB_ID
+      - BUILDKITE_PIPELINE_DEFAULT_BRANCH
+      - BUILDKITE_STEP_ID
+      - GITHUB_API_TOKEN
+    image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''}
+    init: true
+    volumes:
+      - ./:/workdir
+      # Mount agent for Buildkite annotations.
+      - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent
+      # Mount cached dependencies.
+      - /workdir/node_modules

package/template/koa-rest-api/gantry.apply.yml

@@ -0,0 +1,127 @@
+kind: Service
+
+schemaVersion: v0.0
+
+name: '{{values "service"}}'
+
+image: '{{values "image"}}'
+
+env:
+  # https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/node-reusing-connections.html
+  AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1'
+
+  ENVIRONMENT: '{{values "environment"}}'
+  OPENTELEMETRY_ENABLED: '{{.Values.openTelemetry.enabled | default false}}'
+  SERVICE: '{{values "service"}}'
+
+  {{range $key, $value := .Values.env}}
+  {{$key}}: {{$value}}
+  {{end}}
+
+{{if .Values.datadogSecretId}}
+datadogSecretId: '{{values "datadogSecretId"}}'
+{{end}}
+
+openTelemetry:
+  enabled: {{.Values.openTelemetry.enabled | default false}}
+  {{if .Values.isProduction}}
+  datadogEnvironmentName: production
+  {{else}}
+  datadogEnvironmentName: development
+  {{end}}
+{{if .Values.pagerDutyEndpoint}}
+pagerDutyEndpoint: '{{values "pagerDutyEndpoint"}}'
+{{end}}
+
+authentication:
+  ignoreRoutes:
+    - /health
+    - /smoke
+
+sslEnabled: true
+
+autoScaling:
+  maxCount: {{values "maxInstanceCount"}}
+  minCount: {{values "minInstanceCount"}}
+
+  cpuThreshold: 50
+  memoryThreshold: 50
+
+  # Disable Gantry's request-based scaling; the load a request generates can be
+  # highly variable. Instead, scale out when *either* CPU or memory are above
+  # the threshold, and scale in when *both* are 5% below the threshold.
+  #
+  # See https://docs.aws.amazon.com/autoscaling/application/userguide/application-auto-scaling-target-tracking.html
+  autoScalingPolicies:
+    CpuAutoScalingPolicy:
+      Type: AWS::ApplicationAutoScaling::ScalingPolicy
+      Properties:
+        PolicyName: CpuAutoScalingPolicy
+        PolicyType: TargetTrackingScaling
+        ScalingTargetId:
+          Ref: AutoScalingTarget
+        TargetTrackingScalingPolicyConfiguration:
+          ScaleInCooldown: 300
+          ScaleOutCooldown: 60
+          TargetValue:
+            Ref: CpuUtilisationAutoScalingThreshold
+          PredefinedMetricSpecification:
+            PredefinedMetricType: ECSServiceAverageCPUUtilization
+
+    MemoryAutoScalingPolicy:
+      Type: AWS::ApplicationAutoScaling::ScalingPolicy
+      Properties:
+        PolicyName: MemoryAutoScalingPolicy
+        PolicyType: TargetTrackingScaling
+        ScalingTargetId:
+          Ref: AutoScalingTarget
+        TargetTrackingScalingPolicyConfiguration:
+          ScaleInCooldown: 300
+          ScaleOutCooldown: 60
+          TargetValue:
+            Ref: MemoryUtilisationAutoScalingThreshold
+          PredefinedMetricSpecification:
+            PredefinedMetricType: ECSServiceAverageMemoryUtilization
+
+    RequestsScaleOutAlarm:
+      Type: AWS::CloudWatch::Alarm
+      Properties:
+        AlarmDescription: Stub alarm for Gantry to tag
+        Metrics:
+          - Id: missing
+            Expression: SUM(METRICS())
+            Period: 60
+            ReturnData: true
+        ActionsEnabled: false
+        EvaluationPeriods: 1
+        Threshold: 0
+        ComparisonOperator: LessThanThreshold
+
+    RequestsScaleInAlarm:
+      Type: AWS::CloudWatch::Alarm
+      Properties:
+        AlarmDescription: Stub alarm for Gantry to tag
+        Metrics:
+          - Id: missing
+            Expression: SUM(METRICS())
+            Period: 60
+            ReturnData: true
+        ActionsEnabled: false
+        EvaluationPeriods: 1
+        Threshold: 0
+        ComparisonOperator: LessThanThreshold
+
+cpu: 256
+memory: 512
+
+deployment:
+  smokeTest:
+    path: /smoke
+    useExternalDns: true
+
+tags:
+  {{range $key, $value := .Values.tags}}
+  {{$key}}: '{{$value}}'
+  {{end}}
+
+cpuArchitecture: <%- platformName %>

package/template/koa-rest-api/gantry.build.yml

@@ -0,0 +1,15 @@
+kind: ContainerImage
+
+schemaVersion: v0.0
+
+image: '{{values "image"}}'
+
+buildArgs:
+  # https://github.com/seek-oss/docker-ecr-cache-buildkite-plugin#building-on-the-resulting-image
+  BASE_IMAGE: '{{.Env.BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_IMAGE}}'
+  BASE_TAG: '{{.Env.BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_TAG}}'
+
+# SEEK-Jobs/gantry#1661
+failOnScanFindings: false
+
+cpuArchitecture: <%- platformName %>

package/template/koa-rest-api/package.json

@@ -0,0 +1,47 @@
+{
+  "private": true,
+  "license": "UNLICENSED",
+  "scripts": {
+    "build": "skuba build",
+    "format": "skuba format",
+    "lint": "skuba lint",
+    "start": "skuba start --port <%- port %>",
+    "start:debug": "yarn start --inspect-brk",
+    "test": "skuba test",
+    "test:ci": "skuba test --coverage",
+    "test:watch": "skuba test --watch"
+  },
+  "dependencies": {
+    "@koa/router": "^12.0.0",
+    "@opentelemetry/api": "^1.1.0",
+    "@opentelemetry/exporter-trace-otlp-grpc": "^0.43.0",
+    "@opentelemetry/instrumentation-aws-sdk": "^0.36.0",
+    "@opentelemetry/instrumentation-http": "^0.43.0",
+    "@opentelemetry/sdk-node": "^0.43.0",
+    "@seek/logger": "^6.0.0",
+    "aws-sdk": "^2.1039.0",
+    "hot-shots": "^10.0.0",
+    "koa": "^2.13.4",
+    "koa-bodyparser": "^4.3.0",
+    "koa-compose": "^4.2.0",
+    "seek-datadog-custom-metrics": "^4.2.1",
+    "seek-koala": "^7.0.0",
+    "skuba-dive": "^2.0.0",
+    "zod": "^3.19.1"
+  },
+  "devDependencies": {
+    "@types/chance": "^1.1.3",
+    "@types/koa": "^2.13.4",
+    "@types/koa-bodyparser": "^5.0.2",
+    "@types/koa__router": "^12.0.0",
+    "@types/node": "^18.11.5",
+    "@types/supertest": "^2.0.11",
+    "chance": "^1.1.8",
+    "pino-pretty": "^10.0.0",
+    "skuba": "*",
+    "supertest": "^6.1.6"
+  },
+  "engines": {
+    "node": ">=18.12"
+  }
+}

package/template/koa-rest-api/skuba.template.js

@@ -0,0 +1,49 @@
+/**
+ * Run `skuba configure` to finish templating and remove this file.
+ */
+
+module.exports = {
+  entryPoint: 'src/app.ts',
+  fields: [
+    {
+      name: 'serviceName',
+      message: 'Service slug',
+      initial: 'my-project',
+    },
+    {
+      name: 'devBuildkiteQueueName',
+      message: 'Dev Buildkite queue',
+      initial: 'my-team-aws-account-dev:cicd',
+      validate: (value) => /^.+:.+$/.test(value),
+    },
+    {
+      name: 'prodBuildkiteQueueName',
+      message: 'Prod Buildkite queue',
+      initial: 'my-team-aws-account-prod:cicd',
+      validate: (value) => /^.+:.+$/.test(value),
+    },
+    {
+      name: 'devGantryEnvironmentName',
+      message: 'Dev Gantry environment',
+      initial: 'my-gantry-dev',
+    },
+    {
+      name: 'prodGantryEnvironmentName',
+      message: 'Prod Gantry environment',
+      initial: 'my-gantry-prod',
+    },
+    {
+      name: 'prodAwsAccountId',
+      message: 'Prod AWS account',
+      initial: '123456789012',
+      validate: (value) => /^\d{12}$/.test(value),
+    },
+    {
+      name: 'region',
+      message: 'AWS region',
+      initial: 'my-region-0',
+      validate: (value) => /^[a-z]{2}-[a-z]+-\d+$/.test(value),
+    },
+  ],
+  type: 'application',
+};

package/template/koa-rest-api/src/api/healthCheck.ts

@@ -0,0 +1,13 @@
+import type { Middleware } from 'src/types/koa';
+
+/**
+ * Signifies that the API is available to serve requests.
+ *
+ * The deployment environment calls this endpoint to see if the container is
+ * unhealthy and needs to be recycled.
+ */
+export const healthCheckHandler: Middleware = (ctx) => {
+  ctx.state.skipRequestLogging = true;
+
+  ctx.body = '';
+};

package/template/koa-rest-api/src/api/index.ts

@@ -0,0 +1,10 @@
+import Router from '@koa/router';
+
+import { healthCheckHandler } from './healthCheck';
+import { jobRouter } from './jobs';
+import { smokeTestHandler } from './smokeTest';
+
+export const router = new Router()
+  .get('/health', healthCheckHandler)
+  .get('/smoke', smokeTestHandler)
+  .use('/jobs', jobRouter.routes());

package/template/koa-rest-api/src/api/jobs/getJobs.test.ts

@@ -0,0 +1,9 @@
+import { agentFromRouter } from 'src/testing/server';
+
+import { jobRouter } from '.';
+
+const agent = agentFromRouter(jobRouter);
+
+describe('getJobsHandler', () => {
+  it('provides no results on first load', () => agent.get('/').expect(200, []));
+});