skuba 0.0.0-master-20230318044948

package/template/express-rest-api/src/listen.ts

@@ -0,0 +1,23 @@
+import './register';
+
+import app from './app';
+import { config } from './config';
+import { rootLogger } from './framework/logging';
+
+// If your application is deployed with more than 1 vCPU you can delete this
+// file and use a clustering utility to run `lib/app`.
+
+const listener = app.listen(config.port, () => {
+  const address = listener.address();
+
+  if (typeof address === 'object' && address) {
+    rootLogger.debug(`listening on port ${address.port}`);
+  }
+});
+
+// Gantry ALB default idle timeout is 30 seconds
+// https://nodejs.org/docs/latest-v18.x/api/http.html#serverkeepalivetimeout
+// Node default is 5 seconds
+// https://docs.aws.amazon.com/elasticloadbalancing/latest/application/application-load-balancers.html#connection-idle-timeout
+// AWS recommends setting an application timeout larger than the load balancer
+listener.keepAliveTimeout = 31000;

package/template/express-rest-api/src/register.ts

@@ -0,0 +1 @@
+import 'skuba-dive/register';

package/template/greeter/.buildkite/pipeline.yml

@@ -0,0 +1,37 @@
+agents:
+  queue: <%- prodBuildkiteQueueName %>
+
+configs:
+  plugins:
+    - &aws-sm
+      seek-oss/aws-sm#v2.3.1:
+        env:
+          NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
+
+    - &docker-ecr-cache
+      seek-oss/docker-ecr-cache#v2.0.0:
+        cache-on:
+          - package.json
+          - yarn.lock
+        secrets: id=npm,src=.npmrc
+
+    - &private-npm
+      seek-oss/private-npm#v1.2.0:
+        env: NPM_READ_TOKEN
+
+steps:
+  - label: 🧪 Test & Lint
+    commands:
+      - echo '+++ yarn test:ci'
+      - yarn test:ci
+      - echo '--- yarn lint'
+      - yarn lint
+    env:
+      GET_GITHUB_TOKEN: please
+    plugins:
+      - *aws-sm
+      - *private-npm
+      - *docker-ecr-cache
+      - docker-compose#v4.11.0:
+          run: app
+    timeout_in_minutes: 10

package/template/greeter/.nvmrc

@@ -0,0 +1 @@
+18

package/template/greeter/Dockerfile

@@ -0,0 +1,11 @@
+# syntax=docker/dockerfile:1.2
+
+FROM --platform=${BUILDPLATFORM:-<%- platformName %>} node:18-alpine AS dev-deps
+
+WORKDIR /workdir
+
+COPY package.json yarn.lock ./
+
+RUN \
+  --mount=type=secret,id=npm,dst=/workdir/.npmrc \
+  yarn install --frozen-lockfile --ignore-optional --non-interactive

package/template/greeter/README.md

@@ -0,0 +1,96 @@
+# <%- repoName %>
+
+[![Powered by skuba](https://img.shields.io/badge/🤿%20skuba-powered-009DC4)](https://github.com/seek-oss/skuba)
+
+Next steps:
+
+1. [ ] Finish templating if this was skipped earlier:
+
+   ```shell
+   yarn skuba configure
+   ```
+
+2. [ ] Create a new repository in the appropriate GitHub organisation.
+3. [ ] Add the repository to BuildAgency;
+       see [Builds at SEEK] for more information.
+4. [ ] Push local commits to the upstream GitHub branch.
+5. [ ] Configure [GitHub repository settings].
+6. [ ] Delete this checklist 😌.
+
+[builds at seek]: https://builds-at-seek.ssod.skinfra.xyz
+[github repository settings]: https://github.com/<%-orgName%>/<%-repoName%>/settings
+
+## Design
+
+The `greeter` template is the prototypical _hello world_ project.
+It can function as a playground for the TypeScript tooling prescribed by our [technology strategy],
+or serve as a starting point for a backend project if the other built-in templates are not a good fit.
+
+It's a barebones Node.js application that comprises:
+
+- A [src/app.ts](src/app.ts) that can be run locally to greet the user
+- A [src/app.test.ts](src/app.test.ts) that demonstrates rudimentary Jest usage
+
+## Development
+
+### Test
+
+```shell
+yarn test
+```
+
+### Lint
+
+```shell
+# Fix issues
+yarn format
+
+# Check for issues
+yarn lint
+```
+
+### Start
+
+```shell
+# Start a live-reloading process
+yarn start
+
+# Start with Node.js Inspector enabled
+yarn start:debug
+```
+
+This runs a live-reloading Node.js process pointing to the [src/app.ts](src/app.ts) entrypoint.
+
+## Deploy
+
+The `greeter` template includes a simple [Buildkite pipeline](.buildkite/pipeline.yml) for testing and linting.
+It does not assume a deployment method or environment.
+
+For inspiration in this space, check out:
+
+- The `koa-rest-api` template for containerised deployments
+- The `lambda-sqs-worker` template for Lambda deployments
+
+## Support
+
+### Dev
+
+TODO: add support links for the dev environment.
+
+<!--
+- CloudWatch dashboard
+- Datadog dashboard
+- Splunk logs
+-->
+
+### Prod
+
+TODO: add support links for the prod environment.
+
+<!--
+- CloudWatch dashboard
+- Datadog dashboard
+- Splunk logs
+-->
+
+[technology strategy]: https://tech-strategy.ssod.skinfra.xyz

package/template/greeter/docker-compose.yml

@@ -0,0 +1,22 @@
+version: '3.7'
+
+services:
+  app:
+    environment:
+      # Enable Buildkite + GitHub integrations.
+      - BUILDKITE
+      - BUILDKITE_AGENT_ACCESS_TOKEN
+      - BUILDKITE_BRANCH
+      - BUILDKITE_BUILD_NUMBER
+      - BUILDKITE_JOB_ID
+      - BUILDKITE_PIPELINE_DEFAULT_BRANCH
+      - BUILDKITE_STEP_ID
+      - GITHUB_API_TOKEN
+    image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''}
+    init: true
+    volumes:
+      - ./:/workdir
+      # Mount agent for Buildkite annotations.
+      - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent
+      # Mount cached dependencies.
+      - /workdir/node_modules

package/template/greeter/package.json

@@ -0,0 +1,24 @@
+{
+  "private": true,
+  "license": "UNLICENSED",
+  "scripts": {
+    "build": "skuba build",
+    "format": "skuba format",
+    "lint": "skuba lint",
+    "start": "skuba start",
+    "start:debug": "yarn start --inspect-brk",
+    "test": "skuba test",
+    "test:ci": "skuba test --coverage",
+    "test:watch": "skuba test --watch"
+  },
+  "dependencies": {
+    "skuba-dive": "^2.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^18.11.5",
+    "skuba": "*"
+  },
+  "engines": {
+    "node": ">=18"
+  }
+}

package/template/greeter/skuba.template.js

@@ -0,0 +1,16 @@
+/**
+ * Run `skuba configure` to finish templating and remove this file.
+ */
+
+module.exports = {
+  entryPoint: 'src/app.ts',
+  fields: [
+    {
+      name: 'prodBuildkiteQueueName',
+      message: 'Prod Buildkite queue',
+      initial: 'my-team-aws-account-prod:cicd',
+      validate: (value) => /^.+:.+$/.test(value),
+    },
+  ],
+  type: 'application',
+};

package/template/greeter/src/app.test.ts

@@ -0,0 +1,5 @@
+import * as app from './app';
+
+describe('app', () => {
+  it('is importable', () => expect(app).toEqual({}));
+});

package/template/greeter/src/app.ts

@@ -0,0 +1,4 @@
+import 'skuba-dive/register';
+
+/* eslint-disable-next-line no-console */
+console.log('Hello world');

package/template/koa-rest-api/.buildkite/pipeline.yml

@@ -0,0 +1,103 @@
+agents:
+  queue: <%- prodBuildkiteQueueName %>
+
+configs:
+  plugins:
+    - &aws-sm
+      seek-oss/aws-sm#v2.3.1:
+        env:
+          NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
+
+    - &docker-ecr-cache
+      seek-oss/docker-ecr-cache#v2.0.0:
+        cache-on:
+          - package.json
+          - yarn.lock
+        dockerfile: Dockerfile.dev-deps
+        secrets: id=npm,src=.npmrc
+
+    - &private-npm
+      seek-oss/private-npm#v1.2.0:
+        env: NPM_READ_TOKEN
+
+  base-steps:
+    - &deploy
+      concurrency: 1
+      retry:
+        manual:
+          # Only use this if you need to roll back a deployment ASAP.
+          # Always follow up with a proper revert or fix in Git history.
+          permit_on_passed: true
+
+steps:
+  - label: 🧖‍♀️ Warm Prod
+    command: ':'
+    key: warm-prod
+    plugins:
+      - *aws-sm
+      - *private-npm
+      - *docker-ecr-cache
+
+  - label: 🧪 Test & Lint
+    commands:
+      - echo '+++ yarn test:ci'
+      - yarn test:ci
+      - echo '--- yarn lint'
+      - yarn lint
+    depends_on: warm-prod
+    env:
+      GET_GITHUB_TOKEN: please
+    plugins:
+      - *aws-sm
+      - *private-npm
+      - *docker-ecr-cache
+      - docker-compose#v4.11.0:
+          run: app
+    timeout_in_minutes: 10
+
+  - label: 📦 Build & Package
+    depends_on: warm-prod
+    plugins:
+      - *aws-sm
+      - *private-npm
+      - *docker-ecr-cache
+      - seek-jobs/gantry#v2.0.0:
+          command: build
+          file: gantry.build.yml
+          region: <%- region %>
+          values: .gantry/common.yml
+
+  - wait
+  - block: 🙋🏻‍♀️ Deploy Dev
+    branches: '!${BUILDKITE_PIPELINE_DEFAULT_BRANCH}'
+
+  - <<: *deploy
+    agents:
+      queue: <%- devBuildkiteQueueName %>
+    label: 🤞 Deploy Dev
+    concurrency_group: <%- teamName %>/deploy/gantry/<%- devGantryEnvironmentName %>
+    key: deploy-dev
+    plugins:
+      - seek-jobs/gantry#v2.0.0:
+          command: apply
+          environment: <%- devGantryEnvironmentName %>
+          file: gantry.apply.yml
+          region: <%- region %>
+          values:
+            - .gantry/common.yml
+            - .gantry/dev.yml
+
+  - <<: *deploy
+    label: 🚀 Deploy Prod
+    branches: ${BUILDKITE_PIPELINE_DEFAULT_BRANCH}
+    concurrency_group: <%- teamName %>/deploy/gantry/<%- prodGantryEnvironmentName %>
+    depends_on: deploy-dev
+    plugins:
+      - seek-jobs/gantry#v2.0.0:
+          command: apply
+          environment: <%- prodGantryEnvironmentName %>
+          file: gantry.apply.yml
+          region: <%- region %>
+          values:
+            - .gantry/common.yml
+            - .gantry/prod.yml

package/template/koa-rest-api/.env

@@ -0,0 +1 @@
+ENVIRONMENT=local

package/template/koa-rest-api/.gantry/common.yml

@@ -0,0 +1,20 @@
+prodAccountId: '<%- prodAwsAccountId %>'
+
+image: '{{values "prodAccountId"}}.dkr.ecr.<%- region %>.amazonaws.com/{{values "service"}}:{{.BuildID}}'
+service: '<%- serviceName %>'
+
+# TODO: enable Datadog agent
+# https://backstage.myseek.xyz/docs/default/component/gantry/v1/reference/resources/service/#datadogSecretId
+# datadogSecretId: arn:aws:secretsmanager:<%- region %>:<aws-account-id>:secret:<secret-name>
+
+tags:
+  # TODO: add data classification tags
+  # https://rfc.skinfra.xyz/RFC019-AWS-Tagging-Standard.html#seekdataconsumers
+  # seek:data:consumers: internal
+  # https://rfc.skinfra.xyz/RFC019-AWS-Tagging-Standard.html#seekdatatypes
+  # seek:data:types:restricted: job-ads
+  seek:env:label: '{{values "environment"}}'
+  seek:env:production: '{{values "isProduction"}}'
+  seek:owner:team: '<%- teamName %>'
+  seek:source:sha: '{{.CommitSHA}}'
+  seek:source:url: 'https://github.com/SEEK-Jobs/<%- repoName %>'

package/template/koa-rest-api/.gantry/dev.yml

@@ -0,0 +1,11 @@
+environment: dev
+env:
+  SOME_ENVIRONMENT_VARIABLE: dev-value
+
+isProduction: false
+
+maxInstanceCount: 1
+minInstanceCount: 1
+
+openTelemetry:
+  enabled: false

package/template/koa-rest-api/.gantry/prod.yml

@@ -0,0 +1,11 @@
+environment: prod
+env:
+  SOME_ENVIRONMENT_VARIABLE: prod-value
+
+isProduction: true
+
+maxInstanceCount: 10
+minInstanceCount: 3
+
+openTelemetry:
+  enabled: false

package/template/koa-rest-api/.nvmrc

@@ -0,0 +1 @@
+18

package/template/koa-rest-api/Dockerfile

@@ -0,0 +1,37 @@
+ARG BASE_IMAGE
+ARG BASE_TAG
+
+###
+
+FROM ${BASE_IMAGE}:${BASE_TAG} AS deps
+
+RUN yarn install --ignore-optional --ignore-scripts --non-interactive --offline --production
+
+###
+
+FROM ${BASE_IMAGE}:${BASE_TAG} AS build
+
+COPY . .
+
+RUN yarn build
+
+###
+
+FROM --platform=${BUILDPLATFORM:-<%- platformName %>} gcr.io/distroless/nodejs:18 AS runtime
+
+WORKDIR /workdir
+
+COPY --from=build /workdir/lib lib
+
+COPY --from=deps /workdir/node_modules node_modules
+
+ENV NODE_ENV=production
+
+# https://nodejs.org/api/cli.html#cli_node_options_options
+ENV NODE_OPTIONS=--enable-source-maps
+
+ARG PORT=8001
+ENV PORT=${PORT}
+EXPOSE ${PORT}
+
+CMD ["--require", "./lib/tracing.js", "./lib/listen.js"]

package/template/koa-rest-api/Dockerfile.dev-deps

@@ -0,0 +1,11 @@
+# syntax=docker/dockerfile:1.2
+
+FROM --platform=${BUILDPLATFORM:-<%- platformName %>} node:18-alpine AS dev-deps
+
+WORKDIR /workdir
+
+COPY package.json yarn.lock ./
+
+RUN \
+  --mount=type=secret,id=npm,dst=/workdir/.npmrc \
+  yarn install --frozen-lockfile --ignore-optional --non-interactive

package/template/koa-rest-api/README.md

@@ -0,0 +1,111 @@
+# <%- repoName %>
+
+[![Powered by skuba](https://img.shields.io/badge/🤿%20skuba-powered-009DC4)](https://github.com/seek-oss/skuba)
+
+Next steps:
+
+1. [ ] Finish templating if this was skipped earlier:
+
+   ```shell
+   yarn skuba configure
+   ```
+
+2. [ ] Create a new repository in the appropriate GitHub organisation.
+3. [ ] Add the repository to BuildAgency;
+       see [Builds at SEEK] for more information.
+4. [ ] Add Datadog configuration and data classification tags to [.gantry/common.yml](.gantry/common.yml);
+       see the [Gantry] documentation for more information.
+5. [ ] Push local commits to the upstream GitHub branch.
+6. [ ] Configure [GitHub repository settings].
+7. [ ] Delete this checklist 😌.
+
+[builds at seek]: https://builds-at-seek.ssod.skinfra.xyz
+[github repository settings]: https://github.com/<%-orgName%>/<%-repoName%>/settings
+
+## Design
+
+<%-repoName %> is a Node.js HTTP server built in line with our [technology strategy].
+It uses the [Koa] middleware framework and common SEEK packages.
+Resource APIs enable synchronous interactions and serve as the backbone of SEEK's general service architecture.
+
+The `koa-rest-api` template is modelled after a hypothetical service for posting and retrieving job advertisements.
+It's stubbed out with in-memory [storage](src/storage) which can observed by standing up an environment with multiple instances.
+Storage is local to each instance, so load balancing across the instances may render a read inconsistent with a previous write.
+This would be replaced with an external data store in production.
+
+This project is deployed as a containerised application with [Gantry].
+A typical resource API instance does not require more than 1 vCPU,
+so we eschew clustering configurations in favour of a single Node.js process per container.
+Under load, we autoscale horizontally in terms of container count up to `autoScaling.maxCount`.
+
+Gantry configures [CodeDeploy] for a blue-green deployment approach.
+A smoke test is run against the new version before traffic is switched over,
+providing an opportunity to test access and connectivity to online dependencies.
+This defaults to an HTTP request to the `GET /smoke` endpoint.
+
+## Development
+
+### Test
+
+```shell
+yarn test
+```
+
+### Lint
+
+```shell
+# Fix issues
+yarn format
+
+# Check for issues
+yarn lint
+```
+
+### Start
+
+```shell
+# Start a local HTTP server
+yarn start
+
+# Start with Node.js Inspector enabled
+yarn start:debug
+```
+
+### Deploy
+
+This project is deployed through a [Buildkite pipeline](.buildkite/pipeline.yml).
+
+- Commits to a feature branch can be deployed to the dev environment by unblocking a step in the Buildkite UI
+- Commits to the default branch are automatically deployed to the dev and prod environments in sequence
+
+To rapidly roll back a change,
+retry an individual deployment step from the previous build in Buildkite.
+Note that this will introduce drift between the head of the default Git branch and the live environment;
+use with caution and always follow up with a proper revert or fix in Git history.
+
+## Support
+
+### Dev
+
+TODO: add support links for the dev environment.
+
+<!--
+- CloudWatch dashboard
+- Datadog dashboard
+- Splunk logs
+-->
+
+### Prod
+
+TODO: add support links for the prod environment.
+
+<!--
+- CloudWatch dashboard
+- Datadog dashboard
+- Splunk logs
+-->
+
+[codedeploy]: https://docs.aws.amazon.com/codedeploy
+[gantry]: https://backstage.myseek.xyz/docs/default/component/gantry/
+[koa]: https://koajs.com
+[technology strategy]: https://tech-strategy.ssod.skinfra.xyz

package/template/koa-rest-api/docker-compose.yml

@@ -0,0 +1,22 @@
+version: '3.7'
+
+services:
+  app:
+    environment:
+      # Enable Buildkite + GitHub integrations.
+      - BUILDKITE
+      - BUILDKITE_AGENT_ACCESS_TOKEN
+      - BUILDKITE_BRANCH
+      - BUILDKITE_BUILD_NUMBER
+      - BUILDKITE_JOB_ID
+      - BUILDKITE_PIPELINE_DEFAULT_BRANCH
+      - BUILDKITE_STEP_ID
+      - GITHUB_API_TOKEN
+    image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''}
+    init: true
+    volumes:
+      - ./:/workdir
+      # Mount agent for Buildkite annotations.
+      - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent
+      # Mount cached dependencies.
+      - /workdir/node_modules