skuba 0.0.0-master-20230318044948

package/template/koa-rest-api/gantry.apply.yml

@@ -0,0 +1,129 @@
+kind: Service
+
+schemaVersion: v0.0
+
+name: '{{values "service"}}'
+
+image: '{{values "image"}}'
+
+env:
+  # https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/node-reusing-connections.html
+  AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1'
+
+  ENVIRONMENT: '{{values "environment"}}'
+  OPENTELEMETRY_ENABLED: '{{.Values.openTelemetry.enabled | default false}}'
+  SERVICE: '{{values "service"}}'
+
+  {{range $key, $value := .Values.env}}
+  {{$key}}: {{$value}}
+  {{end}}
+
+{{if .Values.datadogSecretId}}
+datadogSecretId: '{{values "datadogSecretId"}}'
+{{end}}
+
+openTelemetry:
+  enabled: {{.Values.openTelemetry.enabled | default false}}
+  {{if .Values.isProduction}}
+  datadogEnvironmentName: production
+  {{else}}
+  datadogEnvironmentName: development
+  {{end}}
+{{if .Values.pagerDutyEndpoint}}
+pagerDutyEndpoint: '{{values "pagerDutyEndpoint"}}'
+{{end}}
+
+authentication:
+  ignoreRoutes:
+    - /health
+    - /smoke
+
+sslEnabled: true
+
+autoScaling:
+  maxCount: {{values "maxInstanceCount"}}
+  minCount: {{values "minInstanceCount"}}
+
+  cpuThreshold: 50
+  memoryThreshold: 50
+
+  # Disable Gantry's request-based scaling; the load a request generates can be
+  # highly variable. Instead, scale out when *either* CPU or memory are above
+  # the threshold, and scale in when *both* are 5% below the threshold.
+  #
+  # See https://docs.aws.amazon.com/autoscaling/application/userguide/application-auto-scaling-target-tracking.html
+  autoScalingPolicies:
+    CpuAutoScalingPolicy:
+      Type: AWS::ApplicationAutoScaling::ScalingPolicy
+      Properties:
+        PolicyName: CpuAutoScalingPolicy
+        PolicyType: TargetTrackingScaling
+        ScalingTargetId:
+          Ref: AutoScalingTarget
+        TargetTrackingScalingPolicyConfiguration:
+          ScaleInCooldown: 300
+          ScaleOutCooldown: 60
+          TargetValue:
+            Ref: CpuUtilisationAutoScalingThreshold
+          PredefinedMetricSpecification:
+            PredefinedMetricType: ECSServiceAverageCPUUtilization
+
+    MemoryAutoScalingPolicy:
+      Type: AWS::ApplicationAutoScaling::ScalingPolicy
+      Properties:
+        PolicyName: MemoryAutoScalingPolicy
+        PolicyType: TargetTrackingScaling
+        ScalingTargetId:
+          Ref: AutoScalingTarget
+        TargetTrackingScalingPolicyConfiguration:
+          ScaleInCooldown: 300
+          ScaleOutCooldown: 60
+          TargetValue:
+            Ref: MemoryUtilisationAutoScalingThreshold
+          PredefinedMetricSpecification:
+            PredefinedMetricType: ECSServiceAverageMemoryUtilization
+
+    RequestsScaleOutAlarm:
+      Type: AWS::CloudWatch::Alarm
+      Properties:
+        AlarmDescription: Stub alarm for Gantry to tag
+        Metrics:
+          - Id: missing
+            Expression: SUM(METRICS())
+            Period: 60
+            ReturnData: true
+        ActionsEnabled: false
+        EvaluationPeriods: 1
+        Threshold: 0
+        ComparisonOperator: LessThanThreshold
+
+    RequestsScaleInAlarm:
+      Type: AWS::CloudWatch::Alarm
+      Properties:
+        AlarmDescription: Stub alarm for Gantry to tag
+        Metrics:
+          - Id: missing
+            Expression: SUM(METRICS())
+            Period: 60
+            ReturnData: true
+        ActionsEnabled: false
+        EvaluationPeriods: 1
+        Threshold: 0
+        ComparisonOperator: LessThanThreshold
+
+cpu: 256
+memory: 512
+
+deployment:
+  # SEEK-Jobs/gantry#488
+  ignoreAlarms: true
+  smokeTest:
+    path: /smoke
+    useExternalDns: true
+
+tags:
+  {{range $key, $value := .Values.tags}}
+  {{$key}}: '{{$value}}'
+  {{end}}
+
+cpuArchitecture: <%- platformName %>

package/template/koa-rest-api/gantry.build.yml

@@ -0,0 +1,15 @@
+kind: ContainerImage
+
+schemaVersion: v0.0
+
+image: '{{values "image"}}'
+
+buildArgs:
+  # https://github.com/seek-oss/docker-ecr-cache-buildkite-plugin#building-on-the-resulting-image
+  BASE_IMAGE: '{{.Env.BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_IMAGE}}'
+  BASE_TAG: '{{.Env.BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_TAG}}'
+
+# SEEK-Jobs/gantry#1661
+failOnScanFindings: false
+
+cpuArchitecture: <%- platformName %>

package/template/koa-rest-api/package.json

@@ -0,0 +1,47 @@
+{
+  "private": true,
+  "license": "UNLICENSED",
+  "scripts": {
+    "build": "skuba build",
+    "format": "skuba format",
+    "lint": "skuba lint",
+    "start": "skuba start --port <%- port %>",
+    "start:debug": "yarn start --inspect-brk",
+    "test": "skuba test",
+    "test:ci": "skuba test --coverage",
+    "test:watch": "skuba test --watch"
+  },
+  "dependencies": {
+    "@koa/router": "^12.0.0",
+    "@opentelemetry/api": "^1.1.0",
+    "@opentelemetry/exporter-collector-grpc": "^0.25.0",
+    "@opentelemetry/instrumentation-aws-sdk": "^0.34.0",
+    "@opentelemetry/instrumentation-http": "^0.36.0",
+    "@opentelemetry/sdk-node": "^0.35.0",
+    "@seek/logger": "^5.0.1",
+    "aws-sdk": "^2.1039.0",
+    "hot-shots": "^10.0.0",
+    "koa": "^2.13.4",
+    "koa-bodyparser": "^4.3.0",
+    "koa-compose": "^4.2.0",
+    "seek-datadog-custom-metrics": "^4.0.0",
+    "seek-koala": "^6.0.0",
+    "skuba-dive": "^2.0.0",
+    "zod": "^3.19.1"
+  },
+  "devDependencies": {
+    "@types/chance": "^1.1.3",
+    "@types/koa": "^2.13.4",
+    "@types/koa-bodyparser": "^5.0.2",
+    "@types/koa__router": "^12.0.0",
+    "@types/node": "^18.11.5",
+    "@types/supertest": "^2.0.11",
+    "chance": "^1.1.8",
+    "pino-pretty": "^10.0.0",
+    "skuba": "*",
+    "supertest": "^6.1.6"
+  },
+  "engines": {
+    "node": ">=18"
+  }
+}

package/template/koa-rest-api/skuba.template.js

@@ -0,0 +1,49 @@
+/**
+ * Run `skuba configure` to finish templating and remove this file.
+ */
+
+module.exports = {
+  entryPoint: 'src/app.ts',
+  fields: [
+    {
+      name: 'serviceName',
+      message: 'Service slug',
+      initial: 'my-project',
+    },
+    {
+      name: 'devBuildkiteQueueName',
+      message: 'Dev Buildkite queue',
+      initial: 'my-team-aws-account-dev:cicd',
+      validate: (value) => /^.+:.+$/.test(value),
+    },
+    {
+      name: 'prodBuildkiteQueueName',
+      message: 'Prod Buildkite queue',
+      initial: 'my-team-aws-account-prod:cicd',
+      validate: (value) => /^.+:.+$/.test(value),
+    },
+    {
+      name: 'devGantryEnvironmentName',
+      message: 'Dev Gantry environment',
+      initial: 'my-gantry-dev',
+    },
+    {
+      name: 'prodGantryEnvironmentName',
+      message: 'Prod Gantry environment',
+      initial: 'my-gantry-prod',
+    },
+    {
+      name: 'prodAwsAccountId',
+      message: 'Prod AWS account',
+      initial: '123456789012',
+      validate: (value) => /^\d{12}$/.test(value),
+    },
+    {
+      name: 'region',
+      message: 'AWS region',
+      initial: 'my-region-0',
+      validate: (value) => /^[a-z]{2}-[a-z]+-\d+$/.test(value),
+    },
+  ],
+  type: 'application',
+};

package/template/koa-rest-api/src/api/healthCheck.ts

@@ -0,0 +1,13 @@
+import { Middleware } from 'src/types/koa';
+
+/**
+ * Signifies that the API is available to serve requests.
+ *
+ * The deployment environment calls this endpoint to see if the container is
+ * unhealthy and needs to be recycled.
+ */
+export const healthCheckHandler: Middleware = (ctx) => {
+  ctx.state.skipRequestLogging = true;
+
+  ctx.body = '';
+};

package/template/koa-rest-api/src/api/index.ts

@@ -0,0 +1,10 @@
+import Router from '@koa/router';
+
+import { healthCheckHandler } from './healthCheck';
+import { jobRouter } from './jobs';
+import { smokeTestHandler } from './smokeTest';
+
+export const router = new Router()
+  .get('/health', healthCheckHandler)
+  .get('/smoke', smokeTestHandler)
+  .use('/jobs', jobRouter.routes());

package/template/koa-rest-api/src/api/jobs/getJobs.test.ts

@@ -0,0 +1,9 @@
+import { agentFromRouter } from 'src/testing/server';
+
+import { jobRouter } from '.';
+
+const agent = agentFromRouter(jobRouter);
+
+describe('getJobsHandler', () => {
+  it('provides no results on first load', () => agent.get('/').expect(200, []));
+});

package/template/koa-rest-api/src/api/jobs/getJobs.ts

@@ -0,0 +1,15 @@
+import { logger } from 'src/framework/logging';
+import { metricsClient } from 'src/framework/metrics';
+import * as storage from 'src/storage/jobs';
+import { Middleware } from 'src/types/koa';
+
+export const getJobsHandler: Middleware = async (ctx) => {
+  const jobs = await storage.readJobs();
+
+  // no PII in these jobs
+  logger.debug({ jobs }, 'read jobs');
+
+  metricsClient.increment('job.reads', jobs.length);
+
+  ctx.body = jobs;
+};

package/template/koa-rest-api/src/api/jobs/index.ts

@@ -0,0 +1,10 @@
+import Router from '@koa/router';
+
+import { jsonBodyParser } from 'src/framework/middleware';
+
+import { getJobsHandler } from './getJobs';
+import { postJobHandler } from './postJob';
+
+export const jobRouter = new Router()
+  .get('/', getJobsHandler)
+  .post('/', jsonBodyParser, postJobHandler);

package/template/koa-rest-api/src/api/jobs/postJob.test.ts

@@ -0,0 +1,34 @@
+import { agentFromRouter } from 'src/testing/server';
+import { mockJobInput } from 'src/testing/types';
+
+import { jobRouter } from '.';
+
+const agent = agentFromRouter(jobRouter);
+
+describe('postJobHandler', () => {
+  it('200s and allocates an ID on valid input', () => {
+    const jobInput = mockJobInput();
+
+    return agent
+      .post('/')
+      .send(jobInput)
+      .expect(200)
+      .expect(({ body }) =>
+        expect(body).toStrictEqual({ ...jobInput, id: expect.any(String) }),
+      );
+  });
+
+  it('422s on invalid input', () => {
+    const jobInput = {};
+
+    return agent
+      .post('/')
+      .send(jobInput)
+      .expect(422)
+      .expect(({ text }) =>
+        expect(text).toMatchInlineSnapshot(
+          `"{"message":"Input validation failed","invalidFields":{"/hirer":"Required"}}"`,
+        ),
+      );
+  });
+});

package/template/koa-rest-api/src/api/jobs/postJob.ts

@@ -0,0 +1,19 @@
+import { logger } from 'src/framework/logging';
+import { metricsClient } from 'src/framework/metrics';
+import { validateRequestBody } from 'src/framework/validation';
+import * as storage from 'src/storage/jobs';
+import { JobInputSchema } from 'src/types/jobs';
+import { Middleware } from 'src/types/koa';
+
+export const postJobHandler: Middleware = async (ctx) => {
+  const jobInput = validateRequestBody(ctx, JobInputSchema);
+
+  const job = await storage.createJob(jobInput);
+
+  // no PII in these jobs
+  logger.debug({ job }, 'created job');
+
+  metricsClient.increment('job.creations');
+
+  ctx.body = job;
+};

package/template/koa-rest-api/src/api/smokeTest.ts

@@ -0,0 +1,11 @@
+import { smokeTestJobStorage } from 'src/storage/jobs';
+import { Middleware } from 'src/types/koa';
+
+/**
+ * Tests connectivity to ensure appropriate access and network configuration.
+ */
+export const smokeTestHandler: Middleware = async (ctx) => {
+  await Promise.all([smokeTestJobStorage()]);
+
+  ctx.body = 'Smoke test passed';
+};

package/template/koa-rest-api/src/app.test.ts

@@ -0,0 +1,21 @@
+import { agentFromApp } from 'src/testing/server';
+
+import app from './app';
+
+const agent = agentFromApp(app);
+
+describe('app', () => {
+  it('exports callback for skuba start', () =>
+    expect(app).toHaveProperty('callback'));
+
+  it('has a happy health check', () => agent.get('/health').expect(200, ''));
+
+  it('has a reachable smoke test', () =>
+    agent.get('/smoke').expect(({ status }) => status !== 404));
+
+  it('has a reachable nested route', () =>
+    agent.get('/jobs').expect(({ status }) => status !== 404));
+
+  it('has OPTIONS for a nested route', () =>
+    agent.options('/jobs').expect(200).expect('allow', /HEAD/));
+});

package/template/koa-rest-api/src/app.ts

@@ -0,0 +1,8 @@
+import './register';
+
+import { router } from './api';
+import { createApp } from './framework/server';
+
+const app = createApp(router.allowedMethods(), router.routes());
+
+export default app;

package/template/koa-rest-api/src/config.ts

@@ -0,0 +1,57 @@
+import { Env } from 'skuba-dive';
+
+interface Config {
+  environment: Environment;
+
+  logLevel: string;
+  name: string;
+  version: string;
+
+  metricsServer?: string;
+  port?: number;
+}
+
+type Environment = (typeof environments)[number];
+
+const dev = '<%- devGantryEnvironmentName %>';
+const prod = '<%- prodGantryEnvironmentName %>';
+
+const environments = ['local', 'test', dev, prod] as const;
+
+const environment = Env.oneOf(environments)('ENVIRONMENT');
+
+/* istanbul ignore next: config verification makes more sense in a smoke test */
+const configs: Record<Environment, () => Omit<Config, 'environment'>> = {
+  local: () => ({
+    logLevel: 'debug',
+    name: '<%- serviceName %>',
+    version: 'local',
+  }),
+
+  test: () => ({
+    ...configs.local(),
+
+    logLevel: Env.string('LOG_LEVEL', { default: 'silent' }),
+    version: 'test',
+  }),
+
+  [dev]: () => ({
+    ...configs[prod](),
+
+    logLevel: 'debug',
+  }),
+
+  [prod]: () => ({
+    logLevel: 'info',
+    name: Env.string('SERVICE'),
+    version: Env.string('VERSION'),
+
+    metricsServer: 'localhost',
+    port: Env.nonNegativeInteger('PORT'),
+  }),
+};
+
+export const config: Config = {
+  ...configs[environment](),
+  environment,
+};

package/template/koa-rest-api/src/framework/logging.ts

@@ -0,0 +1,25 @@
+import createLogger from '@seek/logger';
+import { RequestLogging } from 'seek-koala';
+
+import { config } from 'src/config';
+
+const { createContextMiddleware, mixin } =
+  RequestLogging.createContextStorage();
+
+export const contextMiddleware = createContextMiddleware();
+
+export const logger = createLogger({
+  base: {
+    environment: config.environment,
+    version: config.version,
+  },
+
+  mixin,
+
+  level: config.logLevel,
+
+  name: config.name,
+
+  transport:
+    config.environment === 'local' ? { target: 'pino-pretty' } : undefined,
+});

package/template/koa-rest-api/src/framework/metrics.ts

@@ -0,0 +1,11 @@
+import { StatsD } from 'hot-shots';
+import { createStatsDClient } from 'seek-datadog-custom-metrics';
+
+import { config } from 'src/config';
+
+import { logger } from './logging';
+
+/* istanbul ignore next: StatsD client is not our responsibility */
+export const metricsClient = createStatsDClient(StatsD, config, (err) =>
+  logger.error({ err }, 'StatsD error'),
+);

package/template/koa-rest-api/src/framework/middleware.ts

@@ -0,0 +1,3 @@
+import bodyParser from 'koa-bodyparser';
+
+export const jsonBodyParser = bodyParser({ enableTypes: ['json'] });