skuba 7.3.0 → 7.4.0-horrible-hacks-20240206023615

package/template/koa-rest-api/src/framework/validation.ts

@@ -1,8 +1,10 @@
 import { ErrorMiddleware } from 'seek-koala';
-import type { z } from 'zod';
+import { ZodIssueCode, type z } from 'zod';
 
 import type { Context } from 'src/types/koa';
 
+type InvalidFields = Record<string, string>;
+
 /**
  * Converts a `ZodError` into an `invalidFields` object
  *
@@ -28,13 +30,33 @@ import type { Context } from 'src/types/koa';
  * ```json
  * { "/advertiserId": "advertiserId is required in the URL" }
  * ```
+ *
+ * For union errors, the path will be appended with `~union${unionIdx}` to indicate which union type failed.
+ * @see [union error example](./validation.test.ts)
  */
-const parseInvalidFieldsFromError = ({
-  errors,
-}: z.ZodError): Record<string, string> =>
-  Object.fromEntries(
-    errors.map((err) => [`/${err.path.join('/')}`, err.message]),
-  );
+const parseInvalidFieldsFromError = (err: z.ZodError): InvalidFields =>
+  Object.fromEntries(parseTuples(err, {}));
+
+const parseTuples = (
+  { errors }: z.ZodError,
+  unions: Record<number, number[]>,
+): Array<readonly [string, string]> =>
+  errors.flatMap((issue) => {
+    if (issue.code === ZodIssueCode.invalid_union) {
+      return issue.unionErrors.flatMap((err, idx) =>
+        parseTuples(err, {
+          ...unions,
+          [issue.path.length]: [...(unions[issue.path.length] ?? []), idx],
+        }),
+      );
+    }
+
+    const path = ['', ...issue.path]
+      .map((prop, idx) => [prop, ...(unions[idx] ?? [])].join('~union'))
+      .join('/');
+
+    return [[path, issue.message]] as const;
+  });
 
 export const validate = <
   Output,
@@ -51,11 +73,12 @@ export const validate = <
 }): Output => {
   const parseResult = schema.safeParse(input);
   if (parseResult.success === false) {
+    const invalidFields = parseInvalidFieldsFromError(parseResult.error);
     return ctx.throw(
       422,
       new ErrorMiddleware.JsonResponse('Input validation failed', {
         message: 'Input validation failed',
-        invalidFields: parseInvalidFieldsFromError(parseResult.error),
+        invalidFields,
       }),
     );
   }

package/template/koa-rest-api/src/testing/types.ts

@@ -5,10 +5,22 @@ import type { JobInput } from 'src/types/jobs';
 
 export type IdDescription = z.infer<typeof IdDescriptionSchema>;
 
-export const IdDescriptionSchema = z.object({
-  id: z.string(),
-  description: z.string(),
-});
+export const IdDescriptionSchema = z.union([
+  z.object({
+    id: z.string(),
+    description: z.union([
+      z.string(),
+      z.object({
+        fontSize: z.number(),
+        content: z.string(),
+      }),
+    ]),
+  }),
+  z.object({
+    id: z.number(),
+    summary: z.string(),
+  }),
+]);
 
 export const chance = new Chance();
 

package/template/lambda-sqs-worker/.buildkite/pipeline.yml

@@ -4,15 +4,13 @@ agents:
 configs:
   plugins:
     - &aws-sm
-      seek-oss/aws-sm#v2.3.1:
+      seek-oss/aws-sm#v2.3.2:
         env:
           NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
 
     - &docker-ecr-cache
-      seek-oss/docker-ecr-cache#v2.1.0: &docker-ecr-cache-defaults
-        cache-on:
-          - package.json
-          - yarn.lock
+      seek-oss/docker-ecr-cache#v2.1.1: &docker-ecr-cache-defaults
+        cache-on: pnpm-lock.yaml
         secrets: id=npm,src=tmp/.npmrc
 
     - &private-npm
@@ -23,11 +21,13 @@ configs:
   base-steps:
     - &deploy
       commands:
-        - echo '+++ yarn deploy'
-        - yarn deploy
+        - echo '--- pnpm install --offline'
+        - pnpm install --offline
+        - echo '+++ pnpm run deploy'
+        - pnpm run deploy
       concurrency: 1
       plugins:
-        - artifacts#v1.9.2:
+        - artifacts#v1.9.3:
             build: ${BUILDKITE_BUILD_ID}
             download: lib/*
         - *aws-sm
@@ -36,6 +36,7 @@ configs:
         - docker-compose#v4.16.0:
             dependencies: false
             run: app
+            propagate-environment: true
       retry:
         manual:
           # Only use this if you need to roll back a deployment ASAP.
@@ -49,12 +50,14 @@ steps:
   - label: 🧪 Test, Lint & Build
     artifact_paths: lib/**/*
     commands:
-      - echo '+++ yarn test:ci'
-      - yarn test:ci
-      - echo '--- yarn lint'
-      - yarn lint
-      - echo '--- yarn build'
-      - yarn build
+      - echo '--- pnpm install --offline'
+      - pnpm install --offline
+      - echo '+++ pnpm run test:ci'
+      - pnpm run test:ci
+      - echo '--- pnpm run lint'
+      - pnpm run lint
+      - echo '--- pnpm run build'
+      - pnpm run build
     env:
       GET_GITHUB_TOKEN: please
     plugins:
@@ -63,6 +66,9 @@ steps:
       - *docker-ecr-cache
       - docker-compose#v4.16.0:
           run: app
+          environment:
+            - GITHUB_API_TOKEN
+          propagate-environment: true
     timeout_in_minutes: 10
 
   - agents:
@@ -73,7 +79,7 @@ steps:
     plugins:
       - *aws-sm
       - *private-npm
-      - seek-oss/docker-ecr-cache#v2.1.0:
+      - seek-oss/docker-ecr-cache#v2.1.1:
           <<: *docker-ecr-cache-defaults
           skip-pull-from-cache: true
 

package/template/lambda-sqs-worker/.nvmrc

@@ -1 +1 @@
-18
+20

package/template/lambda-sqs-worker/Dockerfile

@@ -1,12 +1,12 @@
-# syntax=docker/dockerfile:1.2
+# syntax=docker/dockerfile:1.6
 
-FROM --platform=${BUILDPLATFORM:-<%- platformName %>} node:18-alpine AS dev-deps
+FROM --platform=${BUILDPLATFORM:-<%- platformName %>} node:20-alpine AS dev-deps
 
-WORKDIR /workdir
+RUN corepack enable pnpm
+RUN pnpm config set store-dir /root/.pnpm-store
 
-COPY package.json yarn.lock ./
+WORKDIR /workdir
 
-RUN \
-  --mount=type=secret,id=npm,dst=/root/.npmrc \
-  yarn install --frozen-lockfile --ignore-optional --non-interactive && \
-  yarn cache clean
+RUN --mount=type=bind,source=pnpm-lock.yaml,target=pnpm-lock.yaml \
+    --mount=type=secret,id=npm,dst=/root/.npmrc,required=true \
+    pnpm fetch

package/template/lambda-sqs-worker/README.md

@@ -7,7 +7,7 @@ Next steps:
 1. [ ] Finish templating if this was skipped earlier:
 
    ```shell
-   yarn skuba configure
+   pnpm exec skuba configure
    ```
 
 2. [ ] Create a new repository in the appropriate GitHub organisation.
@@ -47,33 +47,33 @@ This defaults to an invocation with an empty object `{}`, per [src/hooks.ts](src
 
 ```shell
 # Run Jest tests locally
-yarn test
+pnpm run test
 
 # Authenticate to dev account
 awsauth
 
 # Run smoke test against deployed application
-ENVIRONMENT=dev yarn smoke
+ENVIRONMENT=dev pnpm run smoke
 ```
 
 ### Lint
 
 ```shell
 # Fix issues
-yarn format
+pnpm run format
 
 # Check for issues
-yarn lint
+pnpm run lint
 ```
 
 ### Start
 
 ```shell
 # Start a local HTTP server
-yarn start
+pnpm run start
 
 # Start with Node.js Inspector enabled
-yarn start:debug
+pnpm run start:debug
 ```
 
 This serves the Lambda application over HTTP.
@@ -96,7 +96,7 @@ To deploy locally:
 # Authenticate to dev account
 awsauth
 
-ENVIRONMENT=dev yarn deploy
+ENVIRONMENT=dev pnpm run deploy
 ```
 
 To rapidly roll back a change,

package/template/lambda-sqs-worker/_.npmrc

@@ -0,0 +1,12 @@
+# managed by skuba
+public-hoist-pattern[]="@types*"
+public-hoist-pattern[]="*eslint*"
+public-hoist-pattern[]="*prettier*"
+public-hoist-pattern[]="esbuild"
+public-hoist-pattern[]="jest"
+public-hoist-pattern[]="tsconfig-seek"
+# end managed by skuba
+
+# Required for Serverless packaging
+node-linker=hoisted
+shamefully-hoist=true

package/template/lambda-sqs-worker/docker-compose.yml

@@ -2,21 +2,6 @@ version: '3.7'
 
 services:
   app:
-    environment:
-      # Enable Buildkite + GitHub integrations.
-      - BUILDKITE
-      - BUILDKITE_AGENT_ACCESS_TOKEN
-      - BUILDKITE_BRANCH
-      - BUILDKITE_BUILD_NUMBER
-      - BUILDKITE_JOB_ID
-      - BUILDKITE_PIPELINE_DEFAULT_BRANCH
-      - BUILDKITE_STEP_ID
-      - GITHUB_API_TOKEN
-      # Tag AWS resources with the commit hash.
-      - BUILDKITE_COMMIT
-      # Pass through application configuration.
-      - ENVIRONMENT
-      - VERSION
     image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''}
     init: true
     volumes:

package/template/lambda-sqs-worker/package.json

@@ -8,7 +8,7 @@
     "lint": "skuba lint",
     "smoke": "serverless invoke --data '{}' --function Worker",
     "start": "skuba start --port <%- port %>",
-    "start:debug": "yarn start --inspect-brk",
+    "start:debug": "pnpm run --silent start --inspect-brk",
     "test": "skuba test",
     "test:ci": "skuba test --coverage",
     "test:watch": "skuba test --watch"
@@ -17,28 +17,27 @@
     "@aws-sdk/client-codedeploy": "^3.363.0",
     "@aws-sdk/client-lambda": "^3.363.0",
     "@aws-sdk/client-sns": "^3.363.0",
-    "@aws-sdk/util-utf8-node": "^3.259.0",
     "@seek/logger": "^6.0.0",
-    "datadog-lambda-js": "^7.0.0",
-    "dd-trace": "^4.0.0",
+    "datadog-lambda-js": "^7.102.0",
+    "dd-trace": "^5.0.0",
     "skuba-dive": "^2.0.0",
     "zod": "^3.19.1"
   },
   "devDependencies": {
     "@types/aws-lambda": "^8.10.84",
     "@types/chance": "^1.1.3",
-    "@types/node": "^18.11.9",
+    "@types/node": "^20.9.0",
     "aws-sdk-client-mock": "^3.0.0",
     "aws-sdk-client-mock-jest": "^3.0.0",
     "chance": "^1.1.8",
     "pino-pretty": "^10.0.0",
-    "serverless": "^3.25.0",
+    "serverless": "^3.37.0",
     "serverless-plugin-canary-deployments": "^0.8.0",
     "serverless-plugin-datadog": "^5.12.0",
     "serverless-prune-plugin": "^2.0.0",
     "skuba": "*"
   },
   "engines": {
-    "node": ">=18.12"
+    "node": ">=20"
   }
 }

package/template/lambda-sqs-worker/serverless.yml

@@ -9,9 +9,11 @@ params:
   dev:
     deploymentBucket: 'TODO: deployment-bucket-name'
     isProduction: false
+    concurrency: 2
   prod:
     deploymentBucket: 'TODO: deployment-bucket-name'
     isProduction: true
+    concurrency: 20
 
 custom:
   datadog:
@@ -37,7 +39,7 @@ provider:
   logRetentionInDays: 30
   name: aws
   region: ap-southeast-2
-  runtime: nodejs18.x
+  runtime: nodejs20.x
   architecture: <%- lambdaServerlessArchitecture %>
   deploymentMethod: direct
   stackName: ${self:service}
@@ -91,7 +93,7 @@ functions:
     handler: lib/app.handler
     description: ${param:description}
     memorySize: 128
-    reservedConcurrency: 20
+    reservedConcurrency: ${param:concurrency}
     timeout: 30
     deploymentSettings:
       type: AllAtOnce
@@ -107,6 +109,7 @@ functions:
       - sqs:
           arn: !GetAtt MessageQueue.Arn
           batchSize: 1
+          maximumConcurrency: ${param:concurrency}
   WorkerPreHook:
     name: ${self:functions.Worker.name}-pre-hook
     handler: lib/hooks.pre

package/template/lambda-sqs-worker/skuba.template.js

@@ -28,5 +28,6 @@ module.exports = {
       validate: (value) => /^.+:.+$/.test(value),
     },
   ],
+  packageManager: 'pnpm',
   type: 'application',
 };

package/template/lambda-sqs-worker/src/hooks.ts

@@ -7,7 +7,6 @@ import {
   PutLifecycleEventHookExecutionStatusCommand,
 } from '@aws-sdk/client-codedeploy';
 import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda';
-import { toUtf8 } from '@aws-sdk/util-utf8-node';
 
 const codeDeploy = new CodeDeployClient({
   apiVersion: '2014-10-06',
@@ -51,7 +50,7 @@ const smokeTestLambdaFunction = async (): Promise<Status> => {
   if (response.FunctionError) {
     console.error('Error:', response.FunctionError);
     if (response.Payload) {
-      console.error(toUtf8(response.Payload));
+      console.error(response.Payload.transformToString());
     }
     return 'Failed';
   }

package/template/lambda-sqs-worker-cdk/.buildkite/pipeline.yml

@@ -4,15 +4,13 @@ agents:
 configs:
   plugins:
     - &aws-sm
-      seek-oss/aws-sm#v2.3.1:
+      seek-oss/aws-sm#v2.3.2:
         env:
           NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
 
     - &docker-ecr-cache
-      seek-oss/docker-ecr-cache#v2.1.0: &docker-ecr-cache-defaults
-        cache-on:
-          - package.json
-          - yarn.lock
+      seek-oss/docker-ecr-cache#v2.1.1: &docker-ecr-cache-defaults
+        cache-on: pnpm-lock.yaml
         secrets: id=npm,src=tmp/.npmrc
 
     - &private-npm
@@ -23,35 +21,39 @@ configs:
   base-steps:
     - &deploy
       commands:
-        - echo '+++ yarn deploy'
-        - yarn deploy
+        - echo '--- pnpm install --offline'
+        - pnpm install --offline
+        - echo '+++ pnpm run deploy'
+        - pnpm run deploy
       concurrency: 1
       plugins:
-        - artifacts#v1.9.2:
-            build: ${BUILDKITE_BUILD_ID}
-            download: lib/*
         - *aws-sm
         - *private-npm
         - *docker-ecr-cache
         - docker-compose#v4.16.0:
             dependencies: false
             run: app
+            environment:
+              - GITHUB_API_TOKEN
+            propagate-environment: true
       retry:
         manual:
           # Only use this if you need to roll back a deployment ASAP.
           # Always follow up with a proper revert or fix in Git history.
           permit_on_passed: true
 
+env:
+  VERSION: ${BUILDKITE_COMMIT:0:7}.${BUILDKITE_BUILD_NUMBER}
+
 steps:
   - label: 🧪 Test, Lint & Build
-    artifact_paths: lib/**/*
     commands:
-      - echo '+++ yarn test:ci'
-      - yarn test
-      - echo '--- yarn lint'
-      - yarn lint
-      - echo '--- yarn build'
-      - yarn build
+      - echo '--- pnpm install --offline'
+      - pnpm install --offline
+      - echo '+++ pnpm run test:ci'
+      - pnpm run test
+      - echo '--- pnpm run lint'
+      - pnpm run lint
     env:
       GET_GITHUB_TOKEN: please
     plugins:
@@ -60,6 +62,9 @@ steps:
       - *docker-ecr-cache
       - docker-compose#v4.16.0:
           run: app
+          environment:
+            - GITHUB_API_TOKEN
+          propagate-environment: true
     timeout_in_minutes: 10
 
   - agents:
@@ -70,7 +75,7 @@ steps:
     plugins:
       - *aws-sm
       - *private-npm
-      - seek-oss/docker-ecr-cache#v2.1.0:
+      - seek-oss/docker-ecr-cache#v2.1.1:
           <<: *docker-ecr-cache-defaults
           skip-pull-from-cache: true
 
@@ -87,6 +92,25 @@ steps:
     concurrency_group: '<%- repoName %>/deploy/dev'
     key: deploy-dev
 
+  - block: 🙋🏻‍♀️ Deploy Dev (Hotswap)
+    key: deploy-dev-hotswap-block
+    branches: '!${BUILDKITE_PIPELINE_DEFAULT_BRANCH}'
+
+  - <<: *deploy
+    branches: '!${BUILDKITE_PIPELINE_DEFAULT_BRANCH}'
+    depends_on: deploy-dev-hotswap-block
+    agents:
+      queue: <%- devBuildkiteQueueName %>
+    env:
+      ENVIRONMENT: dev
+    commands:
+      - echo '--- pnpm install --offline'
+      - pnpm install --offline
+      - echo '+++ pnpm run deploy:hotswap'
+      - pnpm run deploy:hotswap
+    label: 🤞 Deploy Dev (Hotswap)
+    concurrency_group: '<%- repoName %>/deploy/dev'
+
   - <<: *deploy
     env:
       ENVIRONMENT: prod

package/template/lambda-sqs-worker-cdk/.nvmrc

@@ -1 +1 @@
-18
+20

package/template/lambda-sqs-worker-cdk/Dockerfile

@@ -1,13 +1,15 @@
-# syntax=docker/dockerfile:1.2
+# syntax=docker/dockerfile:1.6
 
-FROM --platform=${BUILDPLATFORM:-<%- platformName %>} node:18-alpine AS dev-deps
+FROM --platform=${BUILDPLATFORM:-<%- platformName %>} node:20-alpine AS dev-deps
 
-WORKDIR /workdir
+# Needed for cdk
+RUN apk add --no-cache bash
+
+RUN corepack enable pnpm
+RUN pnpm config set store-dir /root/.pnpm-store
 
-COPY package.json yarn.lock ./
+WORKDIR /workdir
 
-RUN \
-  --mount=type=secret,id=npm,dst=/root/.npmrc \
-  yarn install --frozen-lockfile --ignore-optional --non-interactive && \
-  yarn package && \
-  yarn cache clean
+RUN --mount=type=bind,source=pnpm-lock.yaml,target=pnpm-lock.yaml \
+    --mount=type=secret,id=npm,dst=/root/.npmrc,required=true \
+    pnpm fetch

package/template/lambda-sqs-worker-cdk/cdk.json

@@ -1,25 +1,31 @@
 {
-  "app": "npx ts-node infra/index.ts",
+  "app": "pnpm exec skuba node infra/index.ts",
   "context": {
     "global": {
       "appName": "<%- serviceName %>"
     },
     "dev": {
       "workerLambda": {
-        "reservedConcurrency": 1,
+        "reservedConcurrency": 2,
         "environment": {
           "SOMETHING": "dev"
         }
-      }
+      },
+      "sourceSnsTopicArn": "TODO: sourceSnsTopicArn"
     },
     "prod": {
       "workerLambda": {
-        "reservedConcurrency": 2,
+        "reservedConcurrency": 20,
         "environment": {
           "SOMETHING": "prod"
         }
-      }
+      },
+      "sourceSnsTopicArn": "TODO: sourceSnsTopicArn"
     }
   },
-  "progress": "events"
+  "progress": "events",
+  "watch": {
+    "include": "src/**/*.ts",
+    "exclude": "src/**/*.test.ts"
+  }
 }

package/template/lambda-sqs-worker-cdk/docker-compose.yml

@@ -2,21 +2,6 @@ version: '3.7'
 
 services:
   app:
-    environment:
-      # Enable Buildkite + GitHub integrations.
-      - BUILDKITE
-      - BUILDKITE_AGENT_ACCESS_TOKEN
-      - BUILDKITE_BRANCH
-      - BUILDKITE_BUILD_NUMBER
-      - BUILDKITE_JOB_ID
-      - BUILDKITE_PIPELINE_DEFAULT_BRANCH
-      - BUILDKITE_STEP_ID
-      - GITHUB_API_TOKEN
-      # Tag AWS resources with the commit hash.
-      - BUILDKITE_COMMIT
-      # Pass through application configuration.
-      - ENVIRONMENT
-      - VERSION
     image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''}
     init: true
     volumes: