skuba 0.0.0-master-20231121000709 → 0.0.0-master-20240206001217

package/template/lambda-sqs-worker/.buildkite/pipeline.yml

@@ -4,15 +4,13 @@ agents:
 configs:
   plugins:
     - &aws-sm
-      seek-oss/aws-sm#v2.3.1:
+      seek-oss/aws-sm#v2.3.2:
         env:
           NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
 
     - &docker-ecr-cache
-      seek-oss/docker-ecr-cache#v2.1.0: &docker-ecr-cache-defaults
-        cache-on:
-          - package.json
-          - yarn.lock
+      seek-oss/docker-ecr-cache#v2.1.1: &docker-ecr-cache-defaults
+        cache-on: pnpm-lock.yaml
         secrets: id=npm,src=tmp/.npmrc
 
     - &private-npm
@@ -23,11 +21,13 @@ configs:
   base-steps:
     - &deploy
       commands:
-        - echo '+++ yarn deploy'
-        - yarn deploy
+        - echo '--- pnpm install --offline'
+        - pnpm install --offline
+        - echo '+++ pnpm run deploy'
+        - pnpm run deploy
       concurrency: 1
       plugins:
-        - artifacts#v1.9.2:
+        - artifacts#v1.9.3:
             build: ${BUILDKITE_BUILD_ID}
             download: lib/*
         - *aws-sm
@@ -36,6 +36,7 @@ configs:
         - docker-compose#v4.16.0:
             dependencies: false
             run: app
+            propagate-environment: true
       retry:
         manual:
           # Only use this if you need to roll back a deployment ASAP.
@@ -49,12 +50,14 @@ steps:
   - label: 🧪 Test, Lint & Build
     artifact_paths: lib/**/*
     commands:
-      - echo '+++ yarn test:ci'
-      - yarn test:ci
-      - echo '--- yarn lint'
-      - yarn lint
-      - echo '--- yarn build'
-      - yarn build
+      - echo '--- pnpm install --offline'
+      - pnpm install --offline
+      - echo '+++ pnpm run test:ci'
+      - pnpm run test:ci
+      - echo '--- pnpm run lint'
+      - pnpm run lint
+      - echo '--- pnpm run build'
+      - pnpm run build
     env:
       GET_GITHUB_TOKEN: please
     plugins:
@@ -63,6 +66,9 @@ steps:
       - *docker-ecr-cache
       - docker-compose#v4.16.0:
           run: app
+          environment:
+            - GITHUB_API_TOKEN
+          propagate-environment: true
     timeout_in_minutes: 10
 
   - agents:
@@ -73,7 +79,7 @@ steps:
     plugins:
       - *aws-sm
       - *private-npm
-      - seek-oss/docker-ecr-cache#v2.1.0:
+      - seek-oss/docker-ecr-cache#v2.1.1:
           <<: *docker-ecr-cache-defaults
           skip-pull-from-cache: true
 

package/template/lambda-sqs-worker/Dockerfile

@@ -2,11 +2,11 @@
 
 FROM --platform=${BUILDPLATFORM:-<%- platformName %>} node:20-alpine AS dev-deps
 
-WORKDIR /workdir
+RUN corepack enable pnpm
+RUN pnpm config set store-dir /root/.pnpm-store
 
-COPY package.json yarn.lock ./
+WORKDIR /workdir
 
-RUN \
-  --mount=type=secret,id=npm,dst=/root/.npmrc \
-  yarn install --frozen-lockfile --ignore-optional --non-interactive && \
-  yarn cache clean
+RUN --mount=type=bind,source=pnpm-lock.yaml,target=pnpm-lock.yaml \
+    --mount=type=secret,id=npm,dst=/root/.npmrc,required=true \
+    pnpm fetch

package/template/lambda-sqs-worker/README.md

@@ -7,7 +7,7 @@ Next steps:
 1. [ ] Finish templating if this was skipped earlier:
 
    ```shell
-   yarn skuba configure
+   pnpm exec skuba configure
    ```
 
 2. [ ] Create a new repository in the appropriate GitHub organisation.
@@ -47,33 +47,33 @@ This defaults to an invocation with an empty object `{}`, per [src/hooks.ts](src
 
 ```shell
 # Run Jest tests locally
-yarn test
+pnpm run test
 
 # Authenticate to dev account
 awsauth
 
 # Run smoke test against deployed application
-ENVIRONMENT=dev yarn smoke
+ENVIRONMENT=dev pnpm run smoke
 ```
 
 ### Lint
 
 ```shell
 # Fix issues
-yarn format
+pnpm run format
 
 # Check for issues
-yarn lint
+pnpm run lint
 ```
 
 ### Start
 
 ```shell
 # Start a local HTTP server
-yarn start
+pnpm run start
 
 # Start with Node.js Inspector enabled
-yarn start:debug
+pnpm run start:debug
 ```
 
 This serves the Lambda application over HTTP.
@@ -96,7 +96,7 @@ To deploy locally:
 # Authenticate to dev account
 awsauth
 
-ENVIRONMENT=dev yarn deploy
+ENVIRONMENT=dev pnpm run deploy
 ```
 
 To rapidly roll back a change,

package/template/lambda-sqs-worker/_.npmrc

@@ -0,0 +1,12 @@
+# managed by skuba
+public-hoist-pattern[]="@types*"
+public-hoist-pattern[]="*eslint*"
+public-hoist-pattern[]="*prettier*"
+public-hoist-pattern[]="esbuild"
+public-hoist-pattern[]="jest"
+public-hoist-pattern[]="tsconfig-seek"
+# end managed by skuba
+
+# Required for Serverless packaging
+node-linker=hoisted
+shamefully-hoist=true

package/template/lambda-sqs-worker/docker-compose.yml

@@ -2,21 +2,6 @@ version: '3.7'
 
 services:
   app:
-    environment:
-      # Enable Buildkite + GitHub integrations.
-      - BUILDKITE
-      - BUILDKITE_AGENT_ACCESS_TOKEN
-      - BUILDKITE_BRANCH
-      - BUILDKITE_BUILD_NUMBER
-      - BUILDKITE_JOB_ID
-      - BUILDKITE_PIPELINE_DEFAULT_BRANCH
-      - BUILDKITE_STEP_ID
-      - GITHUB_API_TOKEN
-      # Tag AWS resources with the commit hash.
-      - BUILDKITE_COMMIT
-      # Pass through application configuration.
-      - ENVIRONMENT
-      - VERSION
     image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''}
     init: true
     volumes:

package/template/lambda-sqs-worker/package.json

@@ -8,7 +8,7 @@
     "lint": "skuba lint",
     "smoke": "serverless invoke --data '{}' --function Worker",
     "start": "skuba start --port <%- port %>",
-    "start:debug": "yarn start --inspect-brk",
+    "start:debug": "pnpm run --silent start --inspect-brk",
     "test": "skuba test",
     "test:ci": "skuba test --coverage",
     "test:watch": "skuba test --watch"
@@ -17,10 +17,9 @@
     "@aws-sdk/client-codedeploy": "^3.363.0",
     "@aws-sdk/client-lambda": "^3.363.0",
     "@aws-sdk/client-sns": "^3.363.0",
-    "@aws-sdk/util-utf8-node": "^3.259.0",
     "@seek/logger": "^6.0.0",
-    "datadog-lambda-js": "^7.0.0",
-    "dd-trace": "^4.0.0",
+    "datadog-lambda-js": "^7.102.0",
+    "dd-trace": "^5.0.0",
     "skuba-dive": "^2.0.0",
     "zod": "^3.19.1"
   },

package/template/lambda-sqs-worker/serverless.yml

@@ -9,9 +9,11 @@ params:
   dev:
     deploymentBucket: 'TODO: deployment-bucket-name'
     isProduction: false
+    concurrency: 2
   prod:
     deploymentBucket: 'TODO: deployment-bucket-name'
     isProduction: true
+    concurrency: 20
 
 custom:
   datadog:
@@ -91,7 +93,7 @@ functions:
     handler: lib/app.handler
     description: ${param:description}
     memorySize: 128
-    reservedConcurrency: 20
+    reservedConcurrency: ${param:concurrency}
     timeout: 30
     deploymentSettings:
       type: AllAtOnce
@@ -107,6 +109,7 @@ functions:
       - sqs:
           arn: !GetAtt MessageQueue.Arn
           batchSize: 1
+          maximumConcurrency: ${param:concurrency}
   WorkerPreHook:
     name: ${self:functions.Worker.name}-pre-hook
     handler: lib/hooks.pre

package/template/lambda-sqs-worker/skuba.template.js

@@ -28,5 +28,6 @@ module.exports = {
       validate: (value) => /^.+:.+$/.test(value),
     },
   ],
+  packageManager: 'pnpm',
   type: 'application',
 };

package/template/lambda-sqs-worker/src/hooks.ts

@@ -7,7 +7,6 @@ import {
   PutLifecycleEventHookExecutionStatusCommand,
 } from '@aws-sdk/client-codedeploy';
 import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda';
-import { toUtf8 } from '@aws-sdk/util-utf8-node';
 
 const codeDeploy = new CodeDeployClient({
   apiVersion: '2014-10-06',
@@ -51,7 +50,7 @@ const smokeTestLambdaFunction = async (): Promise<Status> => {
   if (response.FunctionError) {
     console.error('Error:', response.FunctionError);
     if (response.Payload) {
-      console.error(toUtf8(response.Payload));
+      console.error(response.Payload.transformToString());
     }
     return 'Failed';
   }

package/template/lambda-sqs-worker-cdk/.buildkite/pipeline.yml

@@ -4,15 +4,13 @@ agents:
 configs:
   plugins:
     - &aws-sm
-      seek-oss/aws-sm#v2.3.1:
+      seek-oss/aws-sm#v2.3.2:
         env:
           NPM_READ_TOKEN: arn:aws:secretsmanager:ap-southeast-2:987872074697:secret:npm/npm-read-token
 
     - &docker-ecr-cache
-      seek-oss/docker-ecr-cache#v2.1.0: &docker-ecr-cache-defaults
-        cache-on:
-          - package.json
-          - yarn.lock
+      seek-oss/docker-ecr-cache#v2.1.1: &docker-ecr-cache-defaults
+        cache-on: pnpm-lock.yaml
         secrets: id=npm,src=tmp/.npmrc
 
     - &private-npm
@@ -23,35 +21,39 @@ configs:
   base-steps:
     - &deploy
       commands:
-        - echo '+++ yarn deploy'
-        - yarn deploy
+        - echo '--- pnpm install --offline'
+        - pnpm install --offline
+        - echo '+++ pnpm run deploy'
+        - pnpm run deploy
       concurrency: 1
       plugins:
-        - artifacts#v1.9.2:
-            build: ${BUILDKITE_BUILD_ID}
-            download: lib/*
         - *aws-sm
         - *private-npm
         - *docker-ecr-cache
         - docker-compose#v4.16.0:
             dependencies: false
             run: app
+            environment:
+              - GITHUB_API_TOKEN
+            propagate-environment: true
       retry:
         manual:
           # Only use this if you need to roll back a deployment ASAP.
           # Always follow up with a proper revert or fix in Git history.
           permit_on_passed: true
 
+env:
+  VERSION: ${BUILDKITE_COMMIT:0:7}.${BUILDKITE_BUILD_NUMBER}
+
 steps:
   - label: 🧪 Test, Lint & Build
-    artifact_paths: lib/**/*
     commands:
-      - echo '+++ yarn test:ci'
-      - yarn test
-      - echo '--- yarn lint'
-      - yarn lint
-      - echo '--- yarn build'
-      - yarn build
+      - echo '--- pnpm install --offline'
+      - pnpm install --offline
+      - echo '+++ pnpm run test:ci'
+      - pnpm run test
+      - echo '--- pnpm run lint'
+      - pnpm run lint
     env:
       GET_GITHUB_TOKEN: please
     plugins:
@@ -60,6 +62,9 @@ steps:
       - *docker-ecr-cache
       - docker-compose#v4.16.0:
           run: app
+          environment:
+            - GITHUB_API_TOKEN
+          propagate-environment: true
     timeout_in_minutes: 10
 
   - agents:
@@ -70,7 +75,7 @@ steps:
     plugins:
       - *aws-sm
       - *private-npm
-      - seek-oss/docker-ecr-cache#v2.1.0:
+      - seek-oss/docker-ecr-cache#v2.1.1:
           <<: *docker-ecr-cache-defaults
           skip-pull-from-cache: true
 
@@ -87,6 +92,25 @@ steps:
     concurrency_group: '<%- repoName %>/deploy/dev'
     key: deploy-dev
 
+  - block: 🙋🏻‍♀️ Deploy Dev (Hotswap)
+    key: deploy-dev-hotswap-block
+    branches: '!${BUILDKITE_PIPELINE_DEFAULT_BRANCH}'
+
+  - <<: *deploy
+    branches: '!${BUILDKITE_PIPELINE_DEFAULT_BRANCH}'
+    depends_on: deploy-dev-hotswap-block
+    agents:
+      queue: <%- devBuildkiteQueueName %>
+    env:
+      ENVIRONMENT: dev
+    commands:
+      - echo '--- pnpm install --offline'
+      - pnpm install --offline
+      - echo '+++ pnpm run deploy:hotswap'
+      - pnpm run deploy:hotswap
+    label: 🤞 Deploy Dev (Hotswap)
+    concurrency_group: '<%- repoName %>/deploy/dev'
+
   - <<: *deploy
     env:
       ENVIRONMENT: prod

package/template/lambda-sqs-worker-cdk/.nvmrc

@@ -1 +1 @@
-18
+20

package/template/lambda-sqs-worker-cdk/Dockerfile

@@ -2,12 +2,14 @@
 
 FROM --platform=${BUILDPLATFORM:-<%- platformName %>} node:20-alpine AS dev-deps
 
-WORKDIR /workdir
+# Needed for cdk
+RUN apk add --no-cache bash
+
+RUN corepack enable pnpm
+RUN pnpm config set store-dir /root/.pnpm-store
 
-COPY package.json yarn.lock ./
+WORKDIR /workdir
 
-RUN \
-  --mount=type=secret,id=npm,dst=/root/.npmrc \
-  yarn install --frozen-lockfile --ignore-optional --non-interactive && \
-  yarn package && \
-  yarn cache clean
+RUN --mount=type=bind,source=pnpm-lock.yaml,target=pnpm-lock.yaml \
+    --mount=type=secret,id=npm,dst=/root/.npmrc,required=true \
+    pnpm fetch

package/template/lambda-sqs-worker-cdk/cdk.json

@@ -1,25 +1,31 @@
 {
-  "app": "npx ts-node infra/index.ts",
+  "app": "pnpm exec skuba node infra/index.ts",
   "context": {
     "global": {
       "appName": "<%- serviceName %>"
     },
     "dev": {
       "workerLambda": {
-        "reservedConcurrency": 1,
+        "reservedConcurrency": 2,
         "environment": {
           "SOMETHING": "dev"
         }
-      }
+      },
+      "sourceSnsTopicArn": "TODO: sourceSnsTopicArn"
     },
     "prod": {
       "workerLambda": {
-        "reservedConcurrency": 2,
+        "reservedConcurrency": 20,
         "environment": {
           "SOMETHING": "prod"
         }
-      }
+      },
+      "sourceSnsTopicArn": "TODO: sourceSnsTopicArn"
     }
   },
-  "progress": "events"
+  "progress": "events",
+  "watch": {
+    "include": "src/**/*.ts",
+    "exclude": "src/**/*.test.ts"
+  }
 }

package/template/lambda-sqs-worker-cdk/docker-compose.yml

@@ -2,21 +2,6 @@ version: '3.7'
 
 services:
   app:
-    environment:
-      # Enable Buildkite + GitHub integrations.
-      - BUILDKITE
-      - BUILDKITE_AGENT_ACCESS_TOKEN
-      - BUILDKITE_BRANCH
-      - BUILDKITE_BUILD_NUMBER
-      - BUILDKITE_JOB_ID
-      - BUILDKITE_PIPELINE_DEFAULT_BRANCH
-      - BUILDKITE_STEP_ID
-      - GITHUB_API_TOKEN
-      # Tag AWS resources with the commit hash.
-      - BUILDKITE_COMMIT
-      # Pass through application configuration.
-      - ENVIRONMENT
-      - VERSION
     image: ${BUILDKITE_PLUGIN_DOCKER_IMAGE:-''}
     init: true
     volumes: