skuba 9.0.0 → 9.0.1-upgrade-cdk-template-20241002233314

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skuba",
-  "version": "9.0.0",
+  "version": "9.0.1-upgrade-cdk-template-20241002233314",
   "private": false,
   "description": "SEEK development toolkit for backend applications and packages",
   "homepage": "https://github.com/seek-oss/skuba#readme",

package/template/greeter/package.json

@@ -17,7 +17,7 @@
   },
   "devDependencies": {
     "@types/node": "^20.9.0",
-    "skuba": "*"
+    "skuba": "9.0.1-upgrade-cdk-template-20241002233314"
   },
   "packageManager": "pnpm@9.11.0",
   "engines": {

package/template/lambda-sqs-worker/serverless.yml

@@ -173,7 +173,7 @@ resources:
     #   Properties:
     #     Endpoint: !GetAtt MessageQueue.Arn
     #     Protocol: sqs
-    #     RawMessageDelivery: true
+    #     RawMessageDelivery: true # Remove this property if you require end to end datadog tracing
     #     TopicArn: 'TODO: sourceSnsTopicArn'
 
     DestinationTopic:

package/template/lambda-sqs-worker-cdk/.env

@@ -0,0 +1 @@
+ENVIRONMENT=local

package/template/lambda-sqs-worker-cdk/README.md

@@ -0,0 +1,145 @@
+# <%- repoName %>
+
+[![Powered by skuba](https://img.shields.io/badge/🤿%20skuba-powered-009DC4)](https://github.com/seek-oss/skuba)
+
+Next steps:
+
+1. [ ] Finish templating if this was skipped earlier:
+
+   ```shell
+   pnpm exec skuba configure
+   ```
+
+2. [ ] Create a new repository in the appropriate GitHub organisation.
+3. [ ] Add the repository to BuildAgency;
+       see our internal [Buildkite Docs] for more information.
+4. [ ] Add Datadog extension, deployment bucket configuration and data classification tags to [infra/config.ts](infra/config.ts).
+5. [ ] Push local commits to the upstream GitHub branch.
+6. [ ] Configure [GitHub repository settings].
+7. [ ] Delete this checklist 😌.
+
+[Buildkite Docs]: https://backstage.myseek.xyz/docs/default/component/buildkite-docs
+[GitHub repository settings]: https://github.com/<%-orgName%>/<%-repoName%>/settings
+
+## Design
+
+<%-repoName %> is a Node.js [Lambda] application built in line with our [Technical Guidelines].
+It is backed by a typical SQS message + dead letter queue configuration and uses common SEEK packages.
+Workers enable fault-tolerant asynchronous processing of events.
+
+The `lambda-sqs-worker-cdk` template is modelled after a hypothetical enricher that scores job advertisements.
+It's stubbed out with in-memory [scoring service](src/services/jobScorer.ts).
+This would be replaced with internal logic or an external service in production.
+
+This project is deployed with [AWS CDK].
+The Lambda runtime provisions a single Node.js process per container.
+The supplied [infra/appStack.ts](infra/appStack.ts) starts out with a minimal `memorySize` which may require tuning based on workload.
+Under load, we autoscale horizontally in terms of container count up to `reservedConcurrency`.
+
+[@seek/aws-codedeploy-hooks] configures [CodeDeploy] for a blue-green deployment approach.
+A smoke test is run against the new version before traffic is switched over,
+providing an opportunity to test access and connectivity to online dependencies.
+This defaults to an invocation with an empty object `{}`.
+
+## Development
+
+### Test
+
+```shell
+# Run Jest tests locally
+pnpm test
+
+# Authenticate to dev account
+awsauth
+
+# Run smoke test against deployed application
+ENVIRONMENT=dev pnpm smoke
+```
+
+### Lint
+
+```shell
+# Fix issues
+pnpm format
+
+# Check for issues
+pnpm lint
+```
+
+### Start
+
+```shell
+# Start a local HTTP server
+pnpm start
+
+# Start with Node.js Inspector enabled
+pnpm start:debug
+```
+
+This serves the Lambda application over HTTP.
+For example, to invoke the handler with an empty object `{}` for smoke testing:
+
+```shell
+curl --data '[{}, {"awsRequestId": "local"}]' --include localhost:<%- port %>
+```
+
+### Deploy
+
+This project is deployed through a [Buildkite pipeline](.buildkite/pipeline.yml).
+
+- Commits to a feature branch can be deployed to the dev environment by unblocking a step in the Buildkite UI
+- Commits to the default branch are automatically deployed to the dev and prod environments in sequence
+
+To deploy locally:
+
+```shell
+# Authenticate to dev account
+awsauth
+
+ENVIRONMENT=dev pnpm run deploy
+```
+
+A hotswap deploy enables faster deployment but come with caveats such as requiring a Lambda to be rebuilt with every build.
+
+To deploy a [hotswap]:
+
+```shell
+# Authenticate to dev account
+awsauth
+
+ENVIRONMENT=dev pnpm run deploy:hotswap
+```
+
+To rapidly roll back a change,
+retry an individual deployment step from the previous build in Buildkite.
+Note that this will introduce drift between the head of the default Git branch and the live environment;
+use with caution and always follow up with a proper revert or fix in Git history.
+
+## Support
+
+### Dev
+
+TODO: add support links for the dev environment.
+
+<!--
+- CloudWatch dashboard
+- Datadog dashboard
+- Splunk logs
+-->
+
+### Prod
+
+TODO: add support links for the prod environment.
+
+<!--
+- CloudWatch dashboard
+- Datadog dashboard
+- Splunk logs
+-->
+
+[@seek/aws-codedeploy-hooks]: https://github.com/seek-oss/aws-codedeploy-hooks
+[AWS CDK]: https://docs.aws.amazon.com/cdk/v2/guide/home.html
+[CodeDeploy]: https://docs.aws.amazon.com/codedeploy
+[Hotswap]: https://docs.aws.amazon.com/cdk/v2/guide/ref-cli-cmd-deploy.html#ref-cli-cmd-deploy-options
+[Lambda]: https://docs.aws.amazon.com/lambda
+[Technical Guidelines]: https://myseek.atlassian.net/wiki/spaces/AA/pages/2358346017/

package/template/lambda-sqs-worker-cdk/infra/__snapshots__/appStack.test.ts.snap

@@ -10,6 +10,40 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
     },
   },
   "Resources": {
+    "datadogapikeysecret046FEF06": {
+      "DeletionPolicy": "Delete",
+      "Properties": {
+        "GenerateSecretString": {},
+      },
+      "Type": "AWS::SecretsManager::Secret",
+      "UpdateReplacePolicy": "Delete",
+    },
+    "destinationtopicDCE2E0B8": {
+      "Properties": {
+        "KmsMasterKeyId": {
+          "Fn::Join": [
+            "",
+            [
+              "arn:",
+              {
+                "Ref": "AWS::Partition",
+              },
+              ":kms:",
+              {
+                "Ref": "AWS::Region",
+              },
+              ":",
+              {
+                "Ref": "AWS::AccountId",
+              },
+              ":alias/aws/sns",
+            ],
+          ],
+        },
+        "TopicName": "serviceName",
+      },
+      "Type": "AWS::SNS::Topic",
+    },
     "kmskey49FBC3B3": {
       "DeletionPolicy": "Retain",
       "Properties": {
@@ -105,17 +139,6 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
               },
               "Resource": "*",
             },
-            {
-              "Action": [
-                "kms:Decrypt",
-                "kms:GenerateDataKey",
-              ],
-              "Effect": "Allow",
-              "Principal": {
-                "Service": "sns.amazonaws.com",
-              },
-              "Resource": "*",
-            },
           ],
           "Version": "2012-10-17",
         },
@@ -135,9 +158,6 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
       },
       "Type": "AWS::KMS::Alias",
     },
-    "sourcetopic7C3DC892": {
-      "Type": "AWS::SNS::Topic",
-    },
     "worker28EA3E30": {
       "DependsOn": [
         "workerServiceRoleDefaultPolicyBA498553",
@@ -156,6 +176,9 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
         "Description": "Updated at 1212-12-12T12:12:12.121Z",
         "Environment": {
           "Variables": {
+            "DESTINATION_SNS_TOPIC_ARN": {
+              "Ref": "destinationtopicDCE2E0B8",
+            },
             "ENVIRONMENT": "dev",
             "NODE_ENV": "production",
             "NODE_OPTIONS": "--enable-source-maps",
@@ -171,6 +194,20 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
             "Arn",
           ],
         },
+        "Layers": [
+          {
+            "Fn::Join": [
+              "",
+              [
+                "arn:aws:lambda:",
+                {
+                  "Ref": "AWS::Region",
+                },
+                ":464622532012:layer:Datadog-Extension-ARM:58",
+              ],
+            ],
+          },
+        ],
         "ReservedConcurrentExecutions": 2,
         "Role": {
           "Fn::GetAtt": [
@@ -611,59 +648,6 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
       "Type": "AWS::SQS::Queue",
       "UpdateReplacePolicy": "Delete",
     },
-    "workerqueuePolicy97054CB4": {
-      "Properties": {
-        "PolicyDocument": {
-          "Statement": [
-            {
-              "Action": "sqs:SendMessage",
-              "Condition": {
-                "ArnEquals": {
-                  "aws:SourceArn": {
-                    "Ref": "sourcetopic7C3DC892",
-                  },
-                },
-              },
-              "Effect": "Allow",
-              "Principal": {
-                "Service": "sns.amazonaws.com",
-              },
-              "Resource": {
-                "Fn::GetAtt": [
-                  "workerqueueA05CE5C6",
-                  "Arn",
-                ],
-              },
-            },
-          ],
-          "Version": "2012-10-17",
-        },
-        "Queues": [
-          {
-            "Ref": "workerqueueA05CE5C6",
-          },
-        ],
-      },
-      "Type": "AWS::SQS::QueuePolicy",
-    },
-    "workerqueueappStacksourcetopic613C6BDBD2F224F5": {
-      "DependsOn": [
-        "workerqueuePolicy97054CB4",
-      ],
-      "Properties": {
-        "Endpoint": {
-          "Fn::GetAtt": [
-            "workerqueueA05CE5C6",
-            "Arn",
-          ],
-        },
-        "Protocol": "sqs",
-        "TopicArn": {
-          "Ref": "sourcetopic7C3DC892",
-        },
-      },
-      "Type": "AWS::SNS::Subscription",
-    },
     "workerqueuedeadletters83F3505C": {
       "DeletionPolicy": "Delete",
       "Properties": {
@@ -719,6 +703,40 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
     },
   },
   "Resources": {
+    "datadogapikeysecret046FEF06": {
+      "DeletionPolicy": "Delete",
+      "Properties": {
+        "GenerateSecretString": {},
+      },
+      "Type": "AWS::SecretsManager::Secret",
+      "UpdateReplacePolicy": "Delete",
+    },
+    "destinationtopicDCE2E0B8": {
+      "Properties": {
+        "KmsMasterKeyId": {
+          "Fn::Join": [
+            "",
+            [
+              "arn:",
+              {
+                "Ref": "AWS::Partition",
+              },
+              ":kms:",
+              {
+                "Ref": "AWS::Region",
+              },
+              ":",
+              {
+                "Ref": "AWS::AccountId",
+              },
+              ":alias/aws/sns",
+            ],
+          ],
+        },
+        "TopicName": "serviceName",
+      },
+      "Type": "AWS::SNS::Topic",
+    },
     "kmskey49FBC3B3": {
       "DeletionPolicy": "Retain",
       "Properties": {
@@ -814,17 +832,6 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
               },
               "Resource": "*",
             },
-            {
-              "Action": [
-                "kms:Decrypt",
-                "kms:GenerateDataKey",
-              ],
-              "Effect": "Allow",
-              "Principal": {
-                "Service": "sns.amazonaws.com",
-              },
-              "Resource": "*",
-            },
           ],
           "Version": "2012-10-17",
         },
@@ -844,9 +851,6 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
       },
       "Type": "AWS::KMS::Alias",
     },
-    "sourcetopic7C3DC892": {
-      "Type": "AWS::SNS::Topic",
-    },
     "worker28EA3E30": {
       "DependsOn": [
         "workerServiceRoleDefaultPolicyBA498553",
@@ -865,6 +869,9 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
         "Description": "Updated at 1212-12-12T12:12:12.121Z",
         "Environment": {
           "Variables": {
+            "DESTINATION_SNS_TOPIC_ARN": {
+              "Ref": "destinationtopicDCE2E0B8",
+            },
             "ENVIRONMENT": "prod",
             "NODE_ENV": "production",
             "NODE_OPTIONS": "--enable-source-maps",
@@ -880,6 +887,20 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
             "Arn",
           ],
         },
+        "Layers": [
+          {
+            "Fn::Join": [
+              "",
+              [
+                "arn:aws:lambda:",
+                {
+                  "Ref": "AWS::Region",
+                },
+                ":464622532012:layer:Datadog-Extension-ARM:58",
+              ],
+            ],
+          },
+        ],
         "ReservedConcurrentExecutions": 20,
         "Role": {
           "Fn::GetAtt": [
@@ -1320,59 +1341,6 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
       "Type": "AWS::SQS::Queue",
       "UpdateReplacePolicy": "Delete",
     },
-    "workerqueuePolicy97054CB4": {
-      "Properties": {
-        "PolicyDocument": {
-          "Statement": [
-            {
-              "Action": "sqs:SendMessage",
-              "Condition": {
-                "ArnEquals": {
-                  "aws:SourceArn": {
-                    "Ref": "sourcetopic7C3DC892",
-                  },
-                },
-              },
-              "Effect": "Allow",
-              "Principal": {
-                "Service": "sns.amazonaws.com",
-              },
-              "Resource": {
-                "Fn::GetAtt": [
-                  "workerqueueA05CE5C6",
-                  "Arn",
-                ],
-              },
-            },
-          ],
-          "Version": "2012-10-17",
-        },
-        "Queues": [
-          {
-            "Ref": "workerqueueA05CE5C6",
-          },
-        ],
-      },
-      "Type": "AWS::SQS::QueuePolicy",
-    },
-    "workerqueueappStacksourcetopic613C6BDBD2F224F5": {
-      "DependsOn": [
-        "workerqueuePolicy97054CB4",
-      ],
-      "Properties": {
-        "Endpoint": {
-          "Fn::GetAtt": [
-            "workerqueueA05CE5C6",
-            "Arn",
-          ],
-        },
-        "Protocol": "sqs",
-        "TopicArn": {
-          "Ref": "sourcetopic7C3DC892",
-        },
-      },
-      "Type": "AWS::SNS::Subscription",
-    },
     "workerqueuedeadletters83F3505C": {
       "DeletionPolicy": "Delete",
       "Properties": {

package/template/lambda-sqs-worker-cdk/infra/appStack.test.ts

@@ -1,4 +1,4 @@
-import { App, aws_sns } from 'aws-cdk-lib';
+import { App, aws_secretsmanager, aws_sns } from 'aws-cdk-lib';
 import { Template } from 'aws-cdk-lib/assertions';
 
 const currentDate = '1212-12-12T12:12:12.121Z';
@@ -36,6 +36,12 @@ it.each(['dev', 'prod'])(
       .spyOn(aws_sns.Topic, 'fromTopicArn')
       .mockImplementation((scope, id) => new aws_sns.Topic(scope, id));
 
+    jest
+      .spyOn(aws_secretsmanager.Secret, 'fromSecretPartialArn')
+      .mockImplementation(
+        (scope, id) => new aws_secretsmanager.Secret(scope, id),
+      );
+
     const app = new App();
 
     const stack = new AppStack(app, 'appStack');
@@ -47,13 +53,18 @@ it.each(['dev', 'prod'])(
         /"S3Key":"([0-9a-f]+)\.zip"/g,
         (_, hash) => `"S3Key":"${'x'.repeat(hash.length)}.zip"`,
       )
-      .replaceAll(
+      .replace(
         /workerCurrentVersion([0-9a-zA-Z]+)"/g,
         (_, hash) => `workerCurrentVersion${'x'.repeat(hash.length)}"`,
       )
       .replaceAll(
         /"Value":"\d+\.\d+\.\d+-([^"]+)"/g,
         (_, hash) => `"Value": "x.x.x-${'x'.repeat(hash.length)}"`,
+      )
+      .replace(
+        /"DD_TAGS":"git.commit.sha:([0-9a-f]+),git.repository_url:([^\"]+)"/g,
+        (_, sha, url) =>
+          `"DD_TAGS":"git.commit.sha:${'x'.repeat(sha.length)},git.repository_url:${'x'.repeat(url.length)}"`,
       );
     expect(JSON.parse(json)).toMatchSnapshot();
   },

package/template/lambda-sqs-worker-cdk/infra/appStack.ts

@@ -8,11 +8,12 @@ import {
   aws_lambda,
   aws_lambda_event_sources,
   aws_lambda_nodejs,
+  aws_secretsmanager,
   aws_sns,
-  aws_sns_subscriptions,
   aws_sqs,
 } from 'aws-cdk-lib';
 import type { Construct } from 'constructs';
+import { Datadog, getExtensionLayerArn } from 'datadog-cdk-constructs-v2';
 
 import { config } from './config';
 
@@ -49,13 +50,42 @@ export class AppStack extends Stack {
       encryptionMasterKey: kmsKey,
     });
 
-    const topic = aws_sns.Topic.fromTopicArn(
+    // const topic = aws_sns.Topic.fromTopicArn(
+    //   this,
+    //   'source-topic',
+    //   config.sourceSnsTopicArn,
+    // );
+
+    // topic.addSubscription(
+    //   new aws_sns_subscriptions.SqsSubscription(queue, {
+    //     rawMessageDelivery: true, // Remove this property if you require end to end datadog tracing
+    //   }),
+    // );
+
+    const snsKey = aws_kms.Alias.fromAliasName(
       this,
-      'source-topic',
-      config.sourceSnsTopicArn,
+      'alias-aws-sns',
+      'alias/aws/sns',
     );
 
-    topic.addSubscription(new aws_sns_subscriptions.SqsSubscription(queue));
+    const destinationTopic = new aws_sns.Topic(this, 'destination-topic', {
+      masterKey: snsKey,
+      topicName: '<%- serviceName %>',
+    });
+
+    const datadogSecret = aws_secretsmanager.Secret.fromSecretPartialArn(
+      this,
+      'datadog-api-key-secret',
+      config.datadogApiKeySecretArn,
+    );
+
+    const datadog = new Datadog(this, 'datadog', {
+      apiKeySecret: datadogSecret,
+      addLayers: false,
+      enableDatadogLogs: false,
+      flushMetricsToLogs: false,
+      extensionLayerVersion: 58,
+    });
 
     const architecture = '<%- lambdaCdkArchitecture %>';
 
@@ -85,17 +115,33 @@ export class AppStack extends Stack {
       ...defaultWorkerConfig,
       entry: './src/app.ts',
       timeout: Duration.seconds(30),
-      bundling: defaultWorkerBundlingConfig,
+      bundling: {
+        ...defaultWorkerBundlingConfig,
+        nodeModules: ['datadog-lambda-js', 'dd-trace'],
+      },
       functionName: '<%- serviceName %>',
       environment: {
         ...defaultWorkerEnvironment,
         ...config.workerLambda.environment,
+        DESTINATION_SNS_TOPIC_ARN: destinationTopic.topicArn,
       },
       // https://github.com/aws/aws-cdk/issues/28237
       // This forces the lambda to be updated on every deployment
       // If you do not wish to use hotswap, you can remove the new Date().toISOString() from the description
       description: `Updated at ${new Date().toISOString()}`,
       reservedConcurrentExecutions: config.workerLambda.reservedConcurrency,
+      layers: [
+        // Workaround for https://github.com/DataDog/datadog-cdk-constructs/issues/201
+        aws_lambda.LayerVersion.fromLayerVersionArn(
+          this,
+          'datadog-layer',
+          getExtensionLayerArn(
+            this.region,
+            datadog.props.extensionLayerVersion as number,
+            defaultWorkerConfig.architecture === aws_lambda.Architecture.ARM_64,
+          ),
+        ),
+      ],
     });
 
     const workerDeployment = new LambdaDeployment(this, 'workerDeployment', {

package/template/lambda-sqs-worker-cdk/infra/config.ts

@@ -16,6 +16,7 @@ interface Config {
       VERSION: string;
     };
   };
+  datadogApiKeySecretArn: string;
   sourceSnsTopicArn: string;
 }
 
@@ -30,6 +31,7 @@ const configs: Record<Environment, Config> = {
         VERSION: Env.string('VERSION', { default: 'local' }),
       },
     },
+    datadogApiKeySecretArn: 'TODO: datadogApiKeySecretArn',
     sourceSnsTopicArn: 'TODO: sourceSnsTopicArn',
   },
   prod: {
@@ -42,6 +44,7 @@ const configs: Record<Environment, Config> = {
         VERSION: Env.string('VERSION', { default: 'local' }),
       },
     },
+    datadogApiKeySecretArn: 'TODO: datadogApiKeySecretArn',
     sourceSnsTopicArn: 'TODO: sourceSnsTopicArn',
   },
 };

package/template/lambda-sqs-worker-cdk/package.json

@@ -17,18 +17,25 @@
     "@aws-sdk/client-lambda": "^3.363.0",
     "@aws-sdk/client-sns": "^3.363.0",
     "@seek/logger": "^9.0.0",
+    "datadog-lambda-js": "^8.0.0",
+    "dd-trace": "^5.0.0",
     "skuba-dive": "^2.0.0",
     "zod": "^3.19.1"
   },
   "devDependencies": {
     "@seek/aws-codedeploy-infra": "^2.1.0",
     "@types/aws-lambda": "^8.10.82",
+    "@types/chance": "^1.1.3",
     "@types/node": "^20.16.5",
     "aws-cdk": "^2.109.0",
     "aws-cdk-lib": "^2.109.0",
+    "aws-sdk-client-mock": "^4.0.0",
+    "aws-sdk-client-mock-jest": "^4.0.0",
+    "chance": "^1.1.8",
     "constructs": "^10.0.17",
+    "datadog-cdk-constructs-v2": "^1.13.0",
     "pino-pretty": "^11.0.0",
-    "skuba": "*"
+    "skuba": "9.0.1-upgrade-cdk-template-20241002233314"
   },
   "packageManager": "pnpm@9.11.0",
   "engines": {