skuba 9.0.1 → 9.1.0-main-20241019031757

package/template/lambda-sqs-worker-cdk/README.md

@@ -0,0 +1,145 @@
+# <%- repoName %>
+
+[![Powered by skuba](https://img.shields.io/badge/🤿%20skuba-powered-009DC4)](https://github.com/seek-oss/skuba)
+
+Next steps:
+
+1. [ ] Finish templating if this was skipped earlier:
+
+   ```shell
+   pnpm exec skuba configure
+   ```
+
+2. [ ] Create a new repository in the appropriate GitHub organisation.
+3. [ ] Add the repository to BuildAgency;
+       see our internal [Buildkite Docs] for more information.
+4. [ ] Add Datadog extension, deployment bucket configuration and data classification tags to [infra/config.ts](infra/config.ts).
+5. [ ] Push local commits to the upstream GitHub branch.
+6. [ ] Configure [GitHub repository settings].
+7. [ ] Delete this checklist 😌.
+
+[Buildkite Docs]: https://backstage.myseek.xyz/docs/default/component/buildkite-docs
+[GitHub repository settings]: https://github.com/<%-orgName%>/<%-repoName%>/settings
+
+## Design
+
+<%-repoName %> is a Node.js [Lambda] application built in line with our [Technical Guidelines].
+It is backed by a typical SQS message + dead letter queue configuration and uses common SEEK packages.
+Workers enable fault-tolerant asynchronous processing of events.
+
+The `lambda-sqs-worker-cdk` template is modelled after a hypothetical enricher that scores job advertisements.
+It's stubbed out with in-memory [scoring service](src/services/jobScorer.ts).
+This would be replaced with internal logic or an external service in production.
+
+This project is deployed with [AWS CDK].
+The Lambda runtime provisions a single Node.js process per container.
+The supplied [infra/appStack.ts](infra/appStack.ts) starts out with a minimal `memorySize` which may require tuning based on workload.
+Under load, we autoscale horizontally in terms of container count up to `reservedConcurrency`.
+
+[@seek/aws-codedeploy-hooks] configures [CodeDeploy] for a blue-green deployment approach.
+A smoke test is run against the new version before traffic is switched over,
+providing an opportunity to test access and connectivity to online dependencies.
+This defaults to an invocation with an empty object `{}`.
+
+## Development
+
+### Test
+
+```shell
+# Run Jest tests locally
+pnpm test
+
+# Authenticate to dev account
+awsauth
+
+# Run smoke test against deployed application
+ENVIRONMENT=dev pnpm smoke
+```
+
+### Lint
+
+```shell
+# Fix issues
+pnpm format
+
+# Check for issues
+pnpm lint
+```
+
+### Start
+
+```shell
+# Start a local HTTP server
+pnpm start
+
+# Start with Node.js Inspector enabled
+pnpm start:debug
+```
+
+This serves the Lambda application over HTTP.
+For example, to invoke the handler with an empty object `{}` for smoke testing:
+
+```shell
+curl --data '[{}, {"awsRequestId": "local"}]' --include localhost:<%- port %>
+```
+
+### Deploy
+
+This project is deployed through a [Buildkite pipeline](.buildkite/pipeline.yml).
+
+- Commits to a feature branch can be deployed to the dev environment by unblocking a step in the Buildkite UI
+- Commits to the default branch are automatically deployed to the dev and prod environments in sequence
+
+To deploy locally:
+
+```shell
+# Authenticate to dev account
+awsauth
+
+ENVIRONMENT=dev pnpm run deploy
+```
+
+A hotswap deploy enables faster deployment but come with caveats such as requiring a Lambda to be rebuilt with every build.
+
+To deploy a [hotswap]:
+
+```shell
+# Authenticate to dev account
+awsauth
+
+ENVIRONMENT=dev pnpm run deploy:hotswap
+```
+
+To rapidly roll back a change,
+retry an individual deployment step from the previous build in Buildkite.
+Note that this will introduce drift between the head of the default Git branch and the live environment;
+use with caution and always follow up with a proper revert or fix in Git history.
+
+## Support
+
+### Dev
+
+TODO: add support links for the dev environment.
+
+<!--
+- CloudWatch dashboard
+- Datadog dashboard
+- Splunk logs
+-->
+
+### Prod
+
+TODO: add support links for the prod environment.
+
+<!--
+- CloudWatch dashboard
+- Datadog dashboard
+- Splunk logs
+-->
+
+[@seek/aws-codedeploy-hooks]: https://github.com/seek-oss/aws-codedeploy-hooks
+[AWS CDK]: https://docs.aws.amazon.com/cdk/v2/guide/home.html
+[CodeDeploy]: https://docs.aws.amazon.com/codedeploy
+[Hotswap]: https://docs.aws.amazon.com/cdk/v2/guide/ref-cli-cmd-deploy.html#ref-cli-cmd-deploy-options
+[Lambda]: https://docs.aws.amazon.com/lambda
+[Technical Guidelines]: https://myseek.atlassian.net/wiki/spaces/AA/pages/2358346017/

package/template/lambda-sqs-worker-cdk/infra/__snapshots__/appStack.test.ts.snap

@@ -10,6 +10,40 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
     },
   },
   "Resources": {
+    "datadogapikeysecret046FEF06": {
+      "DeletionPolicy": "Delete",
+      "Properties": {
+        "GenerateSecretString": {},
+      },
+      "Type": "AWS::SecretsManager::Secret",
+      "UpdateReplacePolicy": "Delete",
+    },
+    "destinationtopicDCE2E0B8": {
+      "Properties": {
+        "KmsMasterKeyId": {
+          "Fn::Join": [
+            "",
+            [
+              "arn:",
+              {
+                "Ref": "AWS::Partition",
+              },
+              ":kms:",
+              {
+                "Ref": "AWS::Region",
+              },
+              ":",
+              {
+                "Ref": "AWS::AccountId",
+              },
+              ":alias/aws/sns",
+            ],
+          ],
+        },
+        "TopicName": "serviceName",
+      },
+      "Type": "AWS::SNS::Topic",
+    },
     "kmskey49FBC3B3": {
       "DeletionPolicy": "Retain",
       "Properties": {
@@ -105,17 +139,6 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
               },
               "Resource": "*",
             },
-            {
-              "Action": [
-                "kms:Decrypt",
-                "kms:GenerateDataKey",
-              ],
-              "Effect": "Allow",
-              "Principal": {
-                "Service": "sns.amazonaws.com",
-              },
-              "Resource": "*",
-            },
           ],
           "Version": "2012-10-17",
         },
@@ -135,9 +158,6 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
       },
       "Type": "AWS::KMS::Alias",
     },
-    "sourcetopic7C3DC892": {
-      "Type": "AWS::SNS::Topic",
-    },
     "worker28EA3E30": {
       "DependsOn": [
         "workerServiceRoleDefaultPolicyBA498553",
@@ -156,6 +176,22 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
         "Description": "Updated at 1212-12-12T12:12:12.121Z",
         "Environment": {
           "Variables": {
+            "DD_API_KEY_SECRET_ARN": {
+              "Ref": "datadogapikeysecret046FEF06",
+            },
+            "DD_CAPTURE_LAMBDA_PAYLOAD": "false",
+            "DD_FLUSH_TO_LOG": "false",
+            "DD_LAMBDA_HANDLER": "index.handler",
+            "DD_LOGS_INJECTION": "false",
+            "DD_MERGE_XRAY_TRACES": "false",
+            "DD_SERVERLESS_APPSEC_ENABLED": "false",
+            "DD_SERVERLESS_LOGS_ENABLED": "false",
+            "DD_SITE": "datadoghq.com",
+            "DD_TAGS": "git.commit.sha:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx,git.repository_url:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+            "DD_TRACE_ENABLED": "true",
+            "DESTINATION_SNS_TOPIC_ARN": {
+              "Ref": "destinationtopicDCE2E0B8",
+            },
             "ENVIRONMENT": "dev",
             "NODE_ENV": "production",
             "NODE_OPTIONS": "--enable-source-maps",
@@ -164,13 +200,27 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
           },
         },
         "FunctionName": "serviceName",
-        "Handler": "index.handler",
+        "Handler": "node_modules/datadog-lambda-js/dist/handler.handler",
         "KmsKeyArn": {
           "Fn::GetAtt": [
             "kmskey49FBC3B3",
             "Arn",
           ],
         },
+        "Layers": [
+          {
+            "Fn::Join": [
+              "",
+              [
+                "arn:aws:lambda:",
+                {
+                  "Ref": "AWS::Region",
+                },
+                ":464622532012:layer:Datadog-Extension-ARM:x",
+              ],
+            ],
+          },
+        ],
         "ReservedConcurrentExecutions": 2,
         "Role": {
           "Fn::GetAtt": [
@@ -184,6 +234,10 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
             "Key": "aws-codedeploy-hooks",
             "Value": "x.x.x-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
           },
+          {
+            "Key": "dd_cdk_construct",
+            "Value": "vx.x.x",
+          },
         ],
         "Timeout": 30,
       },
@@ -550,6 +604,16 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
       "Properties": {
         "PolicyDocument": {
           "Statement": [
+            {
+              "Action": [
+                "secretsmanager:GetSecretValue",
+                "secretsmanager:DescribeSecret",
+              ],
+              "Effect": "Allow",
+              "Resource": {
+                "Ref": "datadogapikeysecret046FEF06",
+              },
+            },
             {
               "Action": [
                 "sqs:ReceiveMessage",
@@ -611,59 +675,6 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
       "Type": "AWS::SQS::Queue",
       "UpdateReplacePolicy": "Delete",
     },
-    "workerqueuePolicy97054CB4": {
-      "Properties": {
-        "PolicyDocument": {
-          "Statement": [
-            {
-              "Action": "sqs:SendMessage",
-              "Condition": {
-                "ArnEquals": {
-                  "aws:SourceArn": {
-                    "Ref": "sourcetopic7C3DC892",
-                  },
-                },
-              },
-              "Effect": "Allow",
-              "Principal": {
-                "Service": "sns.amazonaws.com",
-              },
-              "Resource": {
-                "Fn::GetAtt": [
-                  "workerqueueA05CE5C6",
-                  "Arn",
-                ],
-              },
-            },
-          ],
-          "Version": "2012-10-17",
-        },
-        "Queues": [
-          {
-            "Ref": "workerqueueA05CE5C6",
-          },
-        ],
-      },
-      "Type": "AWS::SQS::QueuePolicy",
-    },
-    "workerqueueappStacksourcetopic613C6BDBD2F224F5": {
-      "DependsOn": [
-        "workerqueuePolicy97054CB4",
-      ],
-      "Properties": {
-        "Endpoint": {
-          "Fn::GetAtt": [
-            "workerqueueA05CE5C6",
-            "Arn",
-          ],
-        },
-        "Protocol": "sqs",
-        "TopicArn": {
-          "Ref": "sourcetopic7C3DC892",
-        },
-      },
-      "Type": "AWS::SNS::Subscription",
-    },
     "workerqueuedeadletters83F3505C": {
       "DeletionPolicy": "Delete",
       "Properties": {
@@ -719,6 +730,40 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
     },
   },
   "Resources": {
+    "datadogapikeysecret046FEF06": {
+      "DeletionPolicy": "Delete",
+      "Properties": {
+        "GenerateSecretString": {},
+      },
+      "Type": "AWS::SecretsManager::Secret",
+      "UpdateReplacePolicy": "Delete",
+    },
+    "destinationtopicDCE2E0B8": {
+      "Properties": {
+        "KmsMasterKeyId": {
+          "Fn::Join": [
+            "",
+            [
+              "arn:",
+              {
+                "Ref": "AWS::Partition",
+              },
+              ":kms:",
+              {
+                "Ref": "AWS::Region",
+              },
+              ":",
+              {
+                "Ref": "AWS::AccountId",
+              },
+              ":alias/aws/sns",
+            ],
+          ],
+        },
+        "TopicName": "serviceName",
+      },
+      "Type": "AWS::SNS::Topic",
+    },
     "kmskey49FBC3B3": {
       "DeletionPolicy": "Retain",
       "Properties": {
@@ -814,17 +859,6 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
               },
               "Resource": "*",
             },
-            {
-              "Action": [
-                "kms:Decrypt",
-                "kms:GenerateDataKey",
-              ],
-              "Effect": "Allow",
-              "Principal": {
-                "Service": "sns.amazonaws.com",
-              },
-              "Resource": "*",
-            },
           ],
           "Version": "2012-10-17",
         },
@@ -844,9 +878,6 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
       },
       "Type": "AWS::KMS::Alias",
     },
-    "sourcetopic7C3DC892": {
-      "Type": "AWS::SNS::Topic",
-    },
     "worker28EA3E30": {
       "DependsOn": [
         "workerServiceRoleDefaultPolicyBA498553",
@@ -865,6 +896,22 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
         "Description": "Updated at 1212-12-12T12:12:12.121Z",
         "Environment": {
           "Variables": {
+            "DD_API_KEY_SECRET_ARN": {
+              "Ref": "datadogapikeysecret046FEF06",
+            },
+            "DD_CAPTURE_LAMBDA_PAYLOAD": "false",
+            "DD_FLUSH_TO_LOG": "false",
+            "DD_LAMBDA_HANDLER": "index.handler",
+            "DD_LOGS_INJECTION": "false",
+            "DD_MERGE_XRAY_TRACES": "false",
+            "DD_SERVERLESS_APPSEC_ENABLED": "false",
+            "DD_SERVERLESS_LOGS_ENABLED": "false",
+            "DD_SITE": "datadoghq.com",
+            "DD_TAGS": "git.commit.sha:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx,git.repository_url:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+            "DD_TRACE_ENABLED": "true",
+            "DESTINATION_SNS_TOPIC_ARN": {
+              "Ref": "destinationtopicDCE2E0B8",
+            },
             "ENVIRONMENT": "prod",
             "NODE_ENV": "production",
             "NODE_OPTIONS": "--enable-source-maps",
@@ -873,13 +920,27 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
           },
         },
         "FunctionName": "serviceName",
-        "Handler": "index.handler",
+        "Handler": "node_modules/datadog-lambda-js/dist/handler.handler",
         "KmsKeyArn": {
           "Fn::GetAtt": [
             "kmskey49FBC3B3",
             "Arn",
           ],
         },
+        "Layers": [
+          {
+            "Fn::Join": [
+              "",
+              [
+                "arn:aws:lambda:",
+                {
+                  "Ref": "AWS::Region",
+                },
+                ":464622532012:layer:Datadog-Extension-ARM:x",
+              ],
+            ],
+          },
+        ],
         "ReservedConcurrentExecutions": 20,
         "Role": {
           "Fn::GetAtt": [
@@ -893,6 +954,10 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
             "Key": "aws-codedeploy-hooks",
             "Value": "x.x.x-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
           },
+          {
+            "Key": "dd_cdk_construct",
+            "Value": "vx.x.x",
+          },
         ],
         "Timeout": 30,
       },
@@ -1259,6 +1324,16 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
       "Properties": {
         "PolicyDocument": {
           "Statement": [
+            {
+              "Action": [
+                "secretsmanager:GetSecretValue",
+                "secretsmanager:DescribeSecret",
+              ],
+              "Effect": "Allow",
+              "Resource": {
+                "Ref": "datadogapikeysecret046FEF06",
+              },
+            },
             {
               "Action": [
                 "sqs:ReceiveMessage",
@@ -1320,59 +1395,6 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
       "Type": "AWS::SQS::Queue",
       "UpdateReplacePolicy": "Delete",
     },
-    "workerqueuePolicy97054CB4": {
-      "Properties": {
-        "PolicyDocument": {
-          "Statement": [
-            {
-              "Action": "sqs:SendMessage",
-              "Condition": {
-                "ArnEquals": {
-                  "aws:SourceArn": {
-                    "Ref": "sourcetopic7C3DC892",
-                  },
-                },
-              },
-              "Effect": "Allow",
-              "Principal": {
-                "Service": "sns.amazonaws.com",
-              },
-              "Resource": {
-                "Fn::GetAtt": [
-                  "workerqueueA05CE5C6",
-                  "Arn",
-                ],
-              },
-            },
-          ],
-          "Version": "2012-10-17",
-        },
-        "Queues": [
-          {
-            "Ref": "workerqueueA05CE5C6",
-          },
-        ],
-      },
-      "Type": "AWS::SQS::QueuePolicy",
-    },
-    "workerqueueappStacksourcetopic613C6BDBD2F224F5": {
-      "DependsOn": [
-        "workerqueuePolicy97054CB4",
-      ],
-      "Properties": {
-        "Endpoint": {
-          "Fn::GetAtt": [
-            "workerqueueA05CE5C6",
-            "Arn",
-          ],
-        },
-        "Protocol": "sqs",
-        "TopicArn": {
-          "Ref": "sourcetopic7C3DC892",
-        },
-      },
-      "Type": "AWS::SNS::Subscription",
-    },
     "workerqueuedeadletters83F3505C": {
       "DeletionPolicy": "Delete",
       "Properties": {

package/template/lambda-sqs-worker-cdk/infra/appStack.test.ts

@@ -1,4 +1,4 @@
-import { App, aws_sns } from 'aws-cdk-lib';
+import { App, aws_secretsmanager, aws_sns } from 'aws-cdk-lib';
 import { Template } from 'aws-cdk-lib/assertions';
 
 const currentDate = '1212-12-12T12:12:12.121Z';
@@ -36,6 +36,12 @@ it.each(['dev', 'prod'])(
       .spyOn(aws_sns.Topic, 'fromTopicArn')
       .mockImplementation((scope, id) => new aws_sns.Topic(scope, id));
 
+    jest
+      .spyOn(aws_secretsmanager.Secret, 'fromSecretPartialArn')
+      .mockImplementation(
+        (scope, id) => new aws_secretsmanager.Secret(scope, id),
+      );
+
     const app = new App();
 
     const stack = new AppStack(app, 'appStack');
@@ -47,13 +53,23 @@ it.each(['dev', 'prod'])(
         /"S3Key":"([0-9a-f]+)\.zip"/g,
         (_, hash) => `"S3Key":"${'x'.repeat(hash.length)}.zip"`,
       )
-      .replaceAll(
+      .replace(
         /workerCurrentVersion([0-9a-zA-Z]+)"/g,
         (_, hash) => `workerCurrentVersion${'x'.repeat(hash.length)}"`,
       )
       .replaceAll(
         /"Value":"\d+\.\d+\.\d+-([^"]+)"/g,
         (_, hash) => `"Value": "x.x.x-${'x'.repeat(hash.length)}"`,
+      )
+      .replaceAll(/"Value":"v\d+\.\d+\.\d+"/g, (_) => `"Value": "vx.x.x"`)
+      .replace(
+        /"DD_TAGS":"git.commit.sha:([0-9a-f]+),git.repository_url:([^\"]+)"/g,
+        (_, sha, url) =>
+          `"DD_TAGS":"git.commit.sha:${'x'.repeat(sha.length)},git.repository_url:${'x'.repeat(url.length)}"`,
+      )
+      .replaceAll(
+        /(layer:Datadog-Extension-.+?:)\d+/g,
+        (_, layer) => `${layer}x`,
       );
     expect(JSON.parse(json)).toMatchSnapshot();
   },