skuba 9.0.0-renovate-eslint-9.x-20240923103202 → 9.0.1-upgrade-cdk-template-20241002233314

package/template/lambda-sqs-worker-cdk/infra/__snapshots__/appStack.test.ts.snap

@@ -10,6 +10,40 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
     },
   },
   "Resources": {
+    "datadogapikeysecret046FEF06": {
+      "DeletionPolicy": "Delete",
+      "Properties": {
+        "GenerateSecretString": {},
+      },
+      "Type": "AWS::SecretsManager::Secret",
+      "UpdateReplacePolicy": "Delete",
+    },
+    "destinationtopicDCE2E0B8": {
+      "Properties": {
+        "KmsMasterKeyId": {
+          "Fn::Join": [
+            "",
+            [
+              "arn:",
+              {
+                "Ref": "AWS::Partition",
+              },
+              ":kms:",
+              {
+                "Ref": "AWS::Region",
+              },
+              ":",
+              {
+                "Ref": "AWS::AccountId",
+              },
+              ":alias/aws/sns",
+            ],
+          ],
+        },
+        "TopicName": "serviceName",
+      },
+      "Type": "AWS::SNS::Topic",
+    },
     "kmskey49FBC3B3": {
       "DeletionPolicy": "Retain",
       "Properties": {
@@ -105,17 +139,6 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
               },
               "Resource": "*",
             },
-            {
-              "Action": [
-                "kms:Decrypt",
-                "kms:GenerateDataKey",
-              ],
-              "Effect": "Allow",
-              "Principal": {
-                "Service": "sns.amazonaws.com",
-              },
-              "Resource": "*",
-            },
           ],
           "Version": "2012-10-17",
         },
@@ -135,9 +158,6 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
       },
       "Type": "AWS::KMS::Alias",
     },
-    "sourcetopic7C3DC892": {
-      "Type": "AWS::SNS::Topic",
-    },
     "worker28EA3E30": {
       "DependsOn": [
         "workerServiceRoleDefaultPolicyBA498553",
@@ -156,6 +176,9 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
         "Description": "Updated at 1212-12-12T12:12:12.121Z",
         "Environment": {
           "Variables": {
+            "DESTINATION_SNS_TOPIC_ARN": {
+              "Ref": "destinationtopicDCE2E0B8",
+            },
             "ENVIRONMENT": "dev",
             "NODE_ENV": "production",
             "NODE_OPTIONS": "--enable-source-maps",
@@ -171,6 +194,20 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
             "Arn",
           ],
         },
+        "Layers": [
+          {
+            "Fn::Join": [
+              "",
+              [
+                "arn:aws:lambda:",
+                {
+                  "Ref": "AWS::Region",
+                },
+                ":464622532012:layer:Datadog-Extension-ARM:58",
+              ],
+            ],
+          },
+        ],
         "ReservedConcurrentExecutions": 2,
         "Role": {
           "Fn::GetAtt": [
@@ -611,59 +648,6 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
       "Type": "AWS::SQS::Queue",
       "UpdateReplacePolicy": "Delete",
     },
-    "workerqueuePolicy97054CB4": {
-      "Properties": {
-        "PolicyDocument": {
-          "Statement": [
-            {
-              "Action": "sqs:SendMessage",
-              "Condition": {
-                "ArnEquals": {
-                  "aws:SourceArn": {
-                    "Ref": "sourcetopic7C3DC892",
-                  },
-                },
-              },
-              "Effect": "Allow",
-              "Principal": {
-                "Service": "sns.amazonaws.com",
-              },
-              "Resource": {
-                "Fn::GetAtt": [
-                  "workerqueueA05CE5C6",
-                  "Arn",
-                ],
-              },
-            },
-          ],
-          "Version": "2012-10-17",
-        },
-        "Queues": [
-          {
-            "Ref": "workerqueueA05CE5C6",
-          },
-        ],
-      },
-      "Type": "AWS::SQS::QueuePolicy",
-    },
-    "workerqueueappStacksourcetopic613C6BDBD2F224F5": {
-      "DependsOn": [
-        "workerqueuePolicy97054CB4",
-      ],
-      "Properties": {
-        "Endpoint": {
-          "Fn::GetAtt": [
-            "workerqueueA05CE5C6",
-            "Arn",
-          ],
-        },
-        "Protocol": "sqs",
-        "TopicArn": {
-          "Ref": "sourcetopic7C3DC892",
-        },
-      },
-      "Type": "AWS::SNS::Subscription",
-    },
     "workerqueuedeadletters83F3505C": {
       "DeletionPolicy": "Delete",
       "Properties": {
@@ -719,6 +703,40 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
     },
   },
   "Resources": {
+    "datadogapikeysecret046FEF06": {
+      "DeletionPolicy": "Delete",
+      "Properties": {
+        "GenerateSecretString": {},
+      },
+      "Type": "AWS::SecretsManager::Secret",
+      "UpdateReplacePolicy": "Delete",
+    },
+    "destinationtopicDCE2E0B8": {
+      "Properties": {
+        "KmsMasterKeyId": {
+          "Fn::Join": [
+            "",
+            [
+              "arn:",
+              {
+                "Ref": "AWS::Partition",
+              },
+              ":kms:",
+              {
+                "Ref": "AWS::Region",
+              },
+              ":",
+              {
+                "Ref": "AWS::AccountId",
+              },
+              ":alias/aws/sns",
+            ],
+          ],
+        },
+        "TopicName": "serviceName",
+      },
+      "Type": "AWS::SNS::Topic",
+    },
     "kmskey49FBC3B3": {
       "DeletionPolicy": "Retain",
       "Properties": {
@@ -814,17 +832,6 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
               },
               "Resource": "*",
             },
-            {
-              "Action": [
-                "kms:Decrypt",
-                "kms:GenerateDataKey",
-              ],
-              "Effect": "Allow",
-              "Principal": {
-                "Service": "sns.amazonaws.com",
-              },
-              "Resource": "*",
-            },
           ],
           "Version": "2012-10-17",
         },
@@ -844,9 +851,6 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
       },
       "Type": "AWS::KMS::Alias",
     },
-    "sourcetopic7C3DC892": {
-      "Type": "AWS::SNS::Topic",
-    },
     "worker28EA3E30": {
       "DependsOn": [
         "workerServiceRoleDefaultPolicyBA498553",
@@ -865,6 +869,9 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
         "Description": "Updated at 1212-12-12T12:12:12.121Z",
         "Environment": {
           "Variables": {
+            "DESTINATION_SNS_TOPIC_ARN": {
+              "Ref": "destinationtopicDCE2E0B8",
+            },
             "ENVIRONMENT": "prod",
             "NODE_ENV": "production",
             "NODE_OPTIONS": "--enable-source-maps",
@@ -880,6 +887,20 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
             "Arn",
           ],
         },
+        "Layers": [
+          {
+            "Fn::Join": [
+              "",
+              [
+                "arn:aws:lambda:",
+                {
+                  "Ref": "AWS::Region",
+                },
+                ":464622532012:layer:Datadog-Extension-ARM:58",
+              ],
+            ],
+          },
+        ],
         "ReservedConcurrentExecutions": 20,
         "Role": {
           "Fn::GetAtt": [
@@ -1320,59 +1341,6 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
       "Type": "AWS::SQS::Queue",
       "UpdateReplacePolicy": "Delete",
     },
-    "workerqueuePolicy97054CB4": {
-      "Properties": {
-        "PolicyDocument": {
-          "Statement": [
-            {
-              "Action": "sqs:SendMessage",
-              "Condition": {
-                "ArnEquals": {
-                  "aws:SourceArn": {
-                    "Ref": "sourcetopic7C3DC892",
-                  },
-                },
-              },
-              "Effect": "Allow",
-              "Principal": {
-                "Service": "sns.amazonaws.com",
-              },
-              "Resource": {
-                "Fn::GetAtt": [
-                  "workerqueueA05CE5C6",
-                  "Arn",
-                ],
-              },
-            },
-          ],
-          "Version": "2012-10-17",
-        },
-        "Queues": [
-          {
-            "Ref": "workerqueueA05CE5C6",
-          },
-        ],
-      },
-      "Type": "AWS::SQS::QueuePolicy",
-    },
-    "workerqueueappStacksourcetopic613C6BDBD2F224F5": {
-      "DependsOn": [
-        "workerqueuePolicy97054CB4",
-      ],
-      "Properties": {
-        "Endpoint": {
-          "Fn::GetAtt": [
-            "workerqueueA05CE5C6",
-            "Arn",
-          ],
-        },
-        "Protocol": "sqs",
-        "TopicArn": {
-          "Ref": "sourcetopic7C3DC892",
-        },
-      },
-      "Type": "AWS::SNS::Subscription",
-    },
     "workerqueuedeadletters83F3505C": {
       "DeletionPolicy": "Delete",
       "Properties": {

package/template/lambda-sqs-worker-cdk/infra/appStack.test.ts

@@ -1,4 +1,4 @@
-import { App, aws_sns } from 'aws-cdk-lib';
+import { App, aws_secretsmanager, aws_sns } from 'aws-cdk-lib';
 import { Template } from 'aws-cdk-lib/assertions';
 
 const currentDate = '1212-12-12T12:12:12.121Z';
@@ -36,6 +36,12 @@ it.each(['dev', 'prod'])(
       .spyOn(aws_sns.Topic, 'fromTopicArn')
       .mockImplementation((scope, id) => new aws_sns.Topic(scope, id));
 
+    jest
+      .spyOn(aws_secretsmanager.Secret, 'fromSecretPartialArn')
+      .mockImplementation(
+        (scope, id) => new aws_secretsmanager.Secret(scope, id),
+      );
+
     const app = new App();
 
     const stack = new AppStack(app, 'appStack');
@@ -47,13 +53,18 @@ it.each(['dev', 'prod'])(
         /"S3Key":"([0-9a-f]+)\.zip"/g,
         (_, hash) => `"S3Key":"${'x'.repeat(hash.length)}.zip"`,
       )
-      .replaceAll(
+      .replace(
         /workerCurrentVersion([0-9a-zA-Z]+)"/g,
         (_, hash) => `workerCurrentVersion${'x'.repeat(hash.length)}"`,
       )
       .replaceAll(
         /"Value":"\d+\.\d+\.\d+-([^"]+)"/g,
         (_, hash) => `"Value": "x.x.x-${'x'.repeat(hash.length)}"`,
+      )
+      .replace(
+        /"DD_TAGS":"git.commit.sha:([0-9a-f]+),git.repository_url:([^\"]+)"/g,
+        (_, sha, url) =>
+          `"DD_TAGS":"git.commit.sha:${'x'.repeat(sha.length)},git.repository_url:${'x'.repeat(url.length)}"`,
       );
     expect(JSON.parse(json)).toMatchSnapshot();
   },

package/template/lambda-sqs-worker-cdk/infra/appStack.ts

@@ -8,11 +8,12 @@ import {
   aws_lambda,
   aws_lambda_event_sources,
   aws_lambda_nodejs,
+  aws_secretsmanager,
   aws_sns,
-  aws_sns_subscriptions,
   aws_sqs,
 } from 'aws-cdk-lib';
 import type { Construct } from 'constructs';
+import { Datadog, getExtensionLayerArn } from 'datadog-cdk-constructs-v2';
 
 import { config } from './config';
 
@@ -49,13 +50,42 @@ export class AppStack extends Stack {
       encryptionMasterKey: kmsKey,
     });
 
-    const topic = aws_sns.Topic.fromTopicArn(
+    // const topic = aws_sns.Topic.fromTopicArn(
+    //   this,
+    //   'source-topic',
+    //   config.sourceSnsTopicArn,
+    // );
+
+    // topic.addSubscription(
+    //   new aws_sns_subscriptions.SqsSubscription(queue, {
+    //     rawMessageDelivery: true, // Remove this property if you require end to end datadog tracing
+    //   }),
+    // );
+
+    const snsKey = aws_kms.Alias.fromAliasName(
       this,
-      'source-topic',
-      config.sourceSnsTopicArn,
+      'alias-aws-sns',
+      'alias/aws/sns',
     );
 
-    topic.addSubscription(new aws_sns_subscriptions.SqsSubscription(queue));
+    const destinationTopic = new aws_sns.Topic(this, 'destination-topic', {
+      masterKey: snsKey,
+      topicName: '<%- serviceName %>',
+    });
+
+    const datadogSecret = aws_secretsmanager.Secret.fromSecretPartialArn(
+      this,
+      'datadog-api-key-secret',
+      config.datadogApiKeySecretArn,
+    );
+
+    const datadog = new Datadog(this, 'datadog', {
+      apiKeySecret: datadogSecret,
+      addLayers: false,
+      enableDatadogLogs: false,
+      flushMetricsToLogs: false,
+      extensionLayerVersion: 58,
+    });
 
     const architecture = '<%- lambdaCdkArchitecture %>';
 
@@ -85,17 +115,33 @@ export class AppStack extends Stack {
       ...defaultWorkerConfig,
       entry: './src/app.ts',
       timeout: Duration.seconds(30),
-      bundling: defaultWorkerBundlingConfig,
+      bundling: {
+        ...defaultWorkerBundlingConfig,
+        nodeModules: ['datadog-lambda-js', 'dd-trace'],
+      },
       functionName: '<%- serviceName %>',
       environment: {
         ...defaultWorkerEnvironment,
         ...config.workerLambda.environment,
+        DESTINATION_SNS_TOPIC_ARN: destinationTopic.topicArn,
       },
       // https://github.com/aws/aws-cdk/issues/28237
       // This forces the lambda to be updated on every deployment
       // If you do not wish to use hotswap, you can remove the new Date().toISOString() from the description
       description: `Updated at ${new Date().toISOString()}`,
       reservedConcurrentExecutions: config.workerLambda.reservedConcurrency,
+      layers: [
+        // Workaround for https://github.com/DataDog/datadog-cdk-constructs/issues/201
+        aws_lambda.LayerVersion.fromLayerVersionArn(
+          this,
+          'datadog-layer',
+          getExtensionLayerArn(
+            this.region,
+            datadog.props.extensionLayerVersion as number,
+            defaultWorkerConfig.architecture === aws_lambda.Architecture.ARM_64,
+          ),
+        ),
+      ],
     });
 
     const workerDeployment = new LambdaDeployment(this, 'workerDeployment', {

package/template/lambda-sqs-worker-cdk/infra/config.ts

@@ -16,6 +16,7 @@ interface Config {
       VERSION: string;
     };
   };
+  datadogApiKeySecretArn: string;
   sourceSnsTopicArn: string;
 }
 
@@ -30,6 +31,7 @@ const configs: Record<Environment, Config> = {
         VERSION: Env.string('VERSION', { default: 'local' }),
       },
     },
+    datadogApiKeySecretArn: 'TODO: datadogApiKeySecretArn',
     sourceSnsTopicArn: 'TODO: sourceSnsTopicArn',
   },
   prod: {
@@ -42,6 +44,7 @@ const configs: Record<Environment, Config> = {
         VERSION: Env.string('VERSION', { default: 'local' }),
       },
     },
+    datadogApiKeySecretArn: 'TODO: datadogApiKeySecretArn',
     sourceSnsTopicArn: 'TODO: sourceSnsTopicArn',
   },
 };

package/template/lambda-sqs-worker-cdk/package.json

@@ -16,19 +16,26 @@
     "@aws-sdk/client-codedeploy": "^3.363.0",
     "@aws-sdk/client-lambda": "^3.363.0",
     "@aws-sdk/client-sns": "^3.363.0",
-    "@seek/logger": "^6.0.0",
+    "@seek/logger": "^9.0.0",
+    "datadog-lambda-js": "^8.0.0",
+    "dd-trace": "^5.0.0",
     "skuba-dive": "^2.0.0",
     "zod": "^3.19.1"
   },
   "devDependencies": {
     "@seek/aws-codedeploy-infra": "^2.1.0",
     "@types/aws-lambda": "^8.10.82",
+    "@types/chance": "^1.1.3",
     "@types/node": "^20.16.5",
     "aws-cdk": "^2.109.0",
     "aws-cdk-lib": "^2.109.0",
+    "aws-sdk-client-mock": "^4.0.0",
+    "aws-sdk-client-mock-jest": "^4.0.0",
+    "chance": "^1.1.8",
     "constructs": "^10.0.17",
+    "datadog-cdk-constructs-v2": "^1.13.0",
     "pino-pretty": "^11.0.0",
-    "skuba": "9.0.0-renovate-eslint-9.x-20240923103202"
+    "skuba": "9.0.1-upgrade-cdk-template-20241002233314"
   },
   "packageManager": "pnpm@9.11.0",
   "engines": {

package/template/lambda-sqs-worker-cdk/src/app.test.ts

@@ -0,0 +1,116 @@
+import { PublishCommand } from '@aws-sdk/client-sns';
+
+import { metricsClient } from 'src/framework/metrics';
+import { createCtx, createSqsEvent } from 'src/testing/handler';
+import { logger } from 'src/testing/logging';
+import { scoringService, sns } from 'src/testing/services';
+import { chance, mockJobPublishedEvent } from 'src/testing/types';
+
+import * as app from './app';
+
+describe('app', () => {
+  it('exports a handler', () => expect(app).toHaveProperty('handler'));
+});
+
+describe('handler', () => {
+  const ctx = createCtx();
+
+  const jobPublished = mockJobPublishedEvent({ entityId: chance.name() });
+
+  const score = chance.floating({ max: 1, min: 0 });
+
+  const distribution = jest
+    .spyOn(metricsClient, 'distribution')
+    .mockReturnValue();
+
+  beforeAll(logger.spy);
+  beforeAll(scoringService.spy);
+
+  beforeEach(() => {
+    scoringService.request.mockResolvedValue(score);
+    sns.publish.resolves({ MessageId: chance.guid({ version: 4 }) });
+  });
+
+  afterEach(() => {
+    logger.clear();
+    distribution.mockClear();
+    scoringService.clear();
+    sns.clear();
+  });
+
+  it('handles one record', async () => {
+    const event = createSqsEvent([JSON.stringify(jobPublished)]);
+
+    await expect(app.handler(event, ctx)).resolves.toBeUndefined();
+
+    expect(scoringService.request).toHaveBeenCalledTimes(1);
+
+    expect(logger.error).not.toHaveBeenCalled();
+
+    expect(logger.debug.mock.calls).toEqual([
+      [{ count: 1 }, 'Received jobs'],
+      [{ snsMessageId: expect.any(String) }, 'Scored job'],
+      ['Function succeeded'],
+    ]);
+
+    expect(distribution.mock.calls).toEqual([
+      ['job.received', 1],
+      ['job.scored', 1],
+    ]);
+
+    expect(sns.client).toReceiveCommandTimes(PublishCommand, 1);
+  });
+
+  it('throws on invalid input', () => {
+    const event = createSqsEvent(['}']);
+
+    return expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
+  });
+
+  it('bubbles up scoring service error', async () => {
+    const err = Error(chance.sentence());
+
+    scoringService.request.mockRejectedValue(err);
+
+    const event = createSqsEvent([JSON.stringify(jobPublished)]);
+
+    await expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
+
+    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
+  });
+
+  it('bubbles up SNS error', async () => {
+    const err = Error(chance.sentence());
+
+    sns.publish.rejects(err);
+
+    const event = createSqsEvent([JSON.stringify(jobPublished)]);
+
+    await expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
+
+    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
+  });
+
+  it('throws on zero records', async () => {
+    const err = new Error('Received 0 records');
+
+    const event = createSqsEvent([]);
+
+    await expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
+
+    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
+  });
+
+  it('throws on multiple records', async () => {
+    const err = new Error('Received 2 records');
+
+    const event = createSqsEvent([
+      JSON.stringify(jobPublished),
+      JSON.stringify(jobPublished),
+    ]);
+
+    await expect(app.handler(event, ctx)).rejects.toThrow('Function failed');
+
+    expect(logger.error).toHaveBeenCalledWith({ err }, 'Function failed');
+  });
+});

package/template/lambda-sqs-worker-cdk/src/app.ts

@@ -1,35 +1,57 @@
-import createLogger from '@seek/logger';
-import type { SQSEvent, SQSHandler } from 'aws-lambda';
+import 'skuba-dive/register';
 
-import { config } from './config';
+import type { SQSEvent } from 'aws-lambda';
 
-export const logger = createLogger({
-  base: {
-    environment: config.environment,
-    version: config.version,
-  },
-
-  level: config.logLevel,
-
-  name: config.name,
-
-  transport:
-    config.environment === 'local' ? { target: 'pino-pretty' } : undefined,
-});
+import { createHandler } from 'src/framework/handler';
+import { logger } from 'src/framework/logging';
+import { metricsClient } from 'src/framework/metrics';
+import { validateJson } from 'src/framework/validation';
+import { scoreJobPublishedEvent, scoringService } from 'src/services/jobScorer';
+import { sendPipelineEvent } from 'src/services/pipelineEventSender';
+import { JobPublishedEventSchema } from 'src/types/pipelineEvents';
 
 /**
  * Tests connectivity to ensure appropriate access and network configuration.
  */
-const smokeTest = async () => Promise.resolve();
+const smokeTest = async () => {
+  await Promise.all([scoringService.smokeTest(), sendPipelineEvent({}, true)]);
+};
 
-export const handler: SQSHandler = (event: SQSEvent) => {
+export const handler = createHandler<SQSEvent>(async (event) => {
   // Treat an empty object as our smoke test event.
   if (!Object.keys(event).length) {
     logger.debug('Received smoke test request');
     return smokeTest();
   }
 
-  logger.info('Hello World!');
+  const count = event.Records.length;
 
-  return;
-};
+  if (count !== 1) {
+    throw Error(`Received ${count} records`);
+  }
+
+  logger.debug({ count }, 'Received jobs');
+
+  metricsClient.distribution('job.received', event.Records.length);
+
+  const record = event.Records[0];
+  if (!record) {
+    throw new Error('Malformed SQS event with no records');
+  }
+
+  const { body } = record;
+
+  // TODO: this throws an error, which will cause the Lambda function to retry
+  // the event and eventually send it to your dead-letter queue. If you don't
+  // trust your source to provide consistently well-formed input, consider
+  // catching and handling this error in code.
+  const publishedJob = validateJson(body, JobPublishedEventSchema);
+
+  const scoredJob = await scoreJobPublishedEvent(publishedJob);
+
+  const snsMessageId = await sendPipelineEvent(scoredJob);
+
+  logger.debug({ snsMessageId }, 'Scored job');
+
+  metricsClient.distribution('job.scored', 1);
+});