quidproquo-deploy-awscdk 0.0.96 → 0.0.97

package/lib/constants/logging.d.ts

@@ -1,2 +1 @@
 export declare const QPQ_LOG_BUCKET_NAME = "logs";
-export declare const QPQ_FROM_LOG_BUCKET_NAME = "flogs";

package/lib/constants/logging.js

@@ -1,5 +1,4 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.QPQ_FROM_LOG_BUCKET_NAME = exports.QPQ_LOG_BUCKET_NAME = void 0;
+exports.QPQ_LOG_BUCKET_NAME = void 0;
 exports.QPQ_LOG_BUCKET_NAME = 'logs';
-exports.QPQ_FROM_LOG_BUCKET_NAME = 'flogs';

package/lib/constructs/basic/Function.js

@@ -86,7 +86,7 @@ class Function extends QpqConstructBlock_1.QpqConstructBlock {
             //     'aws:ResourceTag/Name': '*qpqlog',
             //   },
             // },
-            resources: ['arn:aws:dynamodb:*:*:table/logs-*', 'arn:aws:dynamodb:*:*:table/flogs-*'],
+            resources: ['arn:aws:dynamodb:*:*:table/logs-*'],
         }));
     }
 }

package/lib/constructs/basic/LogStorage.d.ts

@@ -7,14 +7,16 @@ export interface LogStorageProps extends QpqConstructBlockProps {
 }
 export declare abstract class LogStorageConstructBase extends QpqConstructBlock implements QpqResource {
     abstract bucket: aws_s3.IBucket;
-    abstract table: aws_dynamodb.ITable;
+    abstract storyResultsTable: aws_dynamodb.ITable;
+    abstract fromStoryResultsTable: aws_dynamodb.ITable;
     grantRead(grantee: aws_iam.IGrantable): aws_iam.Grant;
     grantWrite(grantee: aws_iam.IGrantable): aws_iam.Grant;
     grantAll(grantee: aws_iam.IGrantable): void;
 }
 export declare class LogStorage extends QpqConstructBlock {
     readonly bucket: aws_s3.IBucket;
-    readonly table: aws_dynamodb.ITable;
+    readonly storyResultsTable: aws_dynamodb.ITable;
+    readonly fromStoryResultsTable: aws_dynamodb.ITable;
     static fromOtherStack(scope: Construct, id: string, qpqConfig: QPQConfig, awsAccountId: string, serviceBucketName: string): QpqResource;
     constructor(scope: Construct, id: string, props: LogStorageProps);
 }

package/lib/constructs/basic/LogStorage.js

@@ -35,11 +35,13 @@ const constants_1 = require("../../constants");
 const Function_1 = require("./Function");
 class LogStorageConstructBase extends QpqConstructBlock_1.QpqConstructBlock {
     grantRead(grantee) {
-        this.table.grantReadData(grantee);
+        this.storyResultsTable.grantReadData(grantee);
+        this.fromStoryResultsTable.grantReadData(grantee);
         return this.bucket.grantRead(grantee);
     }
     grantWrite(grantee) {
-        this.table.grantWriteData(grantee);
+        this.storyResultsTable.grantWriteData(grantee);
+        this.fromStoryResultsTable.grantWriteData(grantee);
         return this.bucket.grantWrite(grantee);
     }
     grantAll(grantee) {
@@ -54,7 +56,8 @@ class LogStorage extends QpqConstructBlock_1.QpqConstructBlock {
             constructor() {
                 super(...arguments);
                 this.bucket = aws_cdk_lib_1.aws_s3.Bucket.fromBucketName(scope, `${id}-log-storage`, this.qpqResourceName(serviceBucketName, 'log'));
-                this.table = aws_cdk_lib_1.aws_dynamodb.Table.fromTableName(scope, `${id}-table`, this.qpqResourceName(serviceBucketName, 'log'));
+                this.storyResultsTable = aws_cdk_lib_1.aws_dynamodb.Table.fromTableName(scope, `${id}-log-table`, this.qpqResourceName(serviceBucketName, 'log'));
+                this.fromStoryResultsTable = aws_cdk_lib_1.aws_dynamodb.Table.fromTableName(scope, `${id}-flog-table`, this.qpqResourceName(serviceBucketName, 'flog'));
             }
         }
         return new Import(scope, id, { qpqConfig, awsAccountId });
@@ -87,13 +90,14 @@ class LogStorage extends QpqConstructBlock_1.QpqConstructBlock {
         // Also, we cant have a GSI on keys that can be undefined, this table will only contain
         // logs with a fromCorrelation
         const fromStoryResultsTable = new aws_cdk_lib_1.aws_dynamodb.Table(this, 'from-table', {
-            tableName: this.qpqResourceName(constants_1.QPQ_FROM_LOG_BUCKET_NAME, 'flog'),
+            tableName: this.qpqResourceName(constants_1.QPQ_LOG_BUCKET_NAME, 'flog'),
             partitionKey: { name: 'fromCorrelation', type: aws_cdk_lib_1.aws_dynamodb.AttributeType.STRING },
             sortKey: { name: 'startedAtWithCorrelation', type: aws_cdk_lib_1.aws_dynamodb.AttributeType.STRING },
             billingMode: aws_cdk_lib_1.aws_dynamodb.BillingMode.PAY_PER_REQUEST,
             removalPolicy: cdk.RemovalPolicy.DESTROY,
         });
-        this.table = storyResultsTable;
+        this.storyResultsTable = storyResultsTable;
+        this.fromStoryResultsTable = fromStoryResultsTable;
         // Create a lambda that is trigged when files are written to the bucket
         const func = new Function_1.Function(this, 'function', {
             buildPath: path_1.default.join(__dirname, '../..'),

package/lib/lambdas/lambda-utils/getRuntimeCorrelation.js

@@ -5,6 +5,6 @@ const quidproquo_core_1 = require("quidproquo-core");
 const quidproquo_actionprocessor_awslambda_1 = require("quidproquo-actionprocessor-awslambda");
 const getRuntimeCorrelation = (qpqConfig) => {
     const moduleName = quidproquo_core_1.qpqCoreUtils.getApplicationModuleName(qpqConfig);
-    return `${moduleName}-${quidproquo_actionprocessor_awslambda_1.awsLambdaUtils.randomGuid()}`;
+    return `${moduleName}::${quidproquo_actionprocessor_awslambda_1.awsLambdaUtils.randomGuid()}`;
 };
 exports.getRuntimeCorrelation = getRuntimeCorrelation;

package/lib/lambdas/lambda-utils/logger.js

@@ -10,27 +10,19 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getLogger = exports.storyLogger = void 0;
+const client_s3_1 = require("@aws-sdk/client-s3");
+const quidproquo_actionprocessor_awslambda_1 = require("quidproquo-actionprocessor-awslambda");
+const quidproquo_core_1 = require("quidproquo-core");
+const constants_1 = require("../../constants");
 const storyLogger = (result, qpqConfig) => __awaiter(void 0, void 0, void 0, function* () {
-    /*
-  
-    const bucketName = awsNamingUtils.getQpqRuntimeResourceNameFromConfig(
-      QPQ_LOG_BUCKET_NAME,
-      qpqConfig,
-      'log',
-    );
-    const region = qpqCoreUtils.getApplicationModuleDeployRegion(qpqConfig);
-  
-    const s3Client = new S3Client({ region });
-  
-    await s3Client.send(
-      new PutObjectCommand({
+    const bucketName = quidproquo_actionprocessor_awslambda_1.awsNamingUtils.getQpqRuntimeResourceNameFromConfig(constants_1.QPQ_LOG_BUCKET_NAME, qpqConfig, 'log');
+    const region = quidproquo_core_1.qpqCoreUtils.getApplicationModuleDeployRegion(qpqConfig);
+    const s3Client = new client_s3_1.S3Client({ region });
+    yield s3Client.send(new client_s3_1.PutObjectCommand({
         Key: `${result.correlation}.json`,
         Bucket: bucketName,
         Body: JSON.stringify(result),
-      }),
-    );
-    
-    */
+    }));
 });
 exports.storyLogger = storyLogger;
 const getLogger = (qpqConfig) => (result) => __awaiter(void 0, void 0, void 0, function* () {

package/lib/lambdas/lambdaS3FileWriteEvent_log.js

@@ -38,6 +38,7 @@ const writeStoryResultMetadataToDynamo = (tableName, storyResultMetadata, region
                 S: `${storyResultMetadata.startedAt}#${storyResultMetadata.correlation}`,
             },
             runtimeType: { S: storyResultMetadata.runtimeType },
+            correlation: { S: storyResultMetadata.correlation },
             error: { S: storyResultMetadata.error || '' },
             generic: { S: storyResultMetadata.generic },
             moduleName: { S: storyResultMetadata.moduleName },
@@ -70,18 +71,27 @@ const getFromCorrelationBucketNameFromLog = (storyResultMetadata, localBucketNam
     return newBucketName;
 };
 exports.getFromCorrelationBucketNameFromLog = getFromCorrelationBucketNameFromLog;
+function getFromCorrelationDynamoTableNameFromBucketName(bucketName) {
+    const target = 'qpqlog';
+    const replacement = 'qpqflog';
+    if (bucketName.endsWith(target)) {
+        return bucketName.slice(0, bucketName.length - target.length) + replacement;
+    }
+    return bucketName;
+}
 const executeS3FileWriteEvent = (event) => __awaiter(void 0, void 0, void 0, function* () {
     for (const record of event.Records) {
         const bucketName = record.s3.bucket.name;
         const awsRegion = record.awsRegion;
-        const key = record.s3.object.key;
+        const key = decodeURIComponent(record.s3.object.key);
         const storyResultMetadata = yield readLogFromBucket(bucketName, awsRegion, key);
         // Write the main log to the service table
         yield writeStoryResultMetadataToDynamo(bucketName, storyResultMetadata, awsRegion);
         // Write the same log to the table of the from service, so it can query links on the fromGuid GSI
         const fromBucketName = (0, exports.getFromCorrelationBucketNameFromLog)(storyResultMetadata, bucketName);
-        if (fromBucketName) {
-            yield writeStoryResultMetadataToDynamo(fromBucketName, storyResultMetadata, awsRegion);
+        const targetFromCorrelationDynamoTableName = getFromCorrelationDynamoTableNameFromBucketName(fromBucketName);
+        if (targetFromCorrelationDynamoTableName) {
+            yield writeStoryResultMetadataToDynamo(targetFromCorrelationDynamoTableName, storyResultMetadata, awsRegion);
         }
     }
 });

package/lib/stacks/InfQpqServiceStack.js

@@ -9,10 +9,10 @@ class InfQpqServiceStack extends QpqServiceStack_1.QpqServiceStack {
     constructor(scope, id, props) {
         super(scope, id, props);
         // Build the log storage drive
-        // const logBucket = new LogStorage(this, 'logStorage', {
-        //   awsAccountId: props.awsAccountId,
-        //   qpqConfig: props.qpqConfig,
-        // }).bucket;
+        const logBucket = new constructs_1.LogStorage(this, 'logStorage', {
+            awsAccountId: props.awsAccountId,
+            qpqConfig: props.qpqConfig,
+        }).bucket;
         // Build the storage drives
         const storageDrives = quidproquo_core_1.qpqCoreUtils.getStorageDrives(props.qpqConfig).map((setting) => new constructs_1.QpqCoreStorageDriveConstruct(this, quidproquo_core_1.qpqCoreUtils.getUniqueKeyForSetting(setting), {
             awsAccountId: props.awsAccountId,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "quidproquo-deploy-awscdk",
-  "version": "0.0.96",
+  "version": "0.0.97",
   "description": "",
   "main": "./lib/index.js",
   "types": "./lib/index.d.js",

package/src/constants/logging.ts

@@ -1,2 +1 @@
 export const QPQ_LOG_BUCKET_NAME = 'logs';
-export const QPQ_FROM_LOG_BUCKET_NAME = 'flogs';

package/src/constructs/basic/Function.ts

@@ -109,7 +109,7 @@ export class Function extends QpqConstructBlock {
         //     'aws:ResourceTag/Name': '*qpqlog',
         //   },
         // },
-        resources: ['arn:aws:dynamodb:*:*:table/logs-*', 'arn:aws:dynamodb:*:*:table/flogs-*'],
+        resources: ['arn:aws:dynamodb:*:*:table/logs-*'],
       }),
     );
   }

package/src/constructs/basic/LogStorage.ts

@@ -6,7 +6,7 @@ import { aws_s3, aws_iam, aws_dynamodb, aws_s3_notifications } from 'aws-cdk-lib
 import * as cdk from 'aws-cdk-lib';
 import { QpqResource } from '../base';
 import { QPQConfig } from 'quidproquo-core/lib';
-import { QPQ_LOG_BUCKET_NAME, QPQ_FROM_LOG_BUCKET_NAME } from '../../constants';
+import { QPQ_LOG_BUCKET_NAME } from '../../constants';
 import { Function } from './Function';
 // import { qpqWebServerUtils } from '../../utils';
 
@@ -14,15 +14,18 @@ export interface LogStorageProps extends QpqConstructBlockProps {}
 
 export abstract class LogStorageConstructBase extends QpqConstructBlock implements QpqResource {
   abstract bucket: aws_s3.IBucket;
-  abstract table: aws_dynamodb.ITable;
+  abstract storyResultsTable: aws_dynamodb.ITable;
+  abstract fromStoryResultsTable: aws_dynamodb.ITable;
 
   public grantRead(grantee: aws_iam.IGrantable): aws_iam.Grant {
-    this.table.grantReadData(grantee);
+    this.storyResultsTable.grantReadData(grantee);
+    this.fromStoryResultsTable.grantReadData(grantee);
     return this.bucket.grantRead(grantee);
   }
 
   public grantWrite(grantee: aws_iam.IGrantable): aws_iam.Grant {
-    this.table.grantWriteData(grantee);
+    this.storyResultsTable.grantWriteData(grantee);
+    this.fromStoryResultsTable.grantWriteData(grantee);
     return this.bucket.grantWrite(grantee);
   }
 
@@ -34,7 +37,8 @@ export abstract class LogStorageConstructBase extends QpqConstructBlock implemen
 
 export class LogStorage extends QpqConstructBlock {
   public readonly bucket: aws_s3.IBucket;
-  public readonly table: aws_dynamodb.ITable;
+  public readonly storyResultsTable: aws_dynamodb.ITable;
+  public readonly fromStoryResultsTable: aws_dynamodb.ITable;
 
   static fromOtherStack(
     scope: Construct,
@@ -50,11 +54,17 @@ export class LogStorage extends QpqConstructBlock {
         this.qpqResourceName(serviceBucketName, 'log'),
       );
 
-      table = aws_dynamodb.Table.fromTableName(
+      storyResultsTable = aws_dynamodb.Table.fromTableName(
         scope,
-        `${id}-table`,
+        `${id}-log-table`,
         this.qpqResourceName(serviceBucketName, 'log'),
       );
+
+      fromStoryResultsTable = aws_dynamodb.Table.fromTableName(
+        scope,
+        `${id}-flog-table`,
+        this.qpqResourceName(serviceBucketName, 'flog'),
+      );
     }
 
     return new Import(scope, id, { qpqConfig, awsAccountId });
@@ -94,14 +104,15 @@ export class LogStorage extends QpqConstructBlock {
     // Also, we cant have a GSI on keys that can be undefined, this table will only contain
     // logs with a fromCorrelation
     const fromStoryResultsTable = new aws_dynamodb.Table(this, 'from-table', {
-      tableName: this.qpqResourceName(QPQ_FROM_LOG_BUCKET_NAME, 'flog'),
+      tableName: this.qpqResourceName(QPQ_LOG_BUCKET_NAME, 'flog'),
       partitionKey: { name: 'fromCorrelation', type: aws_dynamodb.AttributeType.STRING },
       sortKey: { name: 'startedAtWithCorrelation', type: aws_dynamodb.AttributeType.STRING },
       billingMode: aws_dynamodb.BillingMode.PAY_PER_REQUEST,
       removalPolicy: cdk.RemovalPolicy.DESTROY,
     });
 
-    this.table = storyResultsTable;
+    this.storyResultsTable = storyResultsTable;
+    this.fromStoryResultsTable = fromStoryResultsTable;
 
     // Create a lambda that is trigged when files are written to the bucket
     const func = new Function(this, 'function', {

package/src/lambdas/lambda-utils/getRuntimeCorrelation.ts

@@ -3,5 +3,5 @@ import { awsLambdaUtils } from 'quidproquo-actionprocessor-awslambda';
 
 export const getRuntimeCorrelation = (qpqConfig: QPQConfig): string => {
   const moduleName = qpqCoreUtils.getApplicationModuleName(qpqConfig);
-  return `${moduleName}-${awsLambdaUtils.randomGuid()}`;
+  return `${moduleName}::${awsLambdaUtils.randomGuid()}`;
 };

package/src/lambdas/lambda-utils/logger.ts

@@ -10,8 +10,6 @@ export const storyLogger = async (
   result: StoryResult<any>,
   qpqConfig: QPQConfig,
 ): Promise<void> => {
-  /*
-
   const bucketName = awsNamingUtils.getQpqRuntimeResourceNameFromConfig(
     QPQ_LOG_BUCKET_NAME,
     qpqConfig,
@@ -28,8 +26,6 @@ export const storyLogger = async (
       Body: JSON.stringify(result),
     }),
   );
-  
-  */
 };
 
 export const getLogger = (qpqConfig: QPQConfig) => async (result: StoryResult<any>) => {

package/src/lambdas/lambdaEventBridgeEvent.ts

@@ -17,7 +17,6 @@ import {
   getConfigGetParametersActionProcessor,
   getServiceFunctionActionProcessor,
   getAdminActionProcessor,
-  awsLambdaUtils,
   DynamicModuleLoader,
   LambdaRuntimeConfig,
 } from 'quidproquo-actionprocessor-awslambda';

package/src/lambdas/lambdaS3FileWriteEvent_log.ts

@@ -45,6 +45,7 @@ const writeStoryResultMetadataToDynamo = async (
         S: `${storyResultMetadata.startedAt}#${storyResultMetadata.correlation}`,
       },
       runtimeType: { S: storyResultMetadata.runtimeType },
+      correlation: { S: storyResultMetadata.correlation },
       error: { S: storyResultMetadata.error || '' },
       generic: { S: storyResultMetadata.generic },
       moduleName: { S: storyResultMetadata.moduleName },
@@ -87,11 +88,22 @@ export const getFromCorrelationBucketNameFromLog = (
   return newBucketName;
 };
 
+function getFromCorrelationDynamoTableNameFromBucketName(bucketName: string): string {
+  const target = 'qpqlog';
+  const replacement = 'qpqflog';
+
+  if (bucketName.endsWith(target)) {
+    return bucketName.slice(0, bucketName.length - target.length) + replacement;
+  }
+
+  return bucketName;
+}
+
 export const executeS3FileWriteEvent: S3Handler = async (event: S3Event) => {
   for (const record of event.Records) {
     const bucketName = record.s3.bucket.name;
     const awsRegion = record.awsRegion;
-    const key = record.s3.object.key;
+    const key = decodeURIComponent(record.s3.object.key);
 
     const storyResultMetadata = await readLogFromBucket(bucketName, awsRegion, key);
 
@@ -100,8 +112,16 @@ export const executeS3FileWriteEvent: S3Handler = async (event: S3Event) => {
 
     // Write the same log to the table of the from service, so it can query links on the fromGuid GSI
     const fromBucketName = getFromCorrelationBucketNameFromLog(storyResultMetadata, bucketName);
-    if (fromBucketName) {
-      await writeStoryResultMetadataToDynamo(fromBucketName, storyResultMetadata, awsRegion);
+
+    const targetFromCorrelationDynamoTableName =
+      getFromCorrelationDynamoTableNameFromBucketName(fromBucketName);
+
+    if (targetFromCorrelationDynamoTableName) {
+      await writeStoryResultMetadataToDynamo(
+        targetFromCorrelationDynamoTableName,
+        storyResultMetadata,
+        awsRegion,
+      );
     }
   }
 };

package/src/stacks/InfQpqServiceStack.ts

@@ -24,10 +24,10 @@ export class InfQpqServiceStack extends QpqServiceStack {
     super(scope, id, props);
 
     // Build the log storage drive
-    // const logBucket = new LogStorage(this, 'logStorage', {
-    //   awsAccountId: props.awsAccountId,
-    //   qpqConfig: props.qpqConfig,
-    // }).bucket;
+    const logBucket = new LogStorage(this, 'logStorage', {
+      awsAccountId: props.awsAccountId,
+      qpqConfig: props.qpqConfig,
+    }).bucket;
 
     // Build the storage drives
     const storageDrives = qpqCoreUtils.getStorageDrives(props.qpqConfig).map(