quidproquo-deploy-awscdk 0.0.93 → 0.0.95

package/lib/constants/logging.d.ts

@@ -1 +1,2 @@
 export declare const QPQ_LOG_BUCKET_NAME = "logs";
+export declare const QPQ_FROM_LOG_BUCKET_NAME = "flogs";

package/lib/constants/logging.js

@@ -1,4 +1,5 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.QPQ_LOG_BUCKET_NAME = void 0;
+exports.QPQ_FROM_LOG_BUCKET_NAME = exports.QPQ_LOG_BUCKET_NAME = void 0;
 exports.QPQ_LOG_BUCKET_NAME = 'logs';
+exports.QPQ_FROM_LOG_BUCKET_NAME = 'flogs';

package/lib/constructs/basic/Function.js

@@ -71,6 +71,23 @@ class Function extends QpqConstructBlock_1.QpqConstructBlock {
             actions: ['lambda:InvokeFunction'],
             resources: ['arn:aws:lambda:*:*:function:*sfunc*'],
         }));
+        // Let lambdas write to dynamo logs
+        this.lambdaFunction.addToRolePolicy(new aws_cdk_lib_1.aws_iam.PolicyStatement({
+            actions: ['lambda:InvokeFunction'],
+            resources: ['arn:aws:lambda:*:*:function:*sfunc*'],
+        }));
+        // We need access to dynamo log tables
+        this.lambdaFunction.addToRolePolicy(new aws_cdk_lib_1.aws_iam.PolicyStatement({
+            actions: ['dynamodb:GetItem', 'dynamodb:Scan', 'dynamodb:Query', 'dynamodb:PutItem'],
+            // TODO: Revisit this, the user can make logs tables them selves... ~ Slight security risk here
+            // Consider making tags to identify logs tables, all qpq resourses
+            // conditions: {
+            //   'ForAllValues:StringLike': {
+            //     'aws:ResourceTag/Name': '*qpqlog',
+            //   },
+            // },
+            resources: ['arn:aws:dynamodb:*:*:table/logs-*', 'arn:aws:dynamodb:*:*:table/flogs-*'],
+        }));
     }
 }
 exports.Function = Function;

package/lib/constructs/basic/LogStorage.js

@@ -70,6 +70,7 @@ class LogStorage extends QpqConstructBlock_1.QpqConstructBlock {
             removalPolicy: cdk.RemovalPolicy.DESTROY,
             autoDeleteObjects: true,
         });
+        // Time based query over all logs for a given runtimeType
         const storyResultsTable = new aws_cdk_lib_1.aws_dynamodb.Table(this, 'table', {
             tableName: this.qpqResourceName(constants_1.QPQ_LOG_BUCKET_NAME, 'log'),
             partitionKey: { name: 'runtimeType', type: aws_cdk_lib_1.aws_dynamodb.AttributeType.STRING },
@@ -77,6 +78,21 @@ class LogStorage extends QpqConstructBlock_1.QpqConstructBlock {
             billingMode: aws_cdk_lib_1.aws_dynamodb.BillingMode.PAY_PER_REQUEST,
             removalPolicy: cdk.RemovalPolicy.DESTROY,
         });
+        // Query by correlation
+        storyResultsTable.addGlobalSecondaryIndex({
+            indexName: 'CorrelationIndex',
+            partitionKey: { name: 'correlation', type: aws_cdk_lib_1.aws_dynamodb.AttributeType.STRING },
+        });
+        // Second table for the from logs ~ Second table because a GSI will dupe the data more then needed
+        // Also, we cant have a GSI on keys that can be undefined, this table will only contain
+        // logs with a fromCorrelation
+        const fromStoryResultsTable = new aws_cdk_lib_1.aws_dynamodb.Table(this, 'from-table', {
+            tableName: this.qpqResourceName(constants_1.QPQ_FROM_LOG_BUCKET_NAME, 'flog'),
+            partitionKey: { name: 'fromCorrelation', type: aws_cdk_lib_1.aws_dynamodb.AttributeType.STRING },
+            sortKey: { name: 'startedAtWithCorrelation', type: aws_cdk_lib_1.aws_dynamodb.AttributeType.STRING },
+            billingMode: aws_cdk_lib_1.aws_dynamodb.BillingMode.PAY_PER_REQUEST,
+            removalPolicy: cdk.RemovalPolicy.DESTROY,
+        });
         this.table = storyResultsTable;
         // Create a lambda that is trigged when files are written to the bucket
         const func = new Function_1.Function(this, 'function', {

package/lib/lambdas/lambda-utils/getRuntimeCorrelation.d.ts

@@ -0,0 +1,2 @@
+import { QPQConfig } from 'quidproquo-core';
+export declare const getRuntimeCorrelation: (qpqConfig: QPQConfig) => string;

package/lib/lambdas/lambda-utils/getRuntimeCorrelation.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getRuntimeCorrelation = void 0;
+const quidproquo_core_1 = require("quidproquo-core");
+const quidproquo_actionprocessor_awslambda_1 = require("quidproquo-actionprocessor-awslambda");
+const getRuntimeCorrelation = (qpqConfig) => {
+    const moduleName = quidproquo_core_1.qpqCoreUtils.getApplicationModuleName(qpqConfig);
+    return `${moduleName}-${quidproquo_actionprocessor_awslambda_1.awsLambdaUtils.randomGuid()}`;
+};
+exports.getRuntimeCorrelation = getRuntimeCorrelation;

package/lib/lambdas/lambda-utils/index.d.ts

@@ -0,0 +1,2 @@
+export * from './getRuntimeCorrelation';
+export * from './logger';

package/lib/lambdas/lambda-utils/index.js

@@ -0,0 +1,18 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./getRuntimeCorrelation"), exports);
+__exportStar(require("./logger"), exports);

package/lib/lambdas/{logger → lambda-utils}/logger.js

@@ -10,19 +10,27 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getLogger = exports.storyLogger = void 0;
-const client_s3_1 = require("@aws-sdk/client-s3");
-const quidproquo_actionprocessor_awslambda_1 = require("quidproquo-actionprocessor-awslambda");
-const quidproquo_core_1 = require("quidproquo-core");
-const constants_1 = require("../../constants");
 const storyLogger = (result, qpqConfig) => __awaiter(void 0, void 0, void 0, function* () {
-    const bucketName = quidproquo_actionprocessor_awslambda_1.awsNamingUtils.getQpqRuntimeResourceNameFromConfig(constants_1.QPQ_LOG_BUCKET_NAME, qpqConfig, 'log');
-    const region = quidproquo_core_1.qpqCoreUtils.getApplicationModuleDeployRegion(qpqConfig);
-    const s3Client = new client_s3_1.S3Client({ region });
-    yield s3Client.send(new client_s3_1.PutObjectCommand({
+    /*
+  
+    const bucketName = awsNamingUtils.getQpqRuntimeResourceNameFromConfig(
+      QPQ_LOG_BUCKET_NAME,
+      qpqConfig,
+      'log',
+    );
+    const region = qpqCoreUtils.getApplicationModuleDeployRegion(qpqConfig);
+  
+    const s3Client = new S3Client({ region });
+  
+    await s3Client.send(
+      new PutObjectCommand({
         Key: `${result.correlation}.json`,
         Bucket: bucketName,
         Body: JSON.stringify(result),
-    }));
+      }),
+    );
+    
+    */
 });
 exports.storyLogger = storyLogger;
 const getLogger = (qpqConfig) => (result) => __awaiter(void 0, void 0, void 0, function* () {

package/lib/lambdas/lambdaAPIGatewayEvent.js

@@ -18,7 +18,7 @@ const quidproquo_actionprocessor_awslambda_1 = require("quidproquo-actionprocess
 const quidproquo_webserver_1 = require("quidproquo-webserver");
 const quidproquo_core_1 = require("quidproquo-core");
 const lambdaConfig_1 = require("./lambdaConfig");
-const logger_1 = require("./logger/logger");
+const lambda_utils_1 = require("./lambda-utils");
 const dynamicModuleLoader_1 = require("./dynamicModuleLoader");
 // @ts-ignore - Special webpack loader
 const qpq_custom_action_processors_loader_1 = __importDefault(require("qpq-custom-action-processors-loader!"));
@@ -42,10 +42,14 @@ const ErrorTypeHttpResponseMap = {
 const getAPIGatewayEventExecutor = (getCustomActionProcessors = () => ({})) => {
     return (event, context) => __awaiter(void 0, void 0, void 0, function* () {
         const cdkConfig = yield (0, lambdaConfig_1.getLambdaConfigs)();
+        const accessToken = quidproquo_webserver_1.qpqWebServerUtils.getAccessTokenFromHeaders(event.headers);
         // Build a processor for the session and stuff
         // Remove the non route ones ~ let the story execute action add them
         const storyActionProcessor = Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, quidproquo_actionprocessor_node_1.coreActionProcessor), quidproquo_actionprocessor_node_1.webserverActionProcessor), (0, quidproquo_actionprocessor_awslambda_1.getAPIGatewayEventActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetSecretActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetParameterActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetParametersActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getSystemActionProcessor)(cdkConfig.qpqConfig, dynamicModuleLoader_1.dynamicModuleLoader)), (0, quidproquo_actionprocessor_awslambda_1.getFileActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_node_1.getConfigActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getQueueActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getEventBusActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getUserDirectoryActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getWebEntryActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getServiceFunctionActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getAdminActionProcessor)(cdkConfig.qpqConfig)), getCustomActionProcessors(cdkConfig.qpqConfig)), (0, qpq_custom_action_processors_loader_1.default)());
-        const resolveStory = (0, quidproquo_core_1.createRuntime)(cdkConfig.qpqConfig, {}, storyActionProcessor, getDateNow, (0, logger_1.getLogger)(cdkConfig.qpqConfig), quidproquo_actionprocessor_awslambda_1.awsLambdaUtils.randomGuid, quidproquo_core_1.QpqRuntimeType.API);
+        const resolveStory = (0, quidproquo_core_1.createRuntime)(cdkConfig.qpqConfig, {
+            depth: 0,
+            accessToken: accessToken,
+        }, storyActionProcessor, getDateNow, (0, lambda_utils_1.getLogger)(cdkConfig.qpqConfig), (0, lambda_utils_1.getRuntimeCorrelation)(cdkConfig.qpqConfig), quidproquo_core_1.QpqRuntimeType.API);
         const result = yield resolveStory(quidproquo_core_1.askProcessEvent, [event, context]);
         // Run the callback
         if (!result.error) {

package/lib/lambdas/lambdaEventBridgeEvent.js

@@ -15,10 +15,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.executeEventBridgeEvent = exports.getEventBridgeEventExecutor = void 0;
 const quidproquo_actionprocessor_node_1 = require("quidproquo-actionprocessor-node");
 const quidproquo_actionprocessor_awslambda_1 = require("quidproquo-actionprocessor-awslambda");
-const quidproquo_actionprocessor_node_2 = require("quidproquo-actionprocessor-node");
 const quidproquo_core_1 = require("quidproquo-core");
 const lambdaConfig_1 = require("./lambdaConfig");
-const logger_1 = require("./logger/logger");
+const lambda_utils_1 = require("./lambda-utils");
 const dynamicModuleLoader_1 = require("./dynamicModuleLoader");
 // @ts-ignore - Special webpack loader
 const qpq_custom_action_processors_loader_1 = __importDefault(require("qpq-custom-action-processors-loader!"));
@@ -30,8 +29,10 @@ const getEventBridgeEventExecutor = (dynamicModuleLoader, getCustomActionProcess
         const lambdaRuntimeConfig = JSON.parse(process.env.lambdaRuntimeConfig || '{}');
         // Build a processor for the session and stuff
         // Remove the  non event ones
-        const storyActionProcessor = Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, quidproquo_actionprocessor_node_1.coreActionProcessor), quidproquo_actionprocessor_node_1.webserverActionProcessor), (0, quidproquo_actionprocessor_awslambda_1.getEventBridgeEventActionProcessor)(lambdaRuntimeConfig)), (0, quidproquo_actionprocessor_awslambda_1.getSystemActionProcessor)(cdkConfig.qpqConfig, dynamicModuleLoader)), (0, quidproquo_actionprocessor_awslambda_1.getFileActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetSecretActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetParameterActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetParametersActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_node_2.getConfigActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getQueueActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getEventBusActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getWebEntryActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getServiceFunctionActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getAdminActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getUserDirectoryActionProcessor)(cdkConfig.qpqConfig)), getCustomActionProcessors(cdkConfig.qpqConfig)), (0, qpq_custom_action_processors_loader_1.default)());
-        const resolveStory = (0, quidproquo_core_1.createRuntime)(cdkConfig.qpqConfig, {}, storyActionProcessor, getDateNow, (0, logger_1.getLogger)(cdkConfig.qpqConfig), quidproquo_actionprocessor_awslambda_1.awsLambdaUtils.randomGuid, quidproquo_core_1.QpqRuntimeType.EVENT_BRIDGE_EVENT);
+        const storyActionProcessor = Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, quidproquo_actionprocessor_node_1.coreActionProcessor), quidproquo_actionprocessor_node_1.webserverActionProcessor), (0, quidproquo_actionprocessor_awslambda_1.getEventBridgeEventActionProcessor)(lambdaRuntimeConfig)), (0, quidproquo_actionprocessor_awslambda_1.getSystemActionProcessor)(cdkConfig.qpqConfig, dynamicModuleLoader)), (0, quidproquo_actionprocessor_awslambda_1.getFileActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetSecretActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetParameterActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetParametersActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_node_1.getConfigActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getQueueActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getEventBusActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getWebEntryActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getServiceFunctionActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getAdminActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getUserDirectoryActionProcessor)(cdkConfig.qpqConfig)), getCustomActionProcessors(cdkConfig.qpqConfig)), (0, qpq_custom_action_processors_loader_1.default)());
+        const resolveStory = (0, quidproquo_core_1.createRuntime)(cdkConfig.qpqConfig, {
+            depth: 0,
+        }, storyActionProcessor, getDateNow, (0, lambda_utils_1.getLogger)(cdkConfig.qpqConfig), (0, lambda_utils_1.getRuntimeCorrelation)(cdkConfig.qpqConfig), quidproquo_core_1.QpqRuntimeType.RECURRING_SCHEDULE);
         yield resolveStory(quidproquo_core_1.askProcessEvent, [event, context]);
     });
 };

package/lib/lambdas/lambdaEventOriginRequest.js

@@ -15,10 +15,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.executeEventOriginRequest = exports.getOriginRequestEventExecutor = void 0;
 const quidproquo_actionprocessor_node_1 = require("quidproquo-actionprocessor-node");
 const quidproquo_actionprocessor_awslambda_1 = require("quidproquo-actionprocessor-awslambda");
-const quidproquo_actionprocessor_node_2 = require("quidproquo-actionprocessor-node");
 const lambdaConfig_1 = require("./lambdaConfig");
 const quidproquo_core_1 = require("quidproquo-core");
-const logger_1 = require("./logger/logger");
+const lambda_utils_1 = require("./lambda-utils");
 const dynamicModuleLoader_1 = require("./dynamicModuleLoader");
 // @ts-ignore - Special webpack loader
 const qpq_custom_action_processors_loader_1 = __importDefault(require("qpq-custom-action-processors-loader!"));
@@ -30,8 +29,10 @@ const getOriginRequestEventExecutor = (dynamicModuleLoader, getCustomActionProce
         const cdkConfig = yield (0, lambdaConfig_1.getLambdaConfigs)();
         // Build a processor for the session and stuff
         // Remove the non route ones ~ let the story execute action add them
-        const storyActionProcessor = Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, quidproquo_actionprocessor_node_1.coreActionProcessor), quidproquo_actionprocessor_node_1.webserverActionProcessor), (0, quidproquo_actionprocessor_awslambda_1.getCloudFrontOriginRequestEventActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetSecretActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetParameterActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetParametersActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getSystemActionProcessor)(cdkConfig.qpqConfig, dynamicModuleLoader)), (0, quidproquo_actionprocessor_awslambda_1.getFileActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_node_2.getConfigActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getWebEntryActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getServiceFunctionActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getAdminActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getUserDirectoryActionProcessor)(cdkConfig.qpqConfig)), getCustomActionProcessors(cdkConfig.qpqConfig)), (0, qpq_custom_action_processors_loader_1.default)());
-        const resolveStory = (0, quidproquo_core_1.createRuntime)(cdkConfig.qpqConfig, {}, storyActionProcessor, getDateNow, (0, logger_1.getLogger)(cdkConfig.qpqConfig), quidproquo_actionprocessor_awslambda_1.awsLambdaUtils.randomGuid, quidproquo_core_1.QpqRuntimeType.EVENT_SEO_OR);
+        const storyActionProcessor = Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, quidproquo_actionprocessor_node_1.coreActionProcessor), quidproquo_actionprocessor_node_1.webserverActionProcessor), (0, quidproquo_actionprocessor_awslambda_1.getCloudFrontOriginRequestEventActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetSecretActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetParameterActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetParametersActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getSystemActionProcessor)(cdkConfig.qpqConfig, dynamicModuleLoader)), (0, quidproquo_actionprocessor_awslambda_1.getFileActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_node_1.getConfigActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getWebEntryActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getServiceFunctionActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getAdminActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getUserDirectoryActionProcessor)(cdkConfig.qpqConfig)), getCustomActionProcessors(cdkConfig.qpqConfig)), (0, qpq_custom_action_processors_loader_1.default)());
+        const resolveStory = (0, quidproquo_core_1.createRuntime)(cdkConfig.qpqConfig, {
+            depth: 0,
+        }, storyActionProcessor, getDateNow, (0, lambda_utils_1.getLogger)(cdkConfig.qpqConfig), (0, lambda_utils_1.getRuntimeCorrelation)(cdkConfig.qpqConfig), quidproquo_core_1.QpqRuntimeType.EVENT_SEO_OR);
         const result = yield resolveStory(quidproquo_core_1.askProcessEvent, [event, context]);
         // Run the callback
         if (!result.error) {

package/lib/lambdas/lambdaS3FileWriteEvent_log.d.ts

@@ -1,2 +1,4 @@
 import { S3Handler } from 'aws-lambda';
+import { StoryResultMetadata } from 'quidproquo-core';
+export declare const getFromCorrelationBucketNameFromLog: (storyResultMetadata: StoryResultMetadata, localBucketName: string) => string;
 export declare const executeS3FileWriteEvent: S3Handler;

package/lib/lambdas/lambdaS3FileWriteEvent_log.js

@@ -9,7 +9,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.executeS3FileWriteEvent = void 0;
+exports.executeS3FileWriteEvent = exports.getFromCorrelationBucketNameFromLog = void 0;
 const client_s3_1 = require("@aws-sdk/client-s3");
 const client_dynamodb_1 = require("@aws-sdk/client-dynamodb");
 const storyResultToMetadata_1 = require("./storyResultToMetadata");
@@ -51,13 +51,38 @@ const writeStoryResultMetadataToDynamo = (tableName, storyResultMetadata, region
         console.error(`Failed to insert storyResultMetadata into DynamoDB: ${err}`);
     }
 });
+const getFromCorrelationBucketNameFromLog = (storyResultMetadata, localBucketName) => {
+    if (!storyResultMetadata.fromCorrelation) {
+        return '';
+    }
+    // Get the module name of the service that wants the log
+    const fromModuleName = storyResultMetadata.fromCorrelation.split('::')[0];
+    // Get the name of the current module where we are executing this code
+    const currentModuleName = storyResultMetadata.moduleName;
+    // Split up the bucket / module names into parts
+    const bucketParts = localBucketName.split('-');
+    const moduleParts = fromModuleName.split('-');
+    const currentModuleNameParts = currentModuleName.split('-');
+    // Replace the current module name with the module name of the service that wants the log
+    bucketParts.splice(2, currentModuleNameParts.length, ...moduleParts);
+    // rebuild the new bucket name
+    const newBucketName = bucketParts.join('-');
+    return newBucketName;
+};
+exports.getFromCorrelationBucketNameFromLog = getFromCorrelationBucketNameFromLog;
 const executeS3FileWriteEvent = (event) => __awaiter(void 0, void 0, void 0, function* () {
     for (const record of event.Records) {
         const bucketName = record.s3.bucket.name;
         const awsRegion = record.awsRegion;
         const key = record.s3.object.key;
         const storyResultMetadata = yield readLogFromBucket(bucketName, awsRegion, key);
+        // Write the main log to the service table
         yield writeStoryResultMetadataToDynamo(bucketName, storyResultMetadata, awsRegion);
+        // Write the same log to the table of the from service, so it can query links on the fromGuid GSI
+        const fromBucketName = (0, exports.getFromCorrelationBucketNameFromLog)(storyResultMetadata, bucketName);
+        if (fromBucketName) {
+            yield writeStoryResultMetadataToDynamo(fromBucketName, storyResultMetadata, awsRegion);
+        }
     }
 });
 exports.executeS3FileWriteEvent = executeS3FileWriteEvent;

package/lib/lambdas/lambdaSQSEvent.d.ts

@@ -1,6 +1,7 @@
 import { SQSEvent, SQSBatchResponse, Context } from 'aws-lambda';
 import { DynamicModuleLoader } from 'quidproquo-actionprocessor-awslambda';
+import { StorySession } from 'quidproquo-core';
 import { ActionProcessorListResolver } from './actionProcessorListResolver';
-export declare const getStoryActionRuntime: (dynamicModuleLoader: DynamicModuleLoader, getCustomActionProcessors?: ActionProcessorListResolver) => Promise<(<TArgs extends any[]>(story: (...args: TArgs) => Generator<any, any, import("quidproquo-core").Action<any>>, args: TArgs) => Promise<import("quidproquo-core").StoryResult<any, any>>)>;
+export declare const getStoryActionRuntime: (dynamicModuleLoader: DynamicModuleLoader, getCustomActionProcessors: ActionProcessorListResolver | undefined, callerStorySession: StorySession) => Promise<(<TArgs extends any[]>(story: (...args: TArgs) => Generator<any, any, import("quidproquo-core").Action<any>>, args: TArgs) => Promise<import("quidproquo-core").StoryResult<any, any>>)>;
 export declare const getSQSEventExecutor: (dynamicModuleLoader: DynamicModuleLoader, getCustomActionProcessors?: ActionProcessorListResolver) => (event: SQSEvent, context: Context) => Promise<SQSBatchResponse>;
 export declare const executeSQSEvent: (event: SQSEvent, context: Context) => Promise<SQSBatchResponse>;

package/lib/lambdas/lambdaSQSEvent.js

@@ -18,21 +18,22 @@ const quidproquo_actionprocessor_awslambda_1 = require("quidproquo-actionprocess
 const quidproquo_actionprocessor_node_2 = require("quidproquo-actionprocessor-node");
 const lambdaConfig_1 = require("./lambdaConfig");
 const quidproquo_core_1 = require("quidproquo-core");
-const logger_1 = require("./logger/logger");
+const lambda_utils_1 = require("./lambda-utils");
 const dynamicModuleLoader_1 = require("./dynamicModuleLoader");
 // @ts-ignore - Special webpack loader
 const qpq_custom_action_processors_loader_1 = __importDefault(require("qpq-custom-action-processors-loader!"));
 // TODO: Make this a util or something based on server time or something..
 const getDateNow = () => new Date().toISOString();
-const getStoryActionRuntime = (dynamicModuleLoader, getCustomActionProcessors = () => ({})) => __awaiter(void 0, void 0, void 0, function* () {
+const getStoryActionRuntime = (dynamicModuleLoader, getCustomActionProcessors = () => ({}), callerStorySession) => __awaiter(void 0, void 0, void 0, function* () {
     const cdkConfig = yield (0, lambdaConfig_1.getLambdaConfigs)();
     // Build a processor for the session and stuff
     // Remove the non route ones ~ let the story execute action add them
     const storyActionProcessor = Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, quidproquo_actionprocessor_node_1.coreActionProcessor), quidproquo_actionprocessor_node_1.webserverActionProcessor), (0, quidproquo_actionprocessor_awslambda_1.getSQSEventRecordActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetSecretActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetParameterActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetParametersActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getSystemActionProcessor)(cdkConfig.qpqConfig, dynamicModuleLoader)), (0, quidproquo_actionprocessor_awslambda_1.getFileActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_node_2.getConfigActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getQueueActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getEventBusActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getUserDirectoryActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getWebEntryActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getServiceFunctionActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getAdminActionProcessor)(cdkConfig.qpqConfig)), getCustomActionProcessors(cdkConfig.qpqConfig)), (0, qpq_custom_action_processors_loader_1.default)());
-    const resolveStory = (0, quidproquo_core_1.createRuntime)(cdkConfig.qpqConfig, {}, storyActionProcessor, getDateNow, (0, logger_1.getLogger)(cdkConfig.qpqConfig), quidproquo_actionprocessor_awslambda_1.awsLambdaUtils.randomGuid, quidproquo_core_1.QpqRuntimeType.QUEUE_EVENT);
+    const resolveStory = (0, quidproquo_core_1.createRuntime)(cdkConfig.qpqConfig, callerStorySession, storyActionProcessor, getDateNow, (0, lambda_utils_1.getLogger)(cdkConfig.qpqConfig), (0, lambda_utils_1.getRuntimeCorrelation)(cdkConfig.qpqConfig), quidproquo_core_1.QpqRuntimeType.QUEUE_EVENT);
     return resolveStory;
 });
 exports.getStoryActionRuntime = getStoryActionRuntime;
+// TODO: We could make this entire function a story, that spawns an execute story for each record
 const getSQSEventExecutor = (dynamicModuleLoader, getCustomActionProcessors = () => ({})) => {
     return (event, context) => __awaiter(void 0, void 0, void 0, function* () {
         // const queueQPQConfigSetting = getQueueConfigSetting();
@@ -40,10 +41,15 @@ const getSQSEventExecutor = (dynamicModuleLoader, getCustomActionProcessors = ()
         // in parallel
         console.log('num batch: ', event.Records.length);
         const results = event.Records.map((record) => __awaiter(void 0, void 0, void 0, function* () {
-            const resolveStory = yield (0, exports.getStoryActionRuntime)(dynamicModuleLoader, getCustomActionProcessors);
+            const runtimeData = JSON.parse(record.body);
+            const resolveStory = yield (0, exports.getStoryActionRuntime)(dynamicModuleLoader, getCustomActionProcessors, runtimeData.storySession);
+            const queueMessage = {
+                type: runtimeData.type,
+                payload: runtimeData.payload,
+            };
             return {
                 id: record.messageId,
-                result: yield resolveStory(quidproquo_core_1.askProcessEvent, [record, context]),
+                result: yield resolveStory(quidproquo_core_1.askProcessEvent, [queueMessage, context]),
             };
         }));
         const allResults = yield Promise.all(results);

package/lib/lambdas/lambdaServiceFunctionExecute.d.ts

@@ -1,6 +1,11 @@
 import { ExecuteServiceFunctionEvent } from 'quidproquo-webserver';
 import { DynamicModuleLoader } from 'quidproquo-actionprocessor-awslambda';
+import { StorySession } from 'quidproquo-core';
 import { Context } from 'aws-lambda';
 import { ActionProcessorListResolver } from './actionProcessorListResolver';
-export declare const getServiceFunctionExecuteEventExecutor: (dynamicModuleLoader: DynamicModuleLoader, getCustomActionProcessors?: ActionProcessorListResolver) => (event: ExecuteServiceFunctionEvent<any>, context: Context) => Promise<import("quidproquo-core").StoryResult<any, any>>;
-export declare const executeServiceFunctionExecuteEvent: (event: ExecuteServiceFunctionEvent<any>, context: Context) => Promise<import("quidproquo-core").StoryResult<any, any>>;
+type AnyExecuteServiceFunctionEventWithSession = ExecuteServiceFunctionEvent<any[]> & {
+    storySession: StorySession;
+};
+export declare const getServiceFunctionExecuteEventExecutor: (dynamicModuleLoader: DynamicModuleLoader, getCustomActionProcessors?: ActionProcessorListResolver) => (event: AnyExecuteServiceFunctionEventWithSession, context: Context) => Promise<import("quidproquo-core").StoryResult<any, any>>;
+export declare const executeServiceFunctionExecuteEvent: (event: AnyExecuteServiceFunctionEventWithSession, context: Context) => Promise<import("quidproquo-core").StoryResult<any, any>>;
+export {};

package/lib/lambdas/lambdaServiceFunctionExecute.js

@@ -15,10 +15,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.executeServiceFunctionExecuteEvent = exports.getServiceFunctionExecuteEventExecutor = void 0;
 const quidproquo_actionprocessor_node_1 = require("quidproquo-actionprocessor-node");
 const quidproquo_actionprocessor_awslambda_1 = require("quidproquo-actionprocessor-awslambda");
-const quidproquo_actionprocessor_node_2 = require("quidproquo-actionprocessor-node");
 const quidproquo_core_1 = require("quidproquo-core");
 const lambdaConfig_1 = require("./lambdaConfig");
-const logger_1 = require("./logger/logger");
+const lambda_utils_1 = require("./lambda-utils");
 // @ts-ignore - Special webpack loader
 const dynamicModuleLoader_1 = require("./dynamicModuleLoader");
 // @ts-ignore - Special webpack loader
@@ -30,9 +29,13 @@ const getServiceFunctionExecuteEventExecutor = (dynamicModuleLoader, getCustomAc
         const cdkConfig = yield (0, lambdaConfig_1.getLambdaConfigs)();
         // Build a processor for the session and stuff
         // Remove the non route ones ~ let the story execute action add them
-        const storyActionProcessor = Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, quidproquo_actionprocessor_node_1.coreActionProcessor), quidproquo_actionprocessor_node_1.webserverActionProcessor), (0, quidproquo_actionprocessor_awslambda_1.getServiceFunctionExecuteEventActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetSecretActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetParameterActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetParametersActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getSystemActionProcessor)(cdkConfig.qpqConfig, dynamicModuleLoader)), (0, quidproquo_actionprocessor_awslambda_1.getFileActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_node_2.getConfigActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getQueueActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getEventBusActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getUserDirectoryActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getWebEntryActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getServiceFunctionActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getAdminActionProcessor)(cdkConfig.qpqConfig)), getCustomActionProcessors(cdkConfig.qpqConfig)), (0, qpq_custom_action_processors_loader_1.default)());
-        const resolveStory = (0, quidproquo_core_1.createRuntime)(cdkConfig.qpqConfig, {}, storyActionProcessor, getDateNow, (0, logger_1.getLogger)(cdkConfig.qpqConfig), quidproquo_actionprocessor_awslambda_1.awsLambdaUtils.randomGuid, quidproquo_core_1.QpqRuntimeType.SERVICE_FUNCTION_EXE);
-        const result = yield resolveStory(quidproquo_core_1.askProcessEvent, [event, context]);
+        const storyActionProcessor = Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, quidproquo_actionprocessor_node_1.coreActionProcessor), quidproquo_actionprocessor_node_1.webserverActionProcessor), (0, quidproquo_actionprocessor_awslambda_1.getServiceFunctionExecuteEventActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetSecretActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetParameterActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getConfigGetParametersActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getSystemActionProcessor)(cdkConfig.qpqConfig, dynamicModuleLoader)), (0, quidproquo_actionprocessor_awslambda_1.getFileActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_node_1.getConfigActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getQueueActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getEventBusActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getUserDirectoryActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getWebEntryActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getServiceFunctionActionProcessor)(cdkConfig.qpqConfig)), (0, quidproquo_actionprocessor_awslambda_1.getAdminActionProcessor)(cdkConfig.qpqConfig)), getCustomActionProcessors(cdkConfig.qpqConfig)), (0, qpq_custom_action_processors_loader_1.default)());
+        const resolveStory = (0, quidproquo_core_1.createRuntime)(cdkConfig.qpqConfig, event.storySession, storyActionProcessor, getDateNow, (0, lambda_utils_1.getLogger)(cdkConfig.qpqConfig), (0, lambda_utils_1.getRuntimeCorrelation)(cdkConfig.qpqConfig), quidproquo_core_1.QpqRuntimeType.SERVICE_FUNCTION_EXE);
+        const eventWithNoSession = {
+            functionName: event.functionName,
+            payload: event.payload,
+        };
+        const result = yield resolveStory(quidproquo_core_1.askProcessEvent, [eventWithNoSession, context]);
         // just return the story result
         return result;
     });

package/lib/lambdas/storyResultToMetadata/genericTextExtractors/index.d.ts

@@ -1,2 +1,4 @@
 export * from './apiGenericTextExtractor';
+export * from './queueEventGenericTextExtractor';
+export * from './serviceFunctionExeGenericTextExtractor';
 export * from './unknownGenericTextExtractor';

package/lib/lambdas/storyResultToMetadata/genericTextExtractors/index.js

@@ -15,4 +15,6 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./apiGenericTextExtractor"), exports);
+__exportStar(require("./queueEventGenericTextExtractor"), exports);
+__exportStar(require("./serviceFunctionExeGenericTextExtractor"), exports);
 __exportStar(require("./unknownGenericTextExtractor"), exports);

package/lib/lambdas/storyResultToMetadata/genericTextExtractors/queueEventGenericTextExtractor.d.ts

@@ -0,0 +1,2 @@
+import { StoryResult } from 'quidproquo-core';
+export declare const queueEventGenericTextExtractor: (storyResult: StoryResult<any>) => string;

package/lib/lambdas/storyResultToMetadata/genericTextExtractors/queueEventGenericTextExtractor.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.queueEventGenericTextExtractor = void 0;
+const queueEventGenericTextExtractor = (storyResult) => {
+    var _a;
+    if (storyResult.runtimeType === 'QUEUE_EVENT') {
+        return ((_a = storyResult.input[0]) === null || _a === void 0 ? void 0 : _a.type) || '';
+    }
+    return '';
+};
+exports.queueEventGenericTextExtractor = queueEventGenericTextExtractor;

package/lib/lambdas/storyResultToMetadata/genericTextExtractors/serviceFunctionExeGenericTextExtractor.d.ts

@@ -0,0 +1,2 @@
+import { StoryResult } from 'quidproquo-core';
+export declare const serviceFunctionExeGenericTextExtractor: (storyResult: StoryResult<any>) => string;

package/lib/lambdas/storyResultToMetadata/genericTextExtractors/serviceFunctionExeGenericTextExtractor.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.serviceFunctionExeGenericTextExtractor = void 0;
+const serviceFunctionExeGenericTextExtractor = (storyResult) => {
+    var _a;
+    if (storyResult.runtimeType === 'SERVICE_FUNCTION_EXE') {
+        return ((_a = storyResult.input[0]) === null || _a === void 0 ? void 0 : _a.functionName) || '';
+    }
+    return '';
+};
+exports.serviceFunctionExeGenericTextExtractor = serviceFunctionExeGenericTextExtractor;

package/lib/lambdas/storyResultToMetadata/storyResultToMetadata.js

@@ -5,11 +5,11 @@ const genericTextExtractors_1 = require("./genericTextExtractors");
 // These have to be strings and not enum due to how the lambda is packaged
 const extractors = {
     ['API']: genericTextExtractors_1.apiGenericTextExtractor,
-    ['EVENT_BRIDGE_EVENT']: genericTextExtractors_1.unknownGenericTextExtractor,
+    ['QUEUE_EVENT']: genericTextExtractors_1.queueEventGenericTextExtractor,
+    ['SERVICE_FUNCTION_EXE']: genericTextExtractors_1.serviceFunctionExeGenericTextExtractor,
+    ['RECURRING_SCHEDULE']: genericTextExtractors_1.unknownGenericTextExtractor,
     ['EVENT_SEO_OR']: genericTextExtractors_1.unknownGenericTextExtractor,
     ['EXECUTE_STORY']: genericTextExtractors_1.unknownGenericTextExtractor,
-    ['QUEUE_EVENT']: genericTextExtractors_1.unknownGenericTextExtractor,
-    ['SERVICE_FUNCTION_EXE']: genericTextExtractors_1.unknownGenericTextExtractor,
 };
 const storyResultToMetadata = (storyResult, filepath) => {
     var _a;

package/lib/stacks/InfQpqServiceStack.js

@@ -9,10 +9,10 @@ class InfQpqServiceStack extends QpqServiceStack_1.QpqServiceStack {
     constructor(scope, id, props) {
         super(scope, id, props);
         // Build the log storage drive
-        const logBucket = new constructs_1.LogStorage(this, 'logStorage', {
-            awsAccountId: props.awsAccountId,
-            qpqConfig: props.qpqConfig,
-        }).bucket;
+        // const logBucket = new LogStorage(this, 'logStorage', {
+        //   awsAccountId: props.awsAccountId,
+        //   qpqConfig: props.qpqConfig,
+        // }).bucket;
         // Build the storage drives
         const storageDrives = quidproquo_core_1.qpqCoreUtils.getStorageDrives(props.qpqConfig).map((setting) => new constructs_1.QpqCoreStorageDriveConstruct(this, quidproquo_core_1.qpqCoreUtils.getUniqueKeyForSetting(setting), {
             awsAccountId: props.awsAccountId,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "quidproquo-deploy-awscdk",
-  "version": "0.0.93",
+  "version": "0.0.95",
   "description": "",
   "main": "./lib/index.js",
   "types": "./lib/index.d.js",

package/src/constants/logging.ts

@@ -1 +1,2 @@
 export const QPQ_LOG_BUCKET_NAME = 'logs';
+export const QPQ_FROM_LOG_BUCKET_NAME = 'flogs';

package/src/constructs/basic/Function.ts

@@ -88,5 +88,29 @@ export class Function extends QpqConstructBlock {
         resources: ['arn:aws:lambda:*:*:function:*sfunc*'],
       }),
     );
+
+    // Let lambdas write to dynamo logs
+    this.lambdaFunction.addToRolePolicy(
+      new aws_iam.PolicyStatement({
+        actions: ['lambda:InvokeFunction'],
+        resources: ['arn:aws:lambda:*:*:function:*sfunc*'],
+      }),
+    );
+
+    // We need access to dynamo log tables
+    this.lambdaFunction.addToRolePolicy(
+      new aws_iam.PolicyStatement({
+        actions: ['dynamodb:GetItem', 'dynamodb:Scan', 'dynamodb:Query', 'dynamodb:PutItem'],
+
+        // TODO: Revisit this, the user can make logs tables them selves... ~ Slight security risk here
+        // Consider making tags to identify logs tables, all qpq resourses
+        // conditions: {
+        //   'ForAllValues:StringLike': {
+        //     'aws:ResourceTag/Name': '*qpqlog',
+        //   },
+        // },
+        resources: ['arn:aws:dynamodb:*:*:table/logs-*', 'arn:aws:dynamodb:*:*:table/flogs-*'],
+      }),
+    );
   }
 }

package/src/constructs/basic/LogStorage.ts

@@ -6,7 +6,7 @@ import { aws_s3, aws_iam, aws_dynamodb, aws_s3_notifications } from 'aws-cdk-lib
 import * as cdk from 'aws-cdk-lib';
 import { QpqResource } from '../base';
 import { QPQConfig } from 'quidproquo-core/lib';
-import { QPQ_LOG_BUCKET_NAME } from '../../constants';
+import { QPQ_LOG_BUCKET_NAME, QPQ_FROM_LOG_BUCKET_NAME } from '../../constants';
 import { Function } from './Function';
 // import { qpqWebServerUtils } from '../../utils';
 
@@ -75,6 +75,7 @@ export class LogStorage extends QpqConstructBlock {
       autoDeleteObjects: true,
     });
 
+    // Time based query over all logs for a given runtimeType
     const storyResultsTable = new aws_dynamodb.Table(this, 'table', {
       tableName: this.qpqResourceName(QPQ_LOG_BUCKET_NAME, 'log'),
       partitionKey: { name: 'runtimeType', type: aws_dynamodb.AttributeType.STRING },
@@ -83,6 +84,23 @@ export class LogStorage extends QpqConstructBlock {
       removalPolicy: cdk.RemovalPolicy.DESTROY,
     });
 
+    // Query by correlation
+    storyResultsTable.addGlobalSecondaryIndex({
+      indexName: 'CorrelationIndex',
+      partitionKey: { name: 'correlation', type: aws_dynamodb.AttributeType.STRING },
+    });
+
+    // Second table for the from logs ~ Second table because a GSI will dupe the data more then needed
+    // Also, we cant have a GSI on keys that can be undefined, this table will only contain
+    // logs with a fromCorrelation
+    const fromStoryResultsTable = new aws_dynamodb.Table(this, 'from-table', {
+      tableName: this.qpqResourceName(QPQ_FROM_LOG_BUCKET_NAME, 'flog'),
+      partitionKey: { name: 'fromCorrelation', type: aws_dynamodb.AttributeType.STRING },
+      sortKey: { name: 'startedAtWithCorrelation', type: aws_dynamodb.AttributeType.STRING },
+      billingMode: aws_dynamodb.BillingMode.PAY_PER_REQUEST,
+      removalPolicy: cdk.RemovalPolicy.DESTROY,
+    });
+
     this.table = storyResultsTable;
 
     // Create a lambda that is trigged when files are written to the bucket

package/src/lambdas/lambda-utils/getRuntimeCorrelation.ts

@@ -0,0 +1,7 @@
+import { QPQConfig, qpqCoreUtils } from 'quidproquo-core';
+import { awsLambdaUtils } from 'quidproquo-actionprocessor-awslambda';
+
+export const getRuntimeCorrelation = (qpqConfig: QPQConfig): string => {
+  const moduleName = qpqCoreUtils.getApplicationModuleName(qpqConfig);
+  return `${moduleName}-${awsLambdaUtils.randomGuid()}`;
+};

package/src/lambdas/lambda-utils/index.ts

@@ -0,0 +1,2 @@
+export * from './getRuntimeCorrelation';
+export * from './logger';

package/src/lambdas/{logger → lambda-utils}/logger.ts

@@ -10,6 +10,8 @@ export const storyLogger = async (
   result: StoryResult<any>,
   qpqConfig: QPQConfig,
 ): Promise<void> => {
+  /*
+
   const bucketName = awsNamingUtils.getQpqRuntimeResourceNameFromConfig(
     QPQ_LOG_BUCKET_NAME,
     qpqConfig,
@@ -26,6 +28,8 @@ export const storyLogger = async (
       Body: JSON.stringify(result),
     }),
   );
+  
+  */
 };
 
 export const getLogger = (qpqConfig: QPQConfig) => async (result: StoryResult<any>) => {

package/src/lambdas/lambdaAPIGatewayEvent.ts

@@ -29,7 +29,7 @@ import { APIGatewayEvent, Context } from 'aws-lambda';
 import { getLambdaConfigs } from './lambdaConfig';
 import { ActionProcessorListResolver } from './actionProcessorListResolver';
 
-import { getLogger } from './logger/logger';
+import { getLogger, getRuntimeCorrelation } from './lambda-utils';
 
 import { dynamicModuleLoader } from './dynamicModuleLoader';
 
@@ -60,6 +60,7 @@ export const getAPIGatewayEventExecutor = (
 ) => {
   return async (event: APIGatewayEvent, context: Context) => {
     const cdkConfig = await getLambdaConfigs();
+    const accessToken = qpqWebServerUtils.getAccessTokenFromHeaders(event.headers);
 
     // Build a processor for the session and stuff
     // Remove the non route ones ~ let the story execute action add them
@@ -87,11 +88,14 @@ export const getAPIGatewayEventExecutor = (
 
     const resolveStory = createRuntime(
       cdkConfig.qpqConfig,
-      {},
+      {
+        depth: 0,
+        accessToken: accessToken,
+      },
       storyActionProcessor,
       getDateNow,
       getLogger(cdkConfig.qpqConfig),
-      awsLambdaUtils.randomGuid,
+      getRuntimeCorrelation(cdkConfig.qpqConfig),
       QpqRuntimeType.API,
     );
 

package/src/lambdas/lambdaEventBridgeEvent.ts

@@ -1,4 +1,8 @@
-import { coreActionProcessor, webserverActionProcessor } from 'quidproquo-actionprocessor-node';
+import {
+  coreActionProcessor,
+  webserverActionProcessor,
+  getConfigActionProcessor,
+} from 'quidproquo-actionprocessor-node';
 
 import {
   getEventBridgeEventActionProcessor,
@@ -18,15 +22,13 @@ import {
   LambdaRuntimeConfig,
 } from 'quidproquo-actionprocessor-awslambda';
 
-import { getConfigActionProcessor } from 'quidproquo-actionprocessor-node';
-
 import { createRuntime, askProcessEvent, QpqRuntimeType } from 'quidproquo-core';
 
 import { EventBridgeEvent, Context } from 'aws-lambda';
 
 import { getLambdaConfigs } from './lambdaConfig';
 import { ActionProcessorListResolver } from './actionProcessorListResolver';
-import { getLogger } from './logger/logger';
+import { getLogger, getRuntimeCorrelation } from './lambda-utils';
 
 import { dynamicModuleLoader } from './dynamicModuleLoader';
 
@@ -73,12 +75,14 @@ export const getEventBridgeEventExecutor = (
 
     const resolveStory = createRuntime(
       cdkConfig.qpqConfig,
-      {},
+      {
+        depth: 0,
+      },
       storyActionProcessor,
       getDateNow,
       getLogger(cdkConfig.qpqConfig),
-      awsLambdaUtils.randomGuid,
-      QpqRuntimeType.EVENT_BRIDGE_EVENT,
+      getRuntimeCorrelation(cdkConfig.qpqConfig),
+      QpqRuntimeType.RECURRING_SCHEDULE,
     );
 
     await resolveStory(askProcessEvent, [event, context]);

package/src/lambdas/lambdaEventOriginRequest.ts

@@ -1,4 +1,8 @@
-import { coreActionProcessor, webserverActionProcessor } from 'quidproquo-actionprocessor-node';
+import {
+  coreActionProcessor,
+  webserverActionProcessor,
+  getConfigActionProcessor,
+} from 'quidproquo-actionprocessor-node';
 
 import {
   getCloudFrontOriginRequestEventActionProcessor,
@@ -15,8 +19,6 @@ import {
   DynamicModuleLoader,
 } from 'quidproquo-actionprocessor-awslambda';
 
-import { getConfigActionProcessor } from 'quidproquo-actionprocessor-node';
-
 import { getLambdaConfigs } from './lambdaConfig';
 
 import { createRuntime, askProcessEvent, QpqRuntimeType } from 'quidproquo-core';
@@ -24,7 +26,7 @@ import { createRuntime, askProcessEvent, QpqRuntimeType } from 'quidproquo-core'
 import { CloudFrontRequestEvent, CloudFrontRequestResult, Context } from 'aws-lambda';
 
 import { ActionProcessorListResolver } from './actionProcessorListResolver';
-import { getLogger } from './logger/logger';
+import { getLogger, getRuntimeCorrelation } from './lambda-utils';
 
 import { dynamicModuleLoader } from './dynamicModuleLoader';
 
@@ -69,11 +71,13 @@ export const getOriginRequestEventExecutor = (
 
     const resolveStory = createRuntime(
       cdkConfig.qpqConfig,
-      {},
+      {
+        depth: 0,
+      },
       storyActionProcessor,
       getDateNow,
       getLogger(cdkConfig.qpqConfig),
-      awsLambdaUtils.randomGuid,
+      getRuntimeCorrelation(cdkConfig.qpqConfig),
       QpqRuntimeType.EVENT_SEO_OR,
     );
 

package/src/lambdas/lambdaS3FileWriteEvent_log.ts

@@ -59,6 +59,34 @@ const writeStoryResultMetadataToDynamo = async (
   }
 };
 
+export const getFromCorrelationBucketNameFromLog = (
+  storyResultMetadata: StoryResultMetadata,
+  localBucketName: string,
+): string => {
+  if (!storyResultMetadata.fromCorrelation) {
+    return '';
+  }
+
+  // Get the module name of the service that wants the log
+  const fromModuleName = storyResultMetadata.fromCorrelation.split('::')[0];
+
+  // Get the name of the current module where we are executing this code
+  const currentModuleName = storyResultMetadata.moduleName;
+
+  // Split up the bucket / module names into parts
+  const bucketParts = localBucketName.split('-');
+  const moduleParts = fromModuleName.split('-');
+  const currentModuleNameParts = currentModuleName.split('-');
+
+  // Replace the current module name with the module name of the service that wants the log
+  bucketParts.splice(2, currentModuleNameParts.length, ...moduleParts);
+
+  // rebuild the new bucket name
+  const newBucketName = bucketParts.join('-');
+
+  return newBucketName;
+};
+
 export const executeS3FileWriteEvent: S3Handler = async (event: S3Event) => {
   for (const record of event.Records) {
     const bucketName = record.s3.bucket.name;
@@ -67,6 +95,13 @@ export const executeS3FileWriteEvent: S3Handler = async (event: S3Event) => {
 
     const storyResultMetadata = await readLogFromBucket(bucketName, awsRegion, key);
 
+    // Write the main log to the service table
     await writeStoryResultMetadataToDynamo(bucketName, storyResultMetadata, awsRegion);
+
+    // Write the same log to the table of the from service, so it can query links on the fromGuid GSI
+    const fromBucketName = getFromCorrelationBucketNameFromLog(storyResultMetadata, bucketName);
+    if (fromBucketName) {
+      await writeStoryResultMetadataToDynamo(fromBucketName, storyResultMetadata, awsRegion);
+    }
   }
 };

package/src/lambdas/lambdaSQSEvent.ts

@@ -23,10 +23,16 @@ import { getConfigActionProcessor } from 'quidproquo-actionprocessor-node';
 
 import { getLambdaConfigs } from './lambdaConfig';
 
-import { createRuntime, askProcessEvent, QpqRuntimeType } from 'quidproquo-core';
+import {
+  createRuntime,
+  askProcessEvent,
+  QpqRuntimeType,
+  StorySession,
+  QueueMessage,
+} from 'quidproquo-core';
 
 import { ActionProcessorListResolver } from './actionProcessorListResolver';
-import { getLogger } from './logger/logger';
+import { getLogger, getRuntimeCorrelation } from './lambda-utils';
 
 import { dynamicModuleLoader } from './dynamicModuleLoader';
 
@@ -36,9 +42,15 @@ import qpqCustomActionProcessors from 'qpq-custom-action-processors-loader!';
 // TODO: Make this a util or something based on server time or something..
 const getDateNow = () => new Date().toISOString();
 
+// TODO: Unify this once the lambda code moves from CDK to awslambda
+type AnyQueueMessageWithSession = QueueMessage<any> & {
+  storySession: StorySession;
+};
+
 export const getStoryActionRuntime = async (
   dynamicModuleLoader: DynamicModuleLoader,
   getCustomActionProcessors: ActionProcessorListResolver = () => ({}),
+  callerStorySession: StorySession,
 ) => {
   const cdkConfig = await getLambdaConfigs();
 
@@ -68,17 +80,18 @@ export const getStoryActionRuntime = async (
 
   const resolveStory = createRuntime(
     cdkConfig.qpqConfig,
-    {},
+    callerStorySession,
     storyActionProcessor,
     getDateNow,
     getLogger(cdkConfig.qpqConfig),
-    awsLambdaUtils.randomGuid,
+    getRuntimeCorrelation(cdkConfig.qpqConfig),
     QpqRuntimeType.QUEUE_EVENT,
   );
 
   return resolveStory;
 };
 
+// TODO: We could make this entire function a story, that spawns an execute story for each record
 export const getSQSEventExecutor = (
   dynamicModuleLoader: DynamicModuleLoader,
   getCustomActionProcessors: ActionProcessorListResolver = () => ({}),
@@ -90,13 +103,22 @@ export const getSQSEventExecutor = (
     console.log('num batch: ', event.Records.length);
 
     const results = event.Records.map(async (record) => {
+      const runtimeData = JSON.parse(record.body) as AnyQueueMessageWithSession;
+
       const resolveStory = await getStoryActionRuntime(
         dynamicModuleLoader,
         getCustomActionProcessors,
+        runtimeData.storySession,
       );
+
+      const queueMessage: QueueMessage<any> = {
+        type: runtimeData.type,
+        payload: runtimeData.payload,
+      };
+
       return {
         id: record.messageId,
-        result: await resolveStory(askProcessEvent, [record, context]),
+        result: await resolveStory(askProcessEvent, [queueMessage, context]),
       };
     });
 

package/src/lambdas/lambdaServiceFunctionExecute.ts

@@ -1,4 +1,8 @@
-import { coreActionProcessor, webserverActionProcessor } from 'quidproquo-actionprocessor-node';
+import {
+  coreActionProcessor,
+  webserverActionProcessor,
+  getConfigActionProcessor,
+} from 'quidproquo-actionprocessor-node';
 import { ExecuteServiceFunctionEvent } from 'quidproquo-webserver';
 
 import {
@@ -18,15 +22,13 @@ import {
   DynamicModuleLoader,
 } from 'quidproquo-actionprocessor-awslambda';
 
-import { getConfigActionProcessor } from 'quidproquo-actionprocessor-node';
-
-import { createRuntime, askProcessEvent, QpqRuntimeType } from 'quidproquo-core';
+import { createRuntime, askProcessEvent, QpqRuntimeType, StorySession } from 'quidproquo-core';
 
 import { Context } from 'aws-lambda';
 
 import { getLambdaConfigs } from './lambdaConfig';
 import { ActionProcessorListResolver } from './actionProcessorListResolver';
-import { getLogger } from './logger/logger';
+import { getLogger, getRuntimeCorrelation } from './lambda-utils';
 
 // @ts-ignore - Special webpack loader
 import { dynamicModuleLoader } from './dynamicModuleLoader';
@@ -37,11 +39,16 @@ import qpqCustomActionProcessors from 'qpq-custom-action-processors-loader!';
 // TODO: Make this a util or something based on server time or something..
 const getDateNow = () => new Date().toISOString();
 
+// TODO: Unify this once the lambda code moves from CDK to awslambda
+type AnyExecuteServiceFunctionEventWithSession = ExecuteServiceFunctionEvent<any[]> & {
+  storySession: StorySession;
+};
+
 export const getServiceFunctionExecuteEventExecutor = (
   dynamicModuleLoader: DynamicModuleLoader,
   getCustomActionProcessors: ActionProcessorListResolver = () => ({}),
 ) => {
-  return async (event: ExecuteServiceFunctionEvent<any>, context: Context) => {
+  return async (event: AnyExecuteServiceFunctionEventWithSession, context: Context) => {
     const cdkConfig = await getLambdaConfigs();
 
     // Build a processor for the session and stuff
@@ -71,15 +78,20 @@ export const getServiceFunctionExecuteEventExecutor = (
 
     const resolveStory = createRuntime(
       cdkConfig.qpqConfig,
-      {},
+      event.storySession,
       storyActionProcessor,
       getDateNow,
       getLogger(cdkConfig.qpqConfig),
-      awsLambdaUtils.randomGuid,
+      getRuntimeCorrelation(cdkConfig.qpqConfig),
       QpqRuntimeType.SERVICE_FUNCTION_EXE,
     );
 
-    const result = await resolveStory(askProcessEvent, [event, context]);
+    const eventWithNoSession: ExecuteServiceFunctionEvent<any[]> = {
+      functionName: event.functionName,
+      payload: event.payload,
+    };
+
+    const result = await resolveStory(askProcessEvent, [eventWithNoSession, context]);
 
     // just return the story result
     return result;

package/src/lambdas/storyResultToMetadata/genericTextExtractors/index.ts

@@ -1,2 +1,4 @@
 export * from './apiGenericTextExtractor';
+export * from './queueEventGenericTextExtractor';
+export * from './serviceFunctionExeGenericTextExtractor';
 export * from './unknownGenericTextExtractor';

package/src/lambdas/storyResultToMetadata/genericTextExtractors/queueEventGenericTextExtractor.ts

@@ -0,0 +1,9 @@
+import { StoryResult } from 'quidproquo-core';
+
+export const queueEventGenericTextExtractor = (storyResult: StoryResult<any>): string => {
+  if (storyResult.runtimeType === 'QUEUE_EVENT') {
+    return storyResult.input[0]?.type || '';
+  }
+
+  return '';
+};

package/src/lambdas/storyResultToMetadata/genericTextExtractors/serviceFunctionExeGenericTextExtractor.ts

@@ -0,0 +1,9 @@
+import { StoryResult } from 'quidproquo-core';
+
+export const serviceFunctionExeGenericTextExtractor = (storyResult: StoryResult<any>): string => {
+  if (storyResult.runtimeType === 'SERVICE_FUNCTION_EXE') {
+    return storyResult.input[0]?.functionName || '';
+  }
+
+  return '';
+};

package/src/lambdas/storyResultToMetadata/storyResultToMetadata.ts

@@ -1,14 +1,20 @@
 import { StoryResult, StoryResultMetadata } from 'quidproquo-core';
-import { apiGenericTextExtractor, unknownGenericTextExtractor } from './genericTextExtractors';
+import {
+  apiGenericTextExtractor,
+  unknownGenericTextExtractor,
+  queueEventGenericTextExtractor,
+  serviceFunctionExeGenericTextExtractor,
+} from './genericTextExtractors';
 
 // These have to be strings and not enum due to how the lambda is packaged
 const extractors: Record<string, (sr: StoryResult<any>) => string> = {
   ['API']: apiGenericTextExtractor,
-  ['EVENT_BRIDGE_EVENT']: unknownGenericTextExtractor,
+  ['QUEUE_EVENT']: queueEventGenericTextExtractor,
+  ['SERVICE_FUNCTION_EXE']: serviceFunctionExeGenericTextExtractor,
+
+  ['RECURRING_SCHEDULE']: unknownGenericTextExtractor,
   ['EVENT_SEO_OR']: unknownGenericTextExtractor,
   ['EXECUTE_STORY']: unknownGenericTextExtractor,
-  ['QUEUE_EVENT']: unknownGenericTextExtractor,
-  ['SERVICE_FUNCTION_EXE']: unknownGenericTextExtractor,
 };
 
 export const storyResultToMetadata = (

package/src/stacks/InfQpqServiceStack.ts

@@ -24,10 +24,10 @@ export class InfQpqServiceStack extends QpqServiceStack {
     super(scope, id, props);
 
     // Build the log storage drive
-    const logBucket = new LogStorage(this, 'logStorage', {
-      awsAccountId: props.awsAccountId,
-      qpqConfig: props.qpqConfig,
-    }).bucket;
+    // const logBucket = new LogStorage(this, 'logStorage', {
+    //   awsAccountId: props.awsAccountId,
+    //   qpqConfig: props.qpqConfig,
+    // }).bucket;
 
     // Build the storage drives
     const storageDrives = qpqCoreUtils.getStorageDrives(props.qpqConfig).map(