vtlab-generic-functions 1.0.18 → 1.0.19

package/cloudwatch/index.js

@@ -1,12 +1,14 @@
 const { SQSClient, SendMessageCommand } = require('@aws-sdk/client-sqs');
 const { randomUUID } = require('crypto');
 const config = require('../config');
+const s3bucket = require("../s3bucket");
+const secretsManager = require("../config");
 
 if (!global.LOG_STREAM_NAME) {
     global.LOG_STREAM_NAME = `${process.env.AWS_LAMBDA_FUNCTION_NAME}-${process.env.AWS_LAMBDA_LOG_STREAM_NAME}-${randomUUID()}`;
 }
 
-function safeStringify(obj) {
+const safeStringify = obj => {
     //this avoids circular references
     const cache = new Set();
     return JSON.stringify(obj, (key, value) => {
@@ -19,15 +21,39 @@ function safeStringify(obj) {
         }
         return value;
     });
-}
+};
+
+const generateTimestampWithRandomLetters = () => {
+    const getRandomLetter = () => {
+        const letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
+        return letters.charAt(Math.floor(Math.random() * letters.length));
+    };
+
+    const getRandomLetters = (count) => {
+        let result = '';
+        for (let i = 0; i < count; i++) {
+            result += getRandomLetter();
+        }
+        return result;
+    };
+
+    const timestamp = Math.floor(Date.now() / 1000);
+    const randomLetters = getRandomLetters(4);
+
+    return `${timestamp}${randomLetters}`;
+};
 
 const putLogEvents = async (logEvents = [], logGroupName, logLevel = 2, logStreamName = null) => {
     if (logLevel === 0) {
         return;
     }
 
-    if (!logGroupName) {
-        throw new Error('logEvents and logGroupName are required');
+    if(!logStreamName){
+        logStreamName = global.LOG_STREAM_NAME;
+    }
+
+    if (!logGroupName || !logStreamName) {
+        throw new Error('logEvents and logGroupName and logStreamName are required');
     }
 
     let awsCredentials = await config.get('awsCredentials');
@@ -82,20 +108,40 @@ const putLogEvents = async (logEvents = [], logGroupName, logLevel = 2, logStrea
     }
 
     let cloudWatchLogEvents = [];
-    let maxBytes = 200000;
+    //max 8kb log event
+    let maxBytes = 8*1024;
     for (let logEvent of logEvents) {
         if (logEvent.length > maxBytes) {
-            for (let j = 0; j < logEvent.length; j += maxBytes) {
-                let logEventChunk = logEvent.substring(j, j + maxBytes);
+            let bucketOptions = {
+                ACL: "private", ContentType: 'text/plain'
+            }
+            //get first 300 chars
+            const logFirst300Chars = logEvent.substring(0, 300);
+            try {
+                let groupFolder = logGroupName.replace(/\//g, '_');
+                let streamName = logStreamName.replace(/\//g, '_');
+                const path = `${process.env.NODE_ENV}/errors/${groupFolder}/${streamName}/${generateTimestampWithRandomLetters()}.txt`;
+                //upload logevent to s3
+                let bucketConfirmedPath = await s3bucket.uploadToS3(path, logEvent, bucketOptions, "s3LogsBucket");
+                console.log("bucketConfirmedPath", bucketConfirmedPath);
+                // get bucket name from s3 secrets
+                let bucket = await secretsManager.get('s3LogsBucket');
+                let signedURL = await s3bucket.generatePresignedUrl(bucket, path);
+
+                // Set event with the signed URL
+                cloudWatchLogEvents.push({
+                    message: `${logLevel} ${logFirst300Chars}... Payload size too big. Read the full log here: ${path} ${signedURL}`,
+                    timestamp: new Date().getTime()
+                });
+            } catch (e) {
                 cloudWatchLogEvents.push({
-                    message: `${logLevel} ${logEventChunk}`,
+                    message: `${logLevel} ${logFirst300Chars}... Payload size too big. Error uploading to S3: ${e.message}`,
                     timestamp: new Date().getTime()
                 });
             }
         } else {
             cloudWatchLogEvents.push({
-                message: `${logLevel} ${logEvent}`,
-                timestamp: new Date().getTime()
+                message: `${logLevel} ${logEvent}`, timestamp: new Date().getTime()
             });
         }
     }
@@ -113,7 +159,7 @@ const putLogEvents = async (logEvents = [], logGroupName, logLevel = 2, logStrea
             MessageBody: JSON.stringify({
                 logEvents: [event],
                 logGroupName: logGroupName,
-                logStreamName: logStreamName || global.LOG_STREAM_NAME,
+                logStreamName: logStreamName,
                 logLevel: logLevel
             }),
             QueueUrl

package/config/index.js

@@ -16,27 +16,19 @@ const getObjectByString = (obj, key) => {
 // Cache object
 let cache = {};
 
-const getConfigValuesFromS3Bucket = async (configKey = null, env = null) => {
+const getConfigValuesFromS3Bucket = async (configKey = null, ifValuesFromRoot = null) => {
     const { NODE_ENV, TOOK_SECRETS_BUCKET, TOOK_SECRETS_DEVELOP, TOOK_SECRETS_PRODUCTION } = process.env;
     let defaultEnvironment = NODE_ENV;
-    let valuesFromRoot;
-    if(env==="rds" || env==="notificationEmailForDebug") {
-        valuesFromRoot = env;
-        env = null;
-    }
     const availableEnvironments = ['production', 'develop', 'staging', 'uat', 'demo', 'test', 'testing'];
     if (!defaultEnvironment || !availableEnvironments.includes(defaultEnvironment)) {
         defaultEnvironment = 'develop';
     }
-    let effectiveEnvironment = env || defaultEnvironment;
+    const effectiveEnvironment = ifValuesFromRoot || defaultEnvironment;
     let secretName = TOOK_SECRETS_DEVELOP;
     if (effectiveEnvironment === 'production') {
         secretName = TOOK_SECRETS_PRODUCTION;
     }
     const secretsBucketName = TOOK_SECRETS_BUCKET;
-    if(valuesFromRoot) {
-        effectiveEnvironment = valuesFromRoot;
-    }
 
     // Check cache first
     if (cache[effectiveEnvironment] && (!configKey || cache[effectiveEnvironment][configKey])) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vtlab-generic-functions",
-  "version": "1.0.18",
+  "version": "1.0.19",
   "scripts": {
     "test": "jest"
   },

package/rds/index.js

@@ -9,7 +9,7 @@ const {getValueOrNull} = require('../utils/index');
  */
 const connectMyRDS = async (databaseName) => {
     try {
-        const dbParams = await config.get(databaseName, 'rds');
+        const dbParams = await config.get(`databaseName.rds`);
 
         if (!dbParams) {
             throw new Error('Database name is not valid in the Secret Manager');