vtlab-generic-functions 1.0.41 → 1.0.43

package/lambdaClass/index.js

@@ -18,14 +18,16 @@ module.exports = class Lambda {
   ftpConnection;
   jobsByTrackingId;
   DEBUG;
+  context;
 
   results = {
     totalError: 0, totalSuccess: 0, error: [], success: [],
   };
 
-  constructor({ carrier, event, DEBUG = false }) {
+  constructor({ carrier, event, DEBUG = false, context }) {
     this.DEBUG = false;
     this.carrier = carrier;
+    this.context = context;
     if (event.body) {
       //Locally we receive event.body, event.body needs to be parsed. In AWS all fields are into event. (event={lodId:'...', payload:[]})
       event = JSON.parse(event.body);
@@ -196,7 +198,7 @@ module.exports = class Lambda {
   };
 
   logMessage = async (message) => {
-    await cloudwatch.putLogEvents(`${this.logUuid} ${message}`, this.logGroupName, 2, this.logStreamName);
+    await cloudwatch.putLogEvents(`${this?.context?.awsRequestId || '-'} ${this.logUuid} ${message}`, this.logGroupName, 2, this.logStreamName);
   };
 
   logError = async (error) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vtlab-generic-functions",
-  "version": "1.0.41",
+  "version": "1.0.43",
   "scripts": {
     "test": "jest"
   },
@@ -10,6 +10,7 @@
     "@aws-sdk/client-ses": "^3.478.0",
     "@aws-sdk/client-sqs": "^3.478.0",
     "@aws-sdk/s3-request-presigner": "^3.484.0",
+    "@supabase/supabase-js": "^2.110.0",
     "axios": "^1.6.2",
     "countries-code": "^1.1.0",
     "country-data": "0.0.31",

package/telemetry/index.js

@@ -0,0 +1,407 @@
+const { createClient } = require("@supabase/supabase-js");
+const Joi = require("joi");
+const { randomBytes } = require("crypto");
+
+const VALID_ENVIRONMENTS = [
+    "production",
+    "develop",
+    "demo",
+    "staging",
+    "testing",
+    "uat"
+];
+
+const statusMap = {
+    error: "FAIL",
+    warning: "WARNING",
+    ok: "SUCCESS"
+};
+
+let defaultClient;
+
+const generateRandomString = (length = 10) => {
+    return randomBytes(Math.ceil(length / 2)).toString("hex").slice(0, length);
+};
+
+const formatError = (err) => {
+    if (!err) return "";
+    if (typeof err === "string") return err;
+    if (err.response && err.response.data) {
+        try {
+            return JSON.stringify(err.response.data);
+        } catch (_) {
+            return String(err.response.data);
+        }
+    }
+    if (err.message) return err.message;
+    try {
+        return JSON.stringify(err);
+    } catch (_) {
+        return String(err);
+    }
+};
+
+const normalizeEnvironment = (environment) => {
+    if (!environment || !VALID_ENVIRONMENTS.includes(environment)) {
+        return "develop";
+    }
+
+    return environment;
+};
+
+const createSupabaseAdminClient = ({ supabaseUrl, serviceRoleKey }) => {
+    if (!supabaseUrl || !serviceRoleKey) {
+        throw new Error("Telemetry Supabase configuration missing: explicit supabaseUrl and serviceRoleKey are required");
+    }
+
+    return createClient(supabaseUrl, serviceRoleKey, {
+        auth: {
+            autoRefreshToken: true,
+            persistSession: true
+        },
+        global: {
+            headers: {
+                apikey: serviceRoleKey,
+                Authorization: `Bearer ${serviceRoleKey}`
+            }
+        }
+    });
+};
+
+const chunkArray = (items, chunkSize) => {
+    const chunks = [];
+    for (let i = 0; i < items.length; i += chunkSize) {
+        chunks.push(items.slice(i, i + chunkSize));
+    }
+    return chunks;
+};
+
+const parseContentBeforeSaving = (record) => {
+    return {
+        uuid: record.uuid,
+        environment: record.environment,
+        process_run_id: record.processId,
+        thread_id: record.threadId || null,
+        status: statusMap[record.status],
+        created_at: record.date,
+        message: record.message,
+        last_event: record.lastEvent,
+        details: record.details || {}
+    };
+};
+
+const createTelemetryClient = (options = {}) => {
+    let runtimeConfig;
+    let supabaseClient;
+    const logger = options.logger || console;
+
+    const resolveRuntimeConfig = async () => {
+        if (runtimeConfig) return runtimeConfig;
+
+        runtimeConfig = {
+            environment: normalizeEnvironment(options.environment),
+            supabaseUrl: options.supabaseUrl,
+            serviceRoleKey: options.serviceRoleKey
+        };
+
+        return runtimeConfig;
+    };
+
+    const getSupabaseClient = async () => {
+        if (supabaseClient) return supabaseClient;
+
+        if (options.supabaseClient) {
+            supabaseClient = options.supabaseClient;
+            return supabaseClient;
+        }
+
+        const resolvedConfig = await resolveRuntimeConfig();
+        supabaseClient = createSupabaseAdminClient({
+            supabaseUrl: resolvedConfig.supabaseUrl,
+            serviceRoleKey: resolvedConfig.serviceRoleKey
+        });
+
+        return supabaseClient;
+    };
+
+    const findProcessId = async (name, account, environment) => {
+        if (!account) {
+            logger.log(`No account provided to find processId in supabase for process ${name} in environment ${environment}`);
+            return false;
+        }
+
+        try {
+            const client = await getSupabaseClient();
+            let query = client.from("processes").select("id").eq("name", name).eq("account", account);
+            if (environment) {
+                query = query.eq("environment", environment);
+            }
+
+            const { data: processId, error: processIdError } = await query
+                .order("created_at", { ascending: false })
+                .limit(1)
+                .maybeSingle();
+
+            if (processIdError) {
+                logger.log(`Error finding processId in supabase: ${formatError(processIdError)} for process ${name} in account ${account} in environment ${environment}`);
+                return false;
+            }
+
+            return processId ? processId.id : false;
+        } catch (e) {
+            logger.log(`Error finding processId in supabase: ${formatError(e)} for process ${name} in account ${account} in environment ${environment}`);
+            return false;
+        }
+    };
+
+    const createProcess = async (params) => {
+        const { processName, account, environment, logGroup, logStream } = params;
+
+        if (!account) {
+            logger.log("No account provided to create process in supabase", processName);
+            return false;
+        }
+
+        try {
+            const client = await getSupabaseClient();
+            let { data: processId, error: processIdError } = await client
+                .from("processes")
+                .insert({
+                    name: processName,
+                    description: `Process ${processName}`,
+                    account,
+                    environment,
+                    log_group: logGroup,
+                    log_stream: logStream,
+                })
+                .select("id")
+                .single();
+
+            if (processIdError) {
+                logger.log(`Error creating process in supabase: ${formatError(processIdError)} for process ${processName} in account ${account} in environment ${environment}`);
+                processId = await findProcessId(processName, account, environment);
+            }
+
+            if (!processId) {
+                logger.log(`Error finding processId in supabase for process ${processName} in account ${account} in environment ${environment}`);
+                return false;
+            }
+
+            return processId.id || processId;
+        } catch (e) {
+            logger.log(`Error creating process in supabase: ${formatError(e)} for process ${processName} in account ${account} in environment ${environment}`);
+            return false;
+        }
+    };
+
+    const createProcessRun = async (processId, params) => {
+        if (!processId) {
+            throw new Error(`Error creating process run in supabase: processId is required but got ${processId}`);
+        }
+
+        const client = await getSupabaseClient();
+        let body = {
+            uuid: params.uuid || params.logExecutionUUID || generateRandomString(10),
+            environment: params.environment,
+            account: params.account || "",
+            status: "STARTED"
+        };
+
+        const insertProcessRun = async () => {
+            return client
+                .from("processes_runs")
+                .insert({
+                    process_id: processId,
+                    ...body
+                })
+                .select("id")
+                .single();
+        };
+
+        let { data: processRunId, error: processRunIdError } = await insertProcessRun();
+
+        const duplicateConstraintMessage = 'duplicate key value violates unique constraint "processes_runs_uuid_key"';
+        if ((processRunIdError || !processRunId) && (processRunIdError && (processRunIdError.code === "23505" || (processRunIdError.message || "").includes(duplicateConstraintMessage)))) {
+            body = {
+                ...body,
+                uuid: `${body.uuid}-${generateRandomString(3)}`
+            };
+
+            ({ data: processRunId, error: processRunIdError } = await insertProcessRun());
+        }
+
+        if (processRunIdError || !processRunId) {
+            throw new Error(`Error creating process run in supabase: ${formatError(processRunIdError)}`);
+        }
+
+        return {
+            id: processRunId.id,
+            uuid: body.uuid
+        };
+    };
+
+    const createTelemetryHeader = async (params = {}) => {
+        try {
+            const resolvedConfig = await resolveRuntimeConfig();
+            const telemetryParams = {
+                ...params,
+                environment: params.environment || resolvedConfig.environment
+            };
+
+            let processId = await findProcessId(
+                telemetryParams.processName,
+                telemetryParams.account,
+                telemetryParams.environment
+            );
+
+            if (!processId) {
+                processId = await createProcess(telemetryParams);
+                if (!processId) {
+                    logger.log("Error creating process in supabase", telemetryParams.processName, telemetryParams.account);
+                    return false;
+                }
+            }
+
+            const processRunIdResults = await createProcessRun(processId, telemetryParams);
+
+            return {
+                processId: processRunIdResults.id,
+                uuid: processRunIdResults.uuid
+            };
+        } catch (e) {
+            logger.log("Error creating telemetry header", e);
+            return false;
+        }
+    };
+
+    const saveBulkEvents = async (events) => {
+        const client = await getSupabaseClient();
+        const { error: processEventsError } = await client.from("processes_events").insert(events);
+        if (processEventsError) {
+            throw new Error(`Error saving bulk events in supabase: ${formatError(processEventsError)}`);
+        }
+        return true;
+    };
+
+    const sendTelemetryEvents = async (events = []) => {
+        try {
+            if (!Array.isArray(events) || events.length === 0) {
+                return true;
+            }
+
+            const resolvedConfig = await resolveRuntimeConfig();
+            const validationSchema = Joi.object({
+                environment: Joi.string().valid(...VALID_ENVIRONMENTS).default(resolvedConfig.environment),
+                processId: Joi.string().required(),
+                uuid: Joi.string().required(),
+                date: Joi.string().isoDate().required().default(new Date().toISOString()),
+                threadId: Joi.string().allow(null).optional(),
+                message: Joi.string().max(300).truncate().required(),
+                lastEvent: Joi.boolean().default(false),
+                status: Joi.string().valid("error", "ok", "warning").default("ok"),
+                details: Joi.object().unknown(true).default({})
+            }).required();
+
+            const chunks = chunkArray(events, 10);
+            const sanitizedEvents = chunks.map((chunk) => {
+                return chunk.map((event) => {
+                    try {
+                        const validationResult = validationSchema.validate(event);
+                        if (validationResult.error) {
+                            throw validationResult.error;
+                        }
+                        return validationResult.value;
+                    } catch (e) {
+                        logger.log(e);
+                        return null;
+                    }
+                }).filter((event) => event !== null);
+            });
+
+            for (const chunk of sanitizedEvents) {
+                try {
+                    if (chunk.length === 0) continue;
+                    const processEvents = chunk.map((event) => parseContentBeforeSaving(event));
+                    const processEventsSaved = await saveBulkEvents(processEvents);
+                    if (!processEventsSaved) {
+                        logger.log("Error saving process events in supabase", processEvents);
+                        return false;
+                    }
+                } catch (_) {
+                    continue;
+                }
+            }
+
+            return true;
+        } catch (e) {
+            logger.log("Error sending telemetry events", e);
+            return false;
+        }
+    };
+
+    const createTelemetryEmitter = (telemetryHeader) => {
+        let telemetryEndEmitted = false;
+
+        const normalizeMessage = (message) => {
+            if (typeof message !== "string") {
+                return "";
+            }
+
+            return message.length > 300 ? `${message.slice(0, 297)}...` : message;
+        };
+
+        const emitTelemetryEvent = async (message, status = "ok", extra = {}) => {
+            const isLastEvent = extra && extra.lastEvent === true;
+
+            if (!telemetryHeader) {
+                return false;
+            }
+
+            if (isLastEvent) {
+                telemetryEndEmitted = true;
+            }
+
+            try {
+                await sendTelemetryEvents([
+                    {
+                        processId: telemetryHeader.processId,
+                        uuid: telemetryHeader.uuid,
+                        date: new Date().toISOString(),
+                        message: normalizeMessage(message),
+                        status,
+                        ...extra
+                    }
+                ]);
+                return true;
+            } catch (_) {
+                return false;
+            }
+        };
+
+        return {
+            emitTelemetryEvent,
+            hasEmittedLastEvent: () => telemetryEndEmitted
+        };
+    };
+
+    return {
+        createTelemetryHeader,
+        sendTelemetryEvents,
+        createTelemetryEmitter
+    };
+};
+
+const getDefaultClient = () => {
+    if (!defaultClient) {
+        defaultClient = createTelemetryClient();
+    }
+
+    return defaultClient;
+};
+
+module.exports = {
+    createTelemetryClient,
+    createTelemetryHeader: (...args) => getDefaultClient().createTelemetryHeader(...args),
+    sendTelemetryEvents: (...args) => getDefaultClient().sendTelemetryEvents(...args),
+    createTelemetryEmitter: (...args) => getDefaultClient().createTelemetryEmitter(...args)
+};