tango-app-api-audit 3.5.4 → 3.5.7-stage.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "tango-app-api-audit",
-    "version": "3.5.4",
+    "version": "3.5.7-stage.0",
     "description": "audit & audit metrics apis",
     "main": "index.js",
     "type": "module",
@@ -27,7 +27,7 @@
         "nodemon": "^3.1.3",
         "swagger-ui-express": "^5.0.1",
         "tango-api-schema": "^2.2.113",
-        "tango-app-api-middleware": "^3.1.76",
+        "tango-app-api-middleware": "^3.1.78",
         "winston": "^3.13.0",
         "winston-daily-rotate-file": "^5.0.0"
     },

package/src/controllers/audit.controllers.js

@@ -31,6 +31,7 @@ import {
   listQueue,
   sqsReceive,
   sendMessageToQueue,
+  sendMessageToFIFOQueue,
 } from 'tango-app-api-middleware/src/utils/aws/sqs.js';
 import {
   aggregateStoreAudit,
@@ -536,10 +537,10 @@ export async function getAuditFile( req, res ) {
         fetchData.ContinuationToken = decodeURIComponent( inputData.nextId );
       }
     }
-
-    const getQuery = {
+    const getQuery = msg.audit_type =='ReAudit'?
+    {
       size: inputData?.limit || 2,
-      _source: [ 'module', 'status', 'date', 'store_id', 'outputCluster', 'personPath', 'isEmployee', 'isJunk', 'EmployeeStatusFinal' ], // Only fetch necessary fields
+      _source: [ 'module', 'status', 'date', 'store_id', 'outputCluster', 'personPath', 'REIDCluster', 'isEmployee', 'isJunk', 'EmployeeStatusFinal' ], // Only fetch necessary fields
 
       query: {
         bool: {
@@ -568,6 +569,33 @@ export async function getAuditFile( req, res ) {
         },
       },
 
+    }:
+    {
+      size: inputData?.limit || 2,
+      _source: [ 'module', 'status', 'date', 'store_id', 'personPath', 'REIDCluster', 'EmployeeStatusProposed', 'isJunk', 'EmployeeStatusFinal' ], // Only fetch necessary fields
+
+      query: {
+        bool: {
+          'must': [
+            {
+              term: { 'store_id.keyword': msg.store_id },
+            },
+            {
+              term: { 'date.keyword': msg.curr_date },
+            },
+            {
+              term: { 'module.keyword': 'CUSTOMER' },
+            },
+            {
+              term: { 'status.keyword': 'PP_CLUSTER_FORMED' },
+            },
+            {
+              terms: { EmployeeStatusProposed: [ 1, 2 ] },
+            },
+          ],
+        },
+      },
+
     };
     const list = msg.zone_id === 'track' ? inputData.nextId? await scrollResponse( inputData.nextId ):await searchOpenSearchData( msg.index_name, getQuery ): await listFileByPath( fetchData );
     const folderPath = msg.zone_id === 'track' ? list?.body?.hits?.hits : list.data;
@@ -581,7 +609,7 @@ export async function getAuditFile( req, res ) {
       for ( let i = 0; i < folderPath.length; i++ ) {
         const img =msg.zone_id === 'track' ?folderPath[i]?._source?.personPath?.split( '/' ): folderPath[i].Key.split( '/' );
         const image = img[3]?.split( '.' );
-        const indexes = msg.zone_id === 'track' ? folderPath[i]?._source?.outputCluster : image[0]?.split( '_' );
+        const indexes = msg.zone_id === 'track' ? msg.audit_type === 'ReAudit'? folderPath[i]?._source?.outputCluster ===50000? folderPath[i]?._source?.REIDCluster:folderPath[i]?._source?.outputCluster:folderPath[i]?._source?.REIDCluster: image[0]?.split( '_' );
         fetchData.file_path = msg.zone_id === 'track' ?folderPath[i]?._source?.personPath:folderPath[i].Key;
         const data =msg.zone_id === 'track' ? `${url.trackInput}${folderPath[i]?._source?.personPath}` :await signedUrl( fetchData );
         const mapimg = {
@@ -1416,8 +1444,27 @@ export async function save( req, res ) {
       }
     } else {
       // Junk Mapping
+      let junkCount = [];
+      let empCount = [];
+      let customerCount = [];
       if ( inputData.junkCount >0 ) {
         const chunkedJunkMappingData = await filteredMap( inputData.junk );
+        junkCount = chunkedJunkMappingData;
+        let filter = [
+          {
+            term: { 'store_id.keyword': inputData.storeId },
+          },
+          {
+            term: { 'date.keyword': inputData.fileDate },
+          },
+          {
+            term: { 'module.keyword': 'CUSTOMER' },
+          },
+          {
+            terms: { REIDCluster: chunkedJunkMappingData },
+          },
+        ];
+
         logger.info( { chunkedJunkMappingData: chunkedJunkMappingData } );
         const getQuery = {
           size: 10000,
@@ -1425,42 +1472,31 @@ export async function save( req, res ) {
           stored_fields: [], // exclude all stored fields
           query: {
             bool: {
-              must: [
-                {
-                  term: { 'store_id.keyword': inputData.storeId },
-                },
-                {
-                  terms: { outputCluster: chunkedJunkMappingData },
-                },
-                {
-                  term: { 'date.keyword': inputData.fileDate },
-                },
-                {
-                  term: { 'module.keyword': 'CUSTOMER' },
-                },
-              ],
+              must: filter,
             },
           },
 
         };
-        const getId = await getOpenSearchData( 'tracker_process', getQuery );
+        const getId = await getOpenSearchData( openSearch.trackerInput, getQuery );
         const ids = getId?.body?.hits?.total?.value || null;
         if ( ids ) {
           const temp = getId?.body?.hits?.hits?.map( ( item ) => item._id );
           const updatedField = { isJunk: true }; // your field updates
           const body = temp.flatMap( ( id ) => [
-            { update: { _index: 'tracker_process', _id: id } },
+            { update: { _index: openSearch.trackerInput, _id: id } },
             { doc: updatedField },
           ] );
           const re= await bulkUpdate( body );
-          logger.info( { getId: getId, re: re } );
+          logger.info( { getId: getId, re: re, storeId: inputData.storeId, fileDate: inputData.fileDate, function: 'tracking save - junk' } );
         }
       }
 
+
       // Employee Mapping
+
       if ( inputData.employeeCount >0 ) {
         const chunkedEmpMappingData = await filteredMap( inputData.employee );
-        logger.info( { chunkedEmpMappingData: chunkedEmpMappingData } );
+        empCount = chunkedEmpMappingData;
         const getQuery = {
           size: 10000,
           _source: false, // do not fetch full source
@@ -1472,7 +1508,7 @@ export async function save( req, res ) {
                   term: { 'store_id.keyword': inputData.storeId },
                 },
                 {
-                  terms: { outputCluster: chunkedEmpMappingData },
+                  terms: { REIDCluster: chunkedEmpMappingData },
                 },
                 {
                   term: { 'date.keyword': inputData.fileDate },
@@ -1485,32 +1521,31 @@ export async function save( req, res ) {
           },
 
         };
-        const getId = await getOpenSearchData( 'tracker_process', getQuery );
+        const getId = await getOpenSearchData( openSearch.trackerInput, getQuery );
         const ids = getId?.body?.hits?.total?.value || null;
         if ( ids ) {
           const temp = getId?.body?.hits?.hits?.map( ( item ) => item._id );
+
           const updatedField = { EmployeeStatusFinal: 0 }; // your field updates
           const body = temp.flatMap( ( id ) => [
-            { update: { _index: 'tracker_process', _id: id } },
+            { update: { _index: openSearch.trackerInput, _id: id } },
             { doc: updatedField },
           ] );
           const re= await bulkUpdate( body );
-          logger.info( { getId: getId, re: re } );
+          logger.info( { getId: getId, re: re, storeId: inputData.storeId, fileDate: inputData.fileDate, function: 'tracking save - Employee' } );
         }
       }
 
       // Customer Mapping
       if ( inputData?.customer?.length > 0 ) {
-        logger.info( { lenght: inputData?.customer?.length } );
         const filterCustomerData = await filteredCustomerMap( inputData.customer );
         const chunkedMappingData = await chunkArray( filterCustomerData, 10 );
 
         chunkedMappingData.map( async ( chunk ) => {
-          logger.info( { chunk: chunk } );
-
           chunk.map( async ( item ) => {
             const imgName= item.img_name;
             const mappedId= item.mappedid.map( ( m ) => m.img_name );
+            customerCount.push( ...mappedId );
             const searchQuery = {
               size: 10000,
               _source: false, // do not fetch full source
@@ -1522,7 +1557,7 @@ export async function save( req, res ) {
                       term: { 'store_id.keyword': inputData.storeId },
                     },
                     {
-                      terms: { outputCluster: mappedId },
+                      terms: { REIDCluster: mappedId },
                     },
                     {
                       term: { 'date.keyword': inputData.fileDate },
@@ -1535,21 +1570,151 @@ export async function save( req, res ) {
               },
 
             };
-            const getData =await getOpenSearchData( 'tracker_process', searchQuery );
+            const getData =await getOpenSearchData( openSearch.trackerInput, searchQuery );
             if ( getData?.body?.hits?.hits?.length > 0 ) {
               const temp = getData?.body?.hits?.hits?.map( ( item ) => item._id );
               const body = temp.flatMap( ( id ) => [
-                { update: { _index: 'tracker_process', _id: id } },
+                { update: { _index: openSearch.trackerInput, _id: id } },
                 { doc: { outputCluster: imgName, EmployeeStatusFinal: 1, isJunk: false } },
               ] );
               const re= await bulkUpdate( body );
-              logger.info( { re: re } );
+              logger.info( { re: re, storeId: inputData.storeId, fileDate: inputData.fileDate, function: 'tracking save - customer' } );
             }
           } );
         } );
       }
+
+      if ( inputData.auditType !== 'ReAudit' ) {
+        // junk Mapping
+        const getJunkQuery = {
+          size: 10000,
+          _source: false, // do not fetch full source
+          stored_fields: [], // exclude all stored fields
+          query: {
+            bool: {
+              must_not: {
+                terms: {
+                  REIDCluster: junkCount, // how can i c=handle if incase junkCount === []
+                },
+              },
+              must: [
+                {
+                  term: { 'store_id.keyword': inputData.storeId },
+                },
+                {
+                  term: { 'date.keyword': inputData.fileDate },
+                },
+                {
+                  term: { 'module.keyword': 'CUSTOMER' },
+                },
+              ],
+            },
+          },
+
+        };
+
+        const getJunkId = await getOpenSearchData( openSearch.trackerInput, getJunkQuery );
+        const junkIds = getJunkId?.body?.hits?.total?.value || null;
+        if ( junkIds ) {
+          const temp = getJunkId?.body?.hits?.hits?.map( ( item ) => item._id );
+          const updatedField = { isJunk: false }; // your field updates
+          const body = temp.flatMap( ( id ) => [
+            { update: { _index: openSearch.trackerInput, _id: id } },
+            { doc: updatedField },
+          ] );
+          const re= await bulkUpdate( body );
+          logger.info( { getJunkId: getJunkId, re: re, storeId: inputData.storeId, auditType: inputData.auditType, fileDate: inputData.fileDate, function: 'tracking save - junk false mapping' } );
+        }
+
+
+        // Employee Mapping
+        const getEmployeeQuery = {
+          size: 10000,
+          _source: false, // do not fetch full source
+          stored_fields: [], // exclude all stored fields
+          query: {
+            bool: {
+              must_not: {
+                terms: {
+                  REIDCluster: empCount, // how can i c=handle if incase empCount === []
+                },
+              },
+              must: [
+                {
+                  term: { 'store_id.keyword': inputData.storeId },
+                },
+                {
+                  term: { 'date.keyword': inputData.fileDate },
+                },
+                {
+                  term: { 'module.keyword': 'CUSTOMER' },
+                },
+                {
+                  terms: { EmployeeStatusProposed: [ 1, 2 ] },
+                },
+              ],
+            },
+          },
+
+        };
+
+        const getEmpId = await getOpenSearchData( openSearch.trackerInput, getEmployeeQuery );
+        const empIds = getEmpId?.body?.hits?.total?.value || null;
+        if ( empIds ) {
+          const temp = getEmpId?.body?.hits?.hits?.map( ( item ) => item._id );
+          const updatedField = { EmployeeStatusFinal: 1 }; // your field updates
+          const body = temp.flatMap( ( id ) => [
+            { update: { _index: openSearch.trackerInput, _id: id } },
+            { doc: updatedField },
+          ] );
+          const re= await bulkUpdate( body );
+          logger.info( { getEmpId: getEmpId, re: re, storeId: inputData.storeId, auditType: inputData.auditType, fileDate: inputData.fileDate, function: 'tracking save - junk false mapping' } );
+        }
+
+        // customer Mapping
+        const getCusQuery = {
+          size: 10000,
+          _source: false, // do not fetch full source
+          stored_fields: [], // exclude all stored fields
+          query: {
+            bool: {
+              must_not: {
+                terms: {
+                  outputCluster: customerCount, // how can i c=handle if incase empCount === []
+                },
+              },
+              must: [
+                {
+                  term: { 'store_id.keyword': inputData.storeId },
+                },
+                {
+                  term: { 'date.keyword': inputData.fileDate },
+                },
+                {
+                  term: { 'module.keyword': 'CUSTOMER' },
+                },
+              ],
+            },
+          },
+
+        };
+
+        const getCusId = await getOpenSearchData( openSearch.trackerInput, getCusQuery );
+        const cusIds = getCusId?.body?.hits?.total?.value || null;
+        if ( cusIds ) {
+          const temp = getCusId?.body?.hits?.hits.map( ( item ) => item._id );
+          logger.info( { custoemrTemp: temp } );
+          const body = temp.flatMap( ( id ) => [
+            { update: { _index: openSearch.trackerInput, _id: id } },
+            { doc: { outputCluster: 50000 } },
+          ] );
+          const re= await bulkUpdate( body );
+          logger.info( { getCusId: getCusId, re: re, storeId: inputData.storeId, auditType: inputData.auditType, fileDate: inputData.fileDate, function: 'tracking save - junk false mapping' } );
+        }
+      }
     }
 
+
     await updateOneUserAudit(
         { _id: new mongoose.Types.ObjectId( inputData.auditId ) },
         {
@@ -1589,7 +1754,7 @@ export async function save( req, res ) {
     const ratio = ( ( storeAuditData.beforeCount - inputData.customerCount ) / storeAuditData.beforeCount );
     if ( ratio < clientData[0].auditConfigs.ratio &&
       inputData.auditType == 'Audit' &&
-      inputData.moduleType == 'traffic'
+      inputData.moduleType == 'traffic1'
     ) {
       logger.info(
           `Hit in Reaudit pushing queue  Store Id=${inputData.storeId},File Date=${inputData.fileDate}, /n \nQueue Name = ${clientData[0].auditConfigs.queueName}`,
@@ -1679,33 +1844,82 @@ export async function save( req, res ) {
             timeSpent: inputData.timeSpent,
           },
       );
-      if ( inputData.moduleType !== 'track' ) {
-        const sqsProduceQueue = {
-          QueueUrl: ( auditConfig?.isCheckMinAfterCount ==1 && inputData.customerCount > auditConfig?.minAfterCount ) ? `${sqs.url}${sqs.feature_new}`: `${sqs.url}${sqs.feature}`,
-          MessageBody: JSON.stringify( {
-            store_id: inputData.storeId,
-            store_date: inputData.fileDate,
-            bucket_name: `${bucket.auditUploadBucket}`,
-            zone_id: inputData.zoneName,
-            process_type: 'audit',
-          } ),
-        };
-        logger.info( { sqsProduceQueue: sqsProduceQueue } );
-        const sqsQueue = await sendMessageToQueue(
-            sqsProduceQueue.QueueUrl,
-            sqsProduceQueue.MessageBody,
-        );
-        logger.info( { sqsQueue: sqsQueue } );
-      } else {
-        const query =
-          { 'module': 'REID-TRACKER-V1',
-            'store_date_time': `${inputData.storeId}_${inputData.fileDate}_23:00`,
-            'status': 'POSTPROCESS_AUDIT_SUCCESS',
-            'status_time': Math.floor( Date.now() / 1000 ),
-            'timestamp': new Date() };
+      const getQuery = [
+        {
+          $match: {
+            $and: [
+              {
+                storeId: {
+                  $eq: inputData.storeId,
 
+                },
 
-        await upsertOpenSearchData( 'tracker_process', `${inputData.storeId}_${inputData.fileDate}_23:00`, { doc: query } );
+              },
+              {
+                fileDate: {
+                  $eq: inputData.fileDate,
+                },
+              },
+              {
+                moduleType: {
+                  $eq: 'track',
+                },
+              },
+            ],
+          },
+        },
+        {
+          $project: {
+            _id: 0,
+            sqs: '$sqs.Body',
+          },
+        },
+      ];
+      const getBody = await aggregateAuditStoreData( getQuery );
+      const streams = getBody? getBody?.sqs?.streams : null;
+      const sqsProduceQueue = inputData.moduleType === 'track'?
+      {
+        QueueUrl: `${sqs.url}${sqs.trackOutput}`,
+        MessageBody: JSON.stringify( {
+          store_id: inputData.storeId,
+          zone_id: [ 'traffic_zone' ],
+          streams: streams || [],
+          store_date: inputData.fileDate,
+          bucket_name: 'tango-yolo-v2-output',
+          time: '23:00',
+          retry: 1,
+        } ),
+        MessageGroupId: 'my-group-1', // Required for FIFO queues
+        MessageDeduplicationId: Date.now().toString(), // Or a UUID, if content-based deduplication is off
+      }:
+     {
+       QueueUrl: ( auditConfig?.isCheckMinAfterCount ==1 && inputData.customerCount > auditConfig?.minAfterCount ) ? `${sqs.url}${sqs.feature_new}`: `${sqs.url}${sqs.feature}`,
+       MessageBody: JSON.stringify( {
+         store_id: inputData.storeId,
+         store_date: inputData.fileDate,
+         bucket_name: `${bucket.auditUploadBucket}`,
+         zone_id: inputData.zoneName,
+         process_type: 'audit',
+       } ),
+     };
+      logger.info( { sqsProduceQueue: sqsProduceQueue } );
+      const sqsQueue = inputData.moduleType === 'track'? await sendMessageToFIFOQueue( sqsProduceQueue ) : await sendMessageToQueue(
+          sqsProduceQueue.QueueUrl,
+          sqsProduceQueue.MessageBody,
+
+      );
+
+      logger.info( { sqsQueue: sqsQueue } );
+      if ( inputData.moduleType === 'track' ) {
+        const query =
+          {
+            module: 'REID-TRACKER-V1',
+            store_date_time: `${inputData.storeId}_${inputData.fileDate}_23:00`,
+            status: 'POSTPROCESS_AUDIT_SUCCESS',
+            status_time: Math.floor( Date.now() / 1000 ),
+            timestamp: new Date(),
+          };
+        await upsertOpenSearchData( openSearch.trackerInput, `${inputData.storeId}_${inputData.fileDate}_23:00`, { doc: query } );
       }
     }
 
@@ -1867,9 +2081,6 @@ export async function userAuditHistory( req, res ) {
               timezone: 'Asia/Kolkata',
             },
           },
-          // timeSpent: { $round: [
-          //   { $divide: [ '$timeSpent', 60 ] }, 2,
-          // ] },
           timeSpent: {
 
             $cond: [
@@ -3253,7 +3464,7 @@ export async function reTrigger( req, res ) {
     };
 
 
-    await updateOpenSearchData( 'tracker_process', `${inputData.storeId}_${inputData.fileDate}_23:00`, { doc: updateQuery } );
+    await updateOpenSearchData( openSearch.trackerInput, `${inputData.storeId}_${inputData.fileDate}_23:00`, { doc: updateQuery } );
     return res.sendSuccess( { result: 'The File has been Re-Triggered Succesfully' } );
   } catch ( error ) {
     const err = error.message;