tango-app-api-audit 3.5.3 → 3.5.6

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "tango-app-api-audit",
-    "version": "3.5.3",
+    "version": "3.5.6",
     "description": "audit & audit metrics apis",
     "main": "index.js",
     "type": "module",
@@ -27,7 +27,7 @@
         "nodemon": "^3.1.3",
         "swagger-ui-express": "^5.0.1",
         "tango-api-schema": "^2.2.113",
-        "tango-app-api-middleware": "^3.1.76",
+        "tango-app-api-middleware": "^3.1.78",
         "winston": "^3.13.0",
         "winston-daily-rotate-file": "^5.0.0"
     },

package/src/controllers/audit.controllers.js

@@ -31,6 +31,7 @@ import {
   listQueue,
   sqsReceive,
   sendMessageToQueue,
+  sendMessageToFIFOQueue,
 } from 'tango-app-api-middleware/src/utils/aws/sqs.js';
 import {
   aggregateStoreAudit,
@@ -537,7 +538,40 @@ export async function getAuditFile( req, res ) {
       }
     }
 
-    const getQuery = {
+    const getQuery = msg.audit_type =='Audit'?
+    {
+      size: inputData?.limit || 2,
+      _source: [ 'module', 'status', 'date', 'store_id', 'personPath', 'REIDCluster', 'EmployeeStatusProposed', 'isJunk', 'EmployeeStatusFinal' ], // Only fetch necessary fields
+
+      query: {
+        bool: {
+          'must': [
+            {
+              term: { 'store_id.keyword': msg.store_id },
+            },
+            {
+              term: { 'date.keyword': msg.curr_date },
+            },
+            {
+              term: { 'module.keyword': 'CUSTOMER' },
+            },
+            {
+              term: { 'status.keyword': 'PP_CLUSTER_FORMED' },
+            },
+            {
+              terms: { EmployeeStatusProposed: [ 1, 2 ] },
+            },
+          ],
+          // 'should': [
+          //   { 'term': { 'isJunk': false } },
+          //   { 'bool': { 'must_not': { 'exists': { 'field': 'isJunk' } } } },
+          // ],
+          // 'minimum_should_match': 1,
+        },
+      },
+
+    }:
+    {
       size: inputData?.limit || 2,
       _source: [ 'module', 'status', 'date', 'store_id', 'outputCluster', 'personPath', 'isEmployee', 'isJunk', 'EmployeeStatusFinal' ], // Only fetch necessary fields
 
@@ -581,7 +615,7 @@ export async function getAuditFile( req, res ) {
       for ( let i = 0; i < folderPath.length; i++ ) {
         const img =msg.zone_id === 'track' ?folderPath[i]?._source?.personPath?.split( '/' ): folderPath[i].Key.split( '/' );
         const image = img[3]?.split( '.' );
-        const indexes = msg.zone_id === 'track' ? folderPath[i]?._source?.outputCluster : image[0]?.split( '_' );
+        const indexes = msg.zone_id === 'track' ? folderPath[i]?._source?.outputCluster || folderPath[i]?._source?.REIDCluster : image[0]?.split( '_' );
         fetchData.file_path = msg.zone_id === 'track' ?folderPath[i]?._source?.personPath:folderPath[i].Key;
         const data =msg.zone_id === 'track' ? `${url.trackInput}${folderPath[i]?._source?.personPath}` :await signedUrl( fetchData );
         const mapimg = {
@@ -1416,22 +1450,170 @@ export async function save( req, res ) {
       }
     } else {
       // Junk Mapping
+      let junkCount = [];
+      let empCount = [];
+      // let customerCount = [];
       if ( inputData.junkCount >0 ) {
         const chunkedJunkMappingData = await filteredMap( inputData.junk );
-        logger.info( { chunkedJunkMappingData: chunkedJunkMappingData } );
-        const getQuery = {
+        junkCount = chunkedJunkMappingData;
+        let filter = [
+          {
+            term: { 'store_id.keyword': inputData.storeId },
+          },
+          {
+            term: { 'date.keyword': inputData.fileDate },
+          },
+          {
+            term: { 'module.keyword': 'CUSTOMER' },
+          },
+        ];
+inputData.auditType == 'Audit'? filter.push(
+    {
+      terms: { REIDCluster: chunkedJunkMappingData },
+    },
+):
+filter.push(
+    {
+      terms: { outputCluster: chunkedJunkMappingData },
+    },
+);
+logger.info( { chunkedJunkMappingData: chunkedJunkMappingData } );
+const getQuery = {
+  size: 10000,
+  _source: false, // do not fetch full source
+  stored_fields: [], // exclude all stored fields
+  query: {
+    bool: {
+      must: filter,
+    },
+  },
+
+};
+const getId = await getOpenSearchData( openSearch.trackerInput, getQuery );
+const ids = getId?.body?.hits?.total?.value || null;
+if ( ids ) {
+  const temp = getId?.body?.hits?.hits?.map( ( item ) => item._id );
+  const updatedField = { isJunk: true }; // your field updates
+  const body = temp.flatMap( ( id ) => [
+    { update: { _index: openSearch.trackerInput, _id: id } },
+    { doc: updatedField },
+  ] );
+  const re= await bulkUpdate( body );
+  logger.info( { getId: getId, re: re, storeId: inputData.storeId, fileDate: inputData.fileDate, function: 'tracking save - junk' } );
+}
+      }
+
+
+      // Employee Mapping
+      {
+        if ( inputData.employeeCount >0 ) {
+          const chunkedEmpMappingData = await filteredMap( inputData.employee );
+          empCount = chunkedEmpMappingData;
+          const getQuery = {
+            size: 10000,
+            _source: false, // do not fetch full source
+            stored_fields: [], // exclude all stored fields
+            query: {
+              bool: {
+                must: [
+                  {
+                    term: { 'store_id.keyword': inputData.storeId },
+                  },
+                  {
+                    terms: { outputCluster: chunkedEmpMappingData },
+                  },
+                  {
+                    term: { 'date.keyword': inputData.fileDate },
+                  },
+                  {
+                    term: { 'module.keyword': 'CUSTOMER' },
+                  },
+                ],
+              },
+            },
+
+          };
+          const getId = await getOpenSearchData( openSearch.trackerInput, getQuery );
+          const ids = getId?.body?.hits?.total?.value || null;
+          if ( ids ) {
+            const temp = getId?.body?.hits?.hits?.map( ( item ) => item._id );
+
+            const updatedField = { EmployeeStatusFinal: 0 }; // your field updates
+            const body = temp.flatMap( ( id ) => [
+              { update: { _index: openSearch.trackerInput, _id: id } },
+              { doc: updatedField },
+            ] );
+            const re= await bulkUpdate( body );
+            logger.info( { getId: getId, re: re, storeId: inputData.storeId, fileDate: inputData.fileDate, function: 'tracking save - Employee' } );
+          }
+        }
+      }
+
+      // Customer Mapping
+      if ( inputData?.customer?.length > 0 ) {
+        const filterCustomerData = await filteredCustomerMap( inputData.customer );
+        const chunkedMappingData = await chunkArray( filterCustomerData, 10 );
+
+        chunkedMappingData.map( async ( chunk ) => {
+          chunk.map( async ( item ) => {
+            const imgName= item.img_name;
+            const mappedId= item.mappedid.map( ( m ) => m.img_name );
+            // customerCount=mappedId;
+            const searchQuery = {
+              size: 10000,
+              _source: false, // do not fetch full source
+              stored_fields: [], // exclude all stored fields
+              query: {
+                bool: {
+                  must: [
+                    {
+                      term: { 'store_id.keyword': inputData.storeId },
+                    },
+                    {
+                      terms: { outputCluster: mappedId },
+                    },
+                    {
+                      term: { 'date.keyword': inputData.fileDate },
+                    },
+                    {
+                      term: { 'module.keyword': 'CUSTOMER' },
+                    },
+                  ],
+                },
+              },
+
+            };
+            const getData =await getOpenSearchData( openSearch.trackerInput, searchQuery );
+            if ( getData?.body?.hits?.hits?.length > 0 ) {
+              const temp = getData?.body?.hits?.hits?.map( ( item ) => item._id );
+              const body = temp.flatMap( ( id ) => [
+                { update: { _index: openSearch.trackerInput, _id: id } },
+                { doc: { outputCluster: imgName, EmployeeStatusFinal: 1, isJunk: false } },
+              ] );
+              const re= await bulkUpdate( body );
+              logger.info( { re: re, storeId: inputData.storeId, fileDate: inputData.fileDate, function: 'tracking save - customer' } );
+            }
+          } );
+        } );
+      }
+
+      if ( inputData.auditType == 'Audit' ) {
+        // junk Mapping
+        const getJunkQuery = {
           size: 10000,
           _source: false, // do not fetch full source
           stored_fields: [], // exclude all stored fields
           query: {
             bool: {
+              must_not: {
+                terms: {
+                  REIDCluster: junkCount, // how can i c=handle if incase junkCount === []
+                },
+              },
               must: [
                 {
                   term: { 'store_id.keyword': inputData.storeId },
                 },
-                {
-                  terms: { outputCluster: chunkedJunkMappingData },
-                },
                 {
                   term: { 'date.keyword': inputData.fileDate },
                 },
@@ -1443,113 +1625,110 @@ export async function save( req, res ) {
           },
 
         };
-        const getId = await getOpenSearchData( 'tracker_process', getQuery );
-        const ids = getId?.body?.hits?.total?.value || null;
-        if ( ids ) {
-          const temp = getId?.body?.hits?.hits?.map( ( item ) => item._id );
-          const updatedField = { isJunk: true }; // your field updates
+
+        const getJunkId = await getOpenSearchData( openSearch.trackerInput, getJunkQuery );
+        const junkIds = getJunkId?.body?.hits?.total?.value || null;
+        if ( junkIds ) {
+          const temp = getJunkId?.body?.hits?.hits?.map( ( item ) => item._id );
+          const updatedField = { isJunk: false }; // your field updates
           const body = temp.flatMap( ( id ) => [
-            { update: { _index: 'tracker_process', _id: id } },
+            { update: { _index: openSearch.trackerInput, _id: id } },
             { doc: updatedField },
           ] );
           const re= await bulkUpdate( body );
-          logger.info( { getId: getId, re: re } );
+          logger.info( { getJunkId: getJunkId, re: re, storeId: inputData.storeId, auditType: inputData.auditType, fileDate: inputData.fileDate, function: 'tracking save - junk false mapping' } );
         }
-      }
 
-      // Employee Mapping
-      if ( inputData.employeeCount >0 ) {
-        const chunkedEmpMappingData = await filteredMap( inputData.employee );
-        logger.info( { chunkedEmpMappingData: chunkedEmpMappingData } );
-        const getQuery = {
+
+        // Employee Mapping
+        const getEmployeeQuery = {
           size: 10000,
           _source: false, // do not fetch full source
           stored_fields: [], // exclude all stored fields
           query: {
             bool: {
+              must_not: {
+                terms: {
+                  REIDCluster: empCount, // how can i c=handle if incase empCount === []
+                },
+              },
               must: [
                 {
                   term: { 'store_id.keyword': inputData.storeId },
                 },
-                {
-                  terms: { outputCluster: chunkedEmpMappingData },
-                },
                 {
                   term: { 'date.keyword': inputData.fileDate },
                 },
                 {
                   term: { 'module.keyword': 'CUSTOMER' },
                 },
+                {
+                  term: { EmployeeStatusProposed: 1 },
+                },
               ],
             },
           },
 
         };
-        const getId = await getOpenSearchData( 'tracker_process', getQuery );
-        const ids = getId?.body?.hits?.total?.value || null;
-        if ( ids ) {
-          const temp = getId?.body?.hits?.hits?.map( ( item ) => item._id );
-          const updatedField = { isEmployee: true }; // your field updates
+
+        const getEmpId = await getOpenSearchData( openSearch.trackerInput, getEmployeeQuery );
+        const empIds = getEmpId?.body?.hits?.total?.value || null;
+        if ( empIds ) {
+          const temp = getEmpId?.body?.hits?.hits?.map( ( item ) => item._id );
+          const updatedField = { EmployeeStatusFinal: 1 }; // your field updates
           const body = temp.flatMap( ( id ) => [
-            { update: { _index: 'tracker_process', _id: id } },
+            { update: { _index: openSearch.trackerInput, _id: id } },
             { doc: updatedField },
           ] );
           const re= await bulkUpdate( body );
-          logger.info( { getId: getId, re: re } );
+          logger.info( { getEmpId: getEmpId, re: re, storeId: inputData.storeId, auditType: inputData.auditType, fileDate: inputData.fileDate, function: 'tracking save - junk false mapping' } );
         }
-      }
-
-      // Customer Mapping
-      if ( inputData?.customer?.length > 0 ) {
-        logger.info( { lenght: inputData?.customer?.length } );
-        const filterCustomerData = await filteredCustomerMap( inputData.customer );
-        const chunkedMappingData = await chunkArray( filterCustomerData, 10 );
-
-        chunkedMappingData.map( async ( chunk ) => {
-          logger.info( { chunk: chunk } );
 
-          chunk.map( async ( item ) => {
-            const imgName= item.img_name;
-            const mappedId= item.mappedid.map( ( m ) => m.img_name );
-            const searchQuery = {
-              size: 10000,
-              _source: false, // do not fetch full source
-              stored_fields: [], // exclude all stored fields
-              query: {
-                bool: {
-                  must: [
-                    {
-                      term: { 'store_id.keyword': inputData.storeId },
-                    },
-                    {
-                      terms: { outputCluster: mappedId },
-                    },
-                    {
-                      term: { 'date.keyword': inputData.fileDate },
-                    },
-                    {
-                      term: { 'module.keyword': 'CUSTOMER' },
-                    },
-                  ],
-                },
-              },
-
-            };
-            const getData =await getOpenSearchData( 'tracker_process', searchQuery );
-            if ( getData?.body?.hits?.hits?.length > 0 ) {
-              const temp = getData?.body?.hits?.hits?.map( ( item ) => item._id );
-              const body = temp.flatMap( ( id ) => [
-                { update: { _index: 'tracker_process', _id: id } },
-                { doc: { outputCluster: imgName, isEmployee: false, isJunk: false } },
-              ] );
-              const re= await bulkUpdate( body );
-              logger.info( { re: re } );
-            }
-          } );
-        } );
+        // customer Mapping
+        // const getCusQuery = {
+        //   size: 10000,
+        //   _source: true, // do not fetch full source
+        //   stored_fields: [], // exclude all stored fields
+        //   query: {
+        //     bool: {
+        //       must_not: {
+        //         terms: {
+        //           outputCluster: customerCount, // how can i c=handle if incase empCount === []
+        //         },
+        //       },
+        //       must: [
+        //         {
+        //           term: { 'store_id.keyword': inputData.storeId },
+        //         },
+        //         {
+        //           term: { 'date.keyword': inputData.fileDate },
+        //         },
+        //         {
+        //           term: { 'module.keyword': 'CUSTOMER' },
+        //         },
+        //       ],
+        //     },
+        //   },
+
+        // };
+
+        // const getCusId = await getOpenSearchData( openSearch.trackerInput, getCusQuery );
+        // const cusIds = getCusId?.body?.hits?.total?.value || null;
+        // if ( cusIds ) {
+        //   const temp = getCusId?.body?.hits?.hits; // .map( ( item ) => {id:item._id,reidCluster:item._source.REIDCluster } );
+        //   logger.info( { custoemrTemp: temp } );
+        //   // const updatedField = ; // your field updates
+        //   const body = temp.flatMap( ( id ) => [
+        //     { update: { _index: openSearch.trackerInput, _id: id._id } },
+        //     { doc: { outputCluster: id._source.REIDCluster } },
+        //   ] );
+        //   const re= await bulkUpdate( body );
+        //   logger.info( { getCusId: getCusId, re: re, storeId: inputData.storeId, auditType: inputData.auditType, fileDate: inputData.fileDate, function: 'tracking save - junk false mapping' } );
+        // }
       }
     }
 
+
     await updateOneUserAudit(
         { _id: new mongoose.Types.ObjectId( inputData.auditId ) },
         {
@@ -1679,33 +1858,82 @@ export async function save( req, res ) {
             timeSpent: inputData.timeSpent,
           },
       );
-      if ( inputData.moduleType !== 'track' ) {
-        const sqsProduceQueue = {
-          QueueUrl: ( auditConfig?.isCheckMinAfterCount ==1 && inputData.customerCount > auditConfig?.minAfterCount ) ? `${sqs.url}${sqs.feature_new}`: `${sqs.url}${sqs.feature}`,
-          MessageBody: JSON.stringify( {
-            store_id: inputData.storeId,
-            store_date: inputData.fileDate,
-            bucket_name: `${bucket.auditUploadBucket}`,
-            zone_id: inputData.zoneName,
-            process_type: 'audit',
-          } ),
-        };
-        logger.info( { sqsProduceQueue: sqsProduceQueue } );
-        const sqsQueue = await sendMessageToQueue(
-            sqsProduceQueue.QueueUrl,
-            sqsProduceQueue.MessageBody,
-        );
-        logger.info( { sqsQueue: sqsQueue } );
-      } else {
-        const query =
-          { 'module': 'REID-TRACKER-V1',
-            'store_date_time': `${inputData.storeId}_${inputData.fileDate}_23:00`,
-            'status': 'POSTPROCESS_AUDIT_SUCCESS',
-            'status_time': Math.floor( Date.now() / 1000 ),
-            'timestamp': new Date() };
+      const getQuery = [
+        {
+          $match: {
+            $and: [
+              {
+                storeId: {
+                  $eq: inputData.storeId,
 
+                },
 
-        await upsertOpenSearchData( 'tracker_process', `${inputData.storeId}_${inputData.fileDate}_23:00`, { doc: query } );
+              },
+              {
+                fileDate: {
+                  $eq: inputData.fileDate,
+                },
+              },
+              {
+                moduleType: {
+                  $eq: 'track',
+                },
+              },
+            ],
+          },
+        },
+        {
+          $project: {
+            _id: 0,
+            sqs: '$sqs.Body',
+          },
+        },
+      ];
+      const getBody = await aggregateAuditStoreData( getQuery );
+      const streams = getBody? getBody?.sqs?.streams : null;
+      const sqsProduceQueue = inputData.moduleType === 'track'?
+      {
+        QueueUrl: `${sqs.url}${sqs.trackOutput}`,
+        MessageBody: JSON.stringify( {
+          store_id: inputData.storeId,
+          zone_id: [ 'traffic_zone' ],
+          streams: streams || [],
+          store_date: inputData.fileDate,
+          bucket_name: 'tango-yolo-v2-output',
+          time: '23:00',
+          retry: 1,
+        } ),
+        MessageGroupId: 'my-group-1', // Required for FIFO queues
+        MessageDeduplicationId: Date.now().toString(), // Or a UUID, if content-based deduplication is off
+      }:
+     {
+       QueueUrl: ( auditConfig?.isCheckMinAfterCount ==1 && inputData.customerCount > auditConfig?.minAfterCount ) ? `${sqs.url}${sqs.feature_new}`: `${sqs.url}${sqs.feature}`,
+       MessageBody: JSON.stringify( {
+         store_id: inputData.storeId,
+         store_date: inputData.fileDate,
+         bucket_name: `${bucket.auditUploadBucket}`,
+         zone_id: inputData.zoneName,
+         process_type: 'audit',
+       } ),
+     };
+      logger.info( { sqsProduceQueue: sqsProduceQueue } );
+      const sqsQueue = inputData.moduleType === 'track'? await sendMessageToFIFOQueue( sqsProduceQueue ) : await sendMessageToQueue(
+          sqsProduceQueue.QueueUrl,
+          sqsProduceQueue.MessageBody,
+
+      );
+
+      logger.info( { sqsQueue: sqsQueue } );
+      if ( inputData.moduleType === 'track' ) {
+        const query =
+          {
+            module: 'REID-TRACKER-V1',
+            store_date_time: `${inputData.storeId}_${inputData.fileDate}_23:00`,
+            status: 'POSTPROCESS_AUDIT_SUCCESS',
+            status_time: Math.floor( Date.now() / 1000 ),
+            timestamp: new Date(),
+          };
+        await upsertOpenSearchData( openSearch.trackerInput, `${inputData.storeId}_${inputData.fileDate}_23:00`, { doc: query } );
       }
     }
 
@@ -1867,9 +2095,6 @@ export async function userAuditHistory( req, res ) {
               timezone: 'Asia/Kolkata',
             },
           },
-          // timeSpent: { $round: [
-          //   { $divide: [ '$timeSpent', 60 ] }, 2,
-          // ] },
           timeSpent: {
 
             $cond: [
@@ -3253,7 +3478,7 @@ export async function reTrigger( req, res ) {
     };
 
 
-    await updateOpenSearchData( 'tracker_process', `${inputData.storeId}_${inputData.fileDate}_23:00`, { doc: updateQuery } );
+    await updateOpenSearchData( openSearch.trackerInput, `${inputData.storeId}_${inputData.fileDate}_23:00`, { doc: updateQuery } );
     return res.sendSuccess( { result: 'The File has been Re-Triggered Succesfully' } );
   } catch ( error ) {
     const err = error.message;