tango-app-api-audit 3.5.8 → 3.5.10

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "tango-app-api-audit",
-    "version": "3.5.8",
+    "version": "3.5.10",
     "description": "audit & audit metrics apis",
     "main": "index.js",
     "type": "module",
@@ -26,8 +26,8 @@
         "mongodb": "^6.7.0",
         "nodemon": "^3.1.3",
         "swagger-ui-express": "^5.0.1",
-        "tango-api-schema": "^2.2.113",
-        "tango-app-api-middleware": "^3.1.78",
+        "tango-api-schema": "^2.2.137",
+        "tango-app-api-middleware": "^3.1.80",
         "winston": "^3.13.0",
         "winston-daily-rotate-file": "^5.0.0"
     },

package/src/controllers/audit.controllers.js

@@ -597,6 +597,7 @@ export async function getAuditFile( req, res ) {
       },
 
     };
+    logger.info( { fetchData: fetchData, msg: msg, inputData: inputData, type: 'debug-bucketNameIssue' } );
     const list = msg.zone_id === 'track' ? inputData.nextId? await scrollResponse( inputData.nextId ):await searchOpenSearchData( msg.index_name, getQuery ): await listFileByPath( fetchData );
     const folderPath = msg.zone_id === 'track' ? list?.body?.hits?.hits : list.data;
     const nextQuery =msg.zone_id !== 'track' ? list.pageToken: '';

package/src/controllers/eyeTestAudit.controllers.js

@@ -1,4 +1,4 @@
-import { getOpenSearchData, getOpenSearchById, logger, insertOpenSearchData, updateOpenSearchData, download, chunkArray } from 'tango-app-api-middleware';
+import { getOpenSearchData, getOpenSearchById, logger, insertOpenSearchData, updateOpenSearchData, download, chunkArray, getOpenSearchCount } from 'tango-app-api-middleware';
 import { findOneUser } from '../service/user.service.js';
 import mongoose from 'mongoose';
 import dayjs from 'dayjs';
@@ -19,7 +19,7 @@ export async function getList( req, res ) {
     const limit = inputData.limit || 10;
     const offset = inputData.offset ? ( inputData.offset - 1 ) * limit : 0;
 
-    const sortBy = inputData?.sortBy ? inputData.sortBy === 'coveredStepsAI'||inputData.sortBy==='updatedAt' ? inputData.sortBy :`${inputData.sortBy}.keyword` : 'storeName.keyword';
+    const sortBy = inputData?.sortBy ? [ 'coveredStepsAI', 'trustScore', 'date' ].includes( inputData.sortBy ) ? inputData.sortBy :`${inputData.sortBy}.keyword` : 'storeName.keyword';
     const order = inputData?.sortOrder || -1;
     let filter= [
       {
@@ -179,10 +179,11 @@ export async function getList( req, res ) {
         },
         'sort': [
           { [sortBy]: { order: order === -1 ?'desc':'asc' } },
+          { '_id': 'asc' },
         ],
+        // 'search_after': null,
       };
     }
-
     if ( inputData.isExport==true ) {
       searchQuery={
         'from': 0,
@@ -194,13 +195,21 @@ export async function getList( req, res ) {
           { 'storeName.keyword': { order: 'desc' } },
         ],
       };
+      // allResult =await fetchAllDocuments( openSearch.EyeTestInput, search );
     }
-    const result = await getOpenSearchData( openSearch.EyeTestInput, searchQuery );
-    const count = result?.body?.hits?.total?.value;
-    if ( !count || count == 0 ) {
-      return res.sendError( 'No data found', 204 );
-    }
-    const searchValue = result?.body?.hits?.hits;
+
+    const countQuery = {
+      query: {
+        bool: search,
+      },
+    };
+    const result = offset >= 10000 ? await fetchDeepPage( openSearch.EyeTestInput, search, limit, offset, sortBy, order, inputData?.searchAfter ) : await getOpenSearchData( openSearch.EyeTestInput, searchQuery );
+    logger.info( { result: result } );
+    const getCount = inputData.isExport==true? null:await getOpenSearchCount( openSearch.EyeTestInput, countQuery );
+    const count =inputData.isExport==true?null: getCount?.body?.count;
+
+    const searchValue = offset >= 10000 ? result : result?.body?.hits?.hits;
+    logger.info( { searchValue: searchValue } );
     if ( !searchValue || searchValue?.length == 0 ) {
       return res.sendError( 'No data found', 204 );
     }
@@ -229,7 +238,7 @@ export async function getList( req, res ) {
               'Queue ID': element?._source?.queueId,
               'Compliance Score': element?._source?.totalSteps>0 ? Math.round( ( element?._source?.coveredStepsAI/ element?._source?.totalSteps )*100 ): 0,
               'Steps Covered By AI': element?._source?.coveredStepsAI || 0,
-              'trustScore': element?._source?.trustScore || 0,
+              'Trust Score': element?._source?.trustScore || 0,
               'Visitor Type': element?._source?.visitorsType,
               'Test Duration': duration || '',
               'Audit Status': element?._source?.auditStatus || '',
@@ -245,7 +254,7 @@ export async function getList( req, res ) {
               'Engagement ID': element?._source?.engagementId,
               'Compliance Score': element?._source?.totalSteps>0 ? Math.round( ( element?._source?.coveredStepsAI/ element?._source?.totalSteps )*100 ): 0,
               'Steps Covered By AI': element?._source?.coveredStepsAI || 0,
-              'trustScore': element?._source?.trustScore ||0,
+              'Trust Score': element?._source?.trustScore ||0,
               'Test Duration': duration || '',
               'Audit Status': element?._source?.auditStatus || '',
               'Audited By': userName?.userName || '',
@@ -284,6 +293,102 @@ export async function getList( req, res ) {
   }
 }
 
+// export async function fetchAllDocuments( index, data ) {
+//   try {
+//     const pageSize = 1000; // safer chunk size
+//     let allHits = [];
+//     let searchAfter = null;
+//     let hasMore = true;
+//     let i =0;
+
+//     while ( hasMore ) {
+//       const body={
+//         size: pageSize,
+//         query: {
+//           bool: data,
+//         },
+//         sort: [
+//           { 'storeName.keyword': 'desc' },
+//           { '_id': 'asc' }, // Required for stable search_after
+//         ],
+//       };
+
+//       if ( searchAfter ) {
+//         body.search_after = searchAfter;
+//       }
+
+//       const response = await getOpenSearchData( index, body );
+//       const hits = response.body.hits.hits;
+//       i = i+hits.length;
+//       logger.info( { index: i, length: hits.length, msg: '........index' } );
+//       allHits.push( ...hits );
+//       if ( allHits.length >= 10000 ) {
+//         hasMore = false;
+//       } else {
+//         searchAfter = hits[hits.length - 1].sort;
+//       }
+//     }
+
+//     logger.info( `Fetched ${allHits.length} documents` );
+//     return allHits;
+//   } catch ( error ) {
+//     const err = error.message || 'Internal Server Error';
+//     return res.sendError( err, 500 );
+//   };
+// };
+export async function fetchDeepPage( index, data, limit, offset, sortBy, order, searchAfter1 ) {
+  try {
+    logger.info( { index: index, data: data, limit: limit, offset: offset } );
+    let searchAfter = searchAfter1 || null;
+    let currentCount = 0;
+    const body={
+      size: limit,
+      query: {
+        bool: data, // assuming your search object is here
+      },
+      sort: [
+        { [sortBy]: { order: order === -1 ? 'desc' : 'asc' } },
+        { _id: 'asc' },
+      ],
+      ...( searchAfter ? { search_after: searchAfter } : {} ),
+    };
+    logger.info( { offset: offset, limmit: limit, currentCount: currentCount } );
+
+    const response = await getOpenSearchData(
+        index,
+        body,
+    );
+
+    const hits = response?.body?.hits?.hits;
+    // while ( currentCount < offset + limit ) {
+    //   const response = await getOpenSearchData(
+    //       index,
+    //       body,
+    //   );
+
+    //   const hits = response?.body?.hits?.hits;
+    //   logger.info( { response: response } );
+    //   if ( !hits.length || offset + limit <= i ) break;
+    //   i = i+hits.length;
+    //   logger.info( { hits: hits, function: '.........hits' } );
+    //   currentCount += hits.length;
+    //   searchAfter = hits[hits.length - 1].sort;
+    //   logger.info( { searchAfter: searchAfter } );
+    //   // Only keep the target range
+    //   const startIndex = Math.max( 0, hits.length - ( currentCount - offset ) );
+    //   const sliced = hits.slice( startIndex, startIndex + limit );
+    //   if ( currentCount >= offset ) {
+    //     finalResults.push( ...sliced );
+    //   }
+    // }
+    return hits;
+  } catch ( error ) {
+    const err = error.message || 'Internal Server Error';
+    logger.error( { error: error, function: 'fetchDeepPage' } );
+    return err;
+  }
+}
+
 export async function viewFile( req, res ) {
   try {
     const inputData = req.params;

package/src/dtos/eyeTestAudit.dtos.js

@@ -26,6 +26,7 @@ export const getListSchema = joi.object(
       isExport: joi.boolean().optional(),
       sortBy: joi.string().optional(),
       sortOrder: joi.number().optional(),
+      searchAfter: joi.array().optional(),
     },
 );